open-agents-ai 0.187.344 → 0.187.346

package/dist/index.js

@@ -259301,6 +259301,7 @@ print(classes[top])
           textParts.push(`Speech: "${episode.audio.transcript}"`);
         episode.text = { content: textParts.join(". ") || "Multi-modal capture", entities: [] };
         this.saveEpisode(episode);
+        this.bridgeToEpisodeStore(episode);
         return {
           success: true,
           output: `Multi-modal episode captured (${episodeId}):
@@ -259379,6 +259380,7 @@ else:
             }
           }
           this.saveEpisode(episode);
+          this.bridgeToEpisodeStore(episode);
         }
         return {
           success: true,
@@ -259550,6 +259552,88 @@ ${lines.join("\n")}`,
         });
         writeFileSync17(MM_INDEX, JSON.stringify(index, null, 2));
       }
+      /**
+       * WO-AM-GAP-02: Bridge multimodal episode to SQLite EpisodeStore + TemporalGraph.
+       * Makes multimodal episodes visible to PPR retrieval, zettelkasten linking,
+       * and the orchestrator's context injection system.
+       */
+      bridgeToEpisodeStore(episode) {
+        try {
+          const { EpisodeStore: EpisodeStore3, TemporalGraph: TemporalGraph3 } = __require("@open-agents/memory");
+          const { existsSync: fsExists } = __require("node:fs");
+          const { join: pathJoin } = __require("node:path");
+          const cwd4 = process.cwd();
+          const oaDirs = [pathJoin(cwd4, ".oa_test"), pathJoin(cwd4, ".oa")];
+          const oaDir = oaDirs.find((d2) => fsExists(d2));
+          if (!oaDir)
+            return;
+          const es = new EpisodeStore3(pathJoin(oaDir, "memory.db"));
+          const tg = new TemporalGraph3(pathJoin(oaDir, "kg.db"));
+          const contentParts = [];
+          if (episode.social?.personName)
+            contentParts.push(`Met ${episode.social.personName}`);
+          if (episode.audio?.transcript)
+            contentParts.push(`Said: "${episode.audio.transcript}"`);
+          if (episode.audio?.soundClass)
+            contentParts.push(`Sound: ${episode.audio.soundClass}`);
+          if (episode.visual?.faceNames?.length)
+            contentParts.push(`Faces: ${episode.visual.faceNames.join(", ")}`);
+          if (episode.visual?.objects?.length)
+            contentParts.push(`Objects: ${episode.visual.objects.join(", ")}`);
+          if (episode.spatial?.locationLabel)
+            contentParts.push(`Location: ${episode.spatial.locationLabel}`);
+          const content = contentParts.join(". ") || episode.text?.content || "Multimodal capture";
+          const modality = episode.social?.personName ? "social" : episode.visual?.imagePath ? "visual" : episode.audio?.recordingPath ? "audio" : "text";
+          const epId = es.insert({
+            sessionId: episode.sessionId,
+            modality,
+            toolName: "multimodal_memory",
+            content,
+            importance: episode.social?.personName ? 9 : 7,
+            decayClass: episode.social?.personName ? "permanent" : "daily",
+            metadata: {
+              multimodal_episode_id: episode.id,
+              has_face: (episode.visual?.faceIds?.length ?? 0) > 0,
+              has_audio: !!episode.audio?.recordingPath,
+              has_gps: !!episode.spatial?.gps
+            }
+          });
+          if (episode.visual?.clipEmbedding) {
+            const emb = new Float32Array(episode.visual.clipEmbedding);
+            es.setEmbedding(epId, emb);
+          }
+          if (episode.social?.personName) {
+            const personId = tg.upsertNode({ text: episode.social.personName, nodeType: "person" });
+            tg.addEdge({
+              srcId: personId,
+              dstId: personId,
+              relation: "discovered_during",
+              fact: `Met ${episode.social.personName} via multimodal capture`,
+              edgeType: "triple",
+              sourceEpisodeId: epId,
+              modality
+            });
+          }
+          if (episode.spatial?.locationLabel) {
+            const locId = tg.upsertNode({ text: episode.spatial.locationLabel, nodeType: "location" });
+            if (episode.social?.personName) {
+              const personNode = tg.findNode(episode.social.personName, "person");
+              if (personNode) {
+                tg.addEdge({
+                  srcId: personNode.id,
+                  dstId: locId,
+                  relation: "appears_in",
+                  fact: `${episode.social.personName} seen at ${episode.spatial.locationLabel}`,
+                  sourceEpisodeId: epId
+                });
+              }
+            }
+          }
+          es.close();
+          tg.close();
+        } catch {
+        }
+      }
       loadEpisode(id) {
         const file = join52(MM_DIR, id, "episode.json");
         try {
@@ -267715,12 +267799,14 @@ function autoImportance(toolName, modality, content) {
     return 5;
   if (toolName?.startsWith("memory_"))
     return 6;
-  if (modality === "visual" || modality === "audio")
-    return 7;
   if (modality === "social")
-    return 8;
-  if (modality === "spatial")
+    return 9;
+  if (modality === "visual")
+    return 7;
+  if (modality === "audio")
     return 6;
+  if (modality === "spatial")
+    return 5;
   if (modality === "reflection")
     return 7;
   if (modality === "gist")
@@ -267738,7 +267824,13 @@ function autoDecayClass(toolName, modality, content) {
     return "daily";
   if (content.toLowerCase().includes("error") || content.toLowerCase().includes("failed"))
     return "procedural";
-  if (["visual", "audio", "social", "spatial"].includes(modality))
+  if (modality === "social")
+    return "permanent";
+  if (modality === "visual")
+    return "procedural";
+  if (modality === "spatial")
+    return "procedural";
+  if (modality === "audio")
     return "daily";
   if (modality === "reflection" || modality === "gist")
     return "procedural";
@@ -267793,6 +267885,10 @@ var init_episodeStore = __esm({
         this.db.exec(`CREATE INDEX IF NOT EXISTS idx_ep_tool ON episodes(tool_name)`);
         this.db.exec(`CREATE INDEX IF NOT EXISTS idx_ep_decay ON episodes(decay_class)`);
         this.db.exec(`CREATE INDEX IF NOT EXISTS idx_ep_hash ON episodes(content_hash, session_id)`);
+        try {
+          this.db.exec(`ALTER TABLE episodes ADD COLUMN clip_embedding BLOB`);
+        } catch {
+        }
         this.db.pragma("journal_mode = WAL");
         this.db.pragma("synchronous = NORMAL");
       }
@@ -267853,7 +267949,37 @@ var init_episodeStore = __esm({
         }
         sql += " ORDER BY timestamp DESC LIMIT ?";
         params.push(Math.min(limit * 5, 500));
-        const rows = this.db.prepare(sql).all(...params);
+        let rows = this.db.prepare(sql).all(...params);
+        if (query.metadataFilter || query.soundClass || query.rmsRange) {
+          rows = rows.filter((row) => {
+            let meta = null;
+            try {
+              meta = row.metadata ? typeof row.metadata === "string" ? JSON.parse(row.metadata) : row.metadata : null;
+            } catch {
+              return false;
+            }
+            if (!meta)
+              return false;
+            if (query.metadataFilter) {
+              for (const [k, v] of Object.entries(query.metadataFilter)) {
+                if (meta[k] !== v)
+                  return false;
+              }
+            }
+            if (query.soundClass && meta.sound_class !== query.soundClass)
+              return false;
+            if (query.rmsRange) {
+              const rms = meta.rms_db ?? meta.rmsDb;
+              if (typeof rms !== "number")
+                return false;
+              if (query.rmsRange.min !== void 0 && rms < query.rmsRange.min)
+                return false;
+              if (query.rmsRange.max !== void 0 && rms > query.rmsRange.max)
+                return false;
+            }
+            return true;
+          });
+        }
         const scored = rows.map((row) => {
           const ep = this.rowToEpisode(row);
           const tau = DECAY_TAU[ep.decayClass] ?? DECAY_TAU.daily;
@@ -267883,7 +268009,8 @@ var init_episodeStore = __esm({
             emb = Math.max(0, Math.min(1, (emb + 1) / 2));
           }
           const relevance = wLex * lex + wEmb * emb;
-          const score = recency + importance + relevance;
+          const strengthBonus = Math.min(1, Math.log2(Math.max(1, ep.strength)));
+          const score = recency + importance + relevance + strengthBonus;
           return { episode: ep, score };
         });
         scored.sort((a2, b) => b.score - a2.score);
@@ -267929,6 +268056,11 @@ var init_episodeStore = __esm({
         const buf = Buffer.from(embedding.buffer);
         this.db.prepare("UPDATE episodes SET embedding = ? WHERE id = ?").run(buf, id);
       }
+      /** WO-AM-GAP-03: Set CLIP embedding for cross-modal retrieval (512d). */
+      setClipEmbedding(id, clipEmbedding) {
+        const buf = Buffer.from(clipEmbedding.buffer);
+        this.db.prepare("UPDATE episodes SET clip_embedding = ? WHERE id = ?").run(buf, id);
+      }
       /** Update gist for an episode (from WO-AM-07 gist compression). */
       setGist(id, gist) {
         this.db.prepare("UPDATE episodes SET gist = ? WHERE id = ?").run(gist, id);
@@ -267961,6 +268093,7 @@ var init_episodeStore = __esm({
           decayClass: row.decay_class,
           strength: typeof row.strength === "number" && Number.isFinite(row.strength) ? row.strength : 1,
           lastRetrieved: row.last_retrieved,
+          clipEmbedding: row.clip_embedding ? new Float32Array(new Uint8Array(row.clip_embedding).buffer) : null,
           gist: row.gist,
           sourceEpisodeId: row.source_episode_id
         };
@@ -268989,6 +269122,319 @@ var init_proceduralMemoryStore = __esm({
   }
 });
 
+// packages/memory/dist/splanifold.js
+function h00(t2) {
+  return 2 * t2 * t2 * t2 - 3 * t2 * t2 + 1;
+}
+function h01(t2) {
+  return -2 * t2 * t2 * t2 + 3 * t2 * t2;
+}
+function h10(t2) {
+  return t2 * (t2 - 1) * (t2 - 1);
+}
+function h11(t2) {
+  return t2 * t2 * (t2 - 1);
+}
+function evaluate(controls, r2) {
+  const { K: K2, N, P: P2, E: Eb, Pd, Ed, expansion, epsSigma, deltaMax } = controls;
+  const eps2 = epsSigma * epsSigma;
+  const u = new Float64Array(K2);
+  const scale = 1 + 2 * expansion;
+  for (let k = 0; k < K2; k++) {
+    u[k] = scale * r2[k] - expansion;
+  }
+  let sigma = 0;
+  for (let k = 0; k < K2; k++)
+    sigma += u[k];
+  const t2 = sigma / K2;
+  const delta = new Float64Array(K2);
+  for (let k = 0; k < K2; k++)
+    delta[k] = u[k] - t2;
+  let deltaTilde = delta;
+  if (isFinite(deltaMax)) {
+    const normSq = delta.reduce((s2, d2) => s2 + d2 * d2, 0);
+    const norm = Math.sqrt(normSq);
+    if (norm > 0) {
+      const warpScale = 1 / Math.sqrt(1 + (norm / deltaMax) ** 2);
+      deltaTilde = new Float64Array(K2);
+      for (let k = 0; k < K2; k++)
+        deltaTilde[k] = delta[k] * warpScale;
+    }
+  }
+  const w = new Float64Array(K2);
+  const denom = sigma * sigma + eps2;
+  for (let k = 0; k < K2; k++) {
+    w[k] = (sigma * u[k] + eps2 / K2) / denom;
+  }
+  const V0 = matVecColMajor(Pd[0], N, K2, w);
+  const V1 = matVecColMajor(Pd[1], N, K2, w);
+  const spine = hermite(P2[0], P2[1], V0, V1, t2, N);
+  const D0 = matVecColMajor(Eb[0], N, K2, deltaTilde);
+  const D1 = matVecColMajor(Eb[1], N, K2, deltaTilde);
+  const T0 = matVecColMajor(Ed[0], N, K2, deltaTilde);
+  const T1 = matVecColMajor(Ed[1], N, K2, deltaTilde);
+  const trans = hermite(D0, D1, T0, T1, t2, N);
+  const out = new Float64Array(N);
+  for (let n2 = 0; n2 < N; n2++)
+    out[n2] = spine[n2] + trans[n2];
+  return out;
+}
+function hermite(A, B, Ad, Bd, t2, N) {
+  const h0 = h00(t2), h1 = h01(t2), d0 = h10(t2), d1 = h11(t2);
+  const out = new Float64Array(N);
+  for (let n2 = 0; n2 < N; n2++) {
+    out[n2] = h0 * A[n2] + h1 * B[n2] + d0 * Ad[n2] + d1 * Bd[n2];
+  }
+  return out;
+}
+function matVecColMajor(M, N, K2, x) {
+  const y = new Float64Array(N);
+  for (let k = 0; k < K2; k++) {
+    const xk = x[k];
+    const offset = k * N;
+    for (let n2 = 0; n2 < N; n2++) {
+      y[n2] += M[offset + n2] * xk;
+    }
+  }
+  return y;
+}
+function fit(samples, K2, N, config) {
+  const expansion = config?.expansion ?? 0;
+  const epsSigma = config?.epsSigma ?? 0.02;
+  const deltaMax = config?.deltaMax ?? Infinity;
+  const lambda = config?.ridgeLambda ?? 1e-4;
+  const eps2 = epsSigma * epsSigma;
+  const S = samples.length;
+  const cols = 2 + 6 * K2;
+  const F = new Float64Array(cols * S);
+  const X = new Float64Array(N * S);
+  const scale = 1 + 2 * expansion;
+  for (let j = 0; j < S; j++) {
+    const r2 = samples[j].input;
+    const x = samples[j].output;
+    const u = new Float64Array(K2);
+    for (let k = 0; k < K2; k++)
+      u[k] = scale * r2[k] - expansion;
+    let sigma = 0;
+    for (let k = 0; k < K2; k++)
+      sigma += u[k];
+    const t2 = sigma / K2;
+    const deltaTilde = new Float64Array(K2);
+    for (let k = 0; k < K2; k++)
+      deltaTilde[k] = u[k] - t2;
+    if (isFinite(deltaMax)) {
+      const normSq = deltaTilde.reduce((s2, d2) => s2 + d2 * d2, 0);
+      const norm = Math.sqrt(normSq);
+      if (norm > 0) {
+        const warpScale = 1 / Math.sqrt(1 + (norm / deltaMax) ** 2);
+        for (let k = 0; k < K2; k++)
+          deltaTilde[k] *= warpScale;
+      }
+    }
+    const w = new Float64Array(K2);
+    const denom = sigma * sigma + eps2;
+    for (let k = 0; k < K2; k++) {
+      w[k] = (sigma * u[k] + eps2 / K2) / denom;
+    }
+    const a0 = h00(t2), a1 = h01(t2), b0 = h10(t2), b1 = h11(t2);
+    let col2 = 0;
+    F[col2 * S + j] = a0;
+    col2++;
+    F[col2 * S + j] = a1;
+    col2++;
+    for (let k = 0; k < K2; k++) {
+      F[col2 * S + j] = b0 * w[k];
+      col2++;
+    }
+    for (let k = 0; k < K2; k++) {
+      F[col2 * S + j] = b1 * w[k];
+      col2++;
+    }
+    for (let k = 0; k < K2; k++) {
+      F[col2 * S + j] = a0 * deltaTilde[k];
+      col2++;
+    }
+    for (let k = 0; k < K2; k++) {
+      F[col2 * S + j] = a1 * deltaTilde[k];
+      col2++;
+    }
+    for (let k = 0; k < K2; k++) {
+      F[col2 * S + j] = b0 * deltaTilde[k];
+      col2++;
+    }
+    for (let k = 0; k < K2; k++) {
+      F[col2 * S + j] = b1 * deltaTilde[k];
+      col2++;
+    }
+    for (let n2 = 0; n2 < N; n2++)
+      X[n2 * S + j] = x[n2];
+  }
+  const G = new Float64Array(cols * cols);
+  for (let i2 = 0; i2 < cols; i2++) {
+    for (let jj = i2; jj < cols; jj++) {
+      let dot = 0;
+      for (let s2 = 0; s2 < S; s2++)
+        dot += F[i2 * S + s2] * F[jj * S + s2];
+      G[i2 * cols + jj] = dot;
+      G[jj * cols + i2] = dot;
+    }
+    G[i2 * cols + i2] += lambda;
+  }
+  const Ginv = invertSPD(G, cols);
+  const XFt = new Float64Array(N * cols);
+  for (let n2 = 0; n2 < N; n2++) {
+    for (let c8 = 0; c8 < cols; c8++) {
+      let dot = 0;
+      for (let s2 = 0; s2 < S; s2++)
+        dot += X[n2 * S + s2] * F[c8 * S + s2];
+      XFt[n2 * cols + c8] = dot;
+    }
+  }
+  const C = new Float64Array(N * cols);
+  for (let n2 = 0; n2 < N; n2++) {
+    for (let c8 = 0; c8 < cols; c8++) {
+      let dot = 0;
+      for (let cc = 0; cc < cols; cc++)
+        dot += XFt[n2 * cols + cc] * Ginv[cc * cols + c8];
+      C[n2 * cols + c8] = dot;
+    }
+  }
+  const extractCol = (idx) => {
+    const v = new Float64Array(N);
+    for (let n2 = 0; n2 < N; n2++)
+      v[n2] = C[n2 * cols + idx];
+    return v;
+  };
+  const extractMat = (startCol, numCols) => {
+    const M = new Float64Array(N * numCols);
+    for (let k = 0; k < numCols; k++) {
+      for (let n2 = 0; n2 < N; n2++) {
+        M[k * N + n2] = C[n2 * cols + startCol + k];
+      }
+    }
+    return M;
+  };
+  let col = 0;
+  const P0 = extractCol(col++);
+  const P1 = extractCol(col++);
+  const Pd0 = extractMat(col, K2);
+  col += K2;
+  const Pd1 = extractMat(col, K2);
+  col += K2;
+  const E0 = extractMat(col, K2);
+  col += K2;
+  const E1 = extractMat(col, K2);
+  col += K2;
+  const Ed0 = extractMat(col, K2);
+  col += K2;
+  const Ed1 = extractMat(col, K2);
+  col += K2;
+  return {
+    K: K2,
+    N,
+    P: [P0, P1],
+    E: [E0, E1],
+    Pd: [Pd0, Pd1],
+    Ed: [Ed0, Ed1],
+    expansion,
+    epsSigma,
+    deltaMax
+  };
+}
+function invertSPD(A, n2) {
+  const L = new Float64Array(n2 * n2);
+  for (let i2 = 0; i2 < n2; i2++) {
+    for (let j = 0; j <= i2; j++) {
+      let sum = A[i2 * n2 + j];
+      for (let k = 0; k < j; k++)
+        sum -= L[i2 * n2 + k] * L[j * n2 + k];
+      if (i2 === j) {
+        if (sum <= 0) {
+          const Areg = new Float64Array(A);
+          for (let ii = 0; ii < n2; ii++)
+            Areg[ii * n2 + ii] += 1e-6;
+          return invertSPD(Areg, n2);
+        }
+        L[i2 * n2 + j] = Math.sqrt(sum);
+      } else {
+        L[i2 * n2 + j] = sum / L[j * n2 + j];
+      }
+    }
+  }
+  const Ainv = new Float64Array(n2 * n2);
+  const col = new Float64Array(n2);
+  for (let c8 = 0; c8 < n2; c8++) {
+    col.fill(0);
+    for (let i2 = 0; i2 < n2; i2++) {
+      let sum = i2 === c8 ? 1 : 0;
+      for (let k = 0; k < i2; k++)
+        sum -= L[i2 * n2 + k] * col[k];
+      col[i2] = sum / L[i2 * n2 + i2];
+    }
+    for (let i2 = n2 - 1; i2 >= 0; i2--) {
+      let sum = col[i2];
+      for (let k = i2 + 1; k < n2; k++)
+        sum -= L[k * n2 + i2] * col[k];
+      col[i2] = sum / L[i2 * n2 + i2];
+    }
+    for (let i2 = 0; i2 < n2; i2++)
+      Ainv[i2 * n2 + c8] = col[i2];
+  }
+  return Ainv;
+}
+function serialize(controls) {
+  return {
+    K: controls.K,
+    N: controls.N,
+    P: [Array.from(controls.P[0]), Array.from(controls.P[1])],
+    E: [Array.from(controls.E[0]), Array.from(controls.E[1])],
+    Pd: [Array.from(controls.Pd[0]), Array.from(controls.Pd[1])],
+    Ed: [Array.from(controls.Ed[0]), Array.from(controls.Ed[1])],
+    expansion: controls.expansion,
+    epsSigma: controls.epsSigma,
+    deltaMax: controls.deltaMax
+  };
+}
+function deserialize(obj) {
+  return {
+    K: obj.K,
+    N: obj.N,
+    P: [obj.P[0], obj.P[1]].map((a2) => new Float64Array(a2)),
+    E: [obj.E[0], obj.E[1]].map((a2) => new Float64Array(a2)),
+    Pd: [obj.Pd[0], obj.Pd[1]].map((a2) => new Float64Array(a2)),
+    Ed: [obj.Ed[0], obj.Ed[1]].map((a2) => new Float64Array(a2)),
+    expansion: obj.expansion,
+    epsSigma: obj.epsSigma,
+    deltaMax: obj.deltaMax
+  };
+}
+function rmse(controls, samples) {
+  let sumSq = 0;
+  for (const s2 of samples) {
+    const pred = evaluate(controls, s2.input);
+    for (let n2 = 0; n2 < controls.N; n2++) {
+      const diff = pred[n2] - s2.output[n2];
+      sumSq += diff * diff;
+    }
+  }
+  return Math.sqrt(sumSq / (samples.length * controls.N));
+}
+function cosine(a2, b) {
+  let dot = 0, na = 0, nb = 0;
+  for (let i2 = 0; i2 < a2.length; i2++) {
+    dot += a2[i2] * b[i2];
+    na += a2[i2] * a2[i2];
+    nb += b[i2] * b[i2];
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom > 0 ? dot / denom : 0;
+}
+var init_splanifold = __esm({
+  "packages/memory/dist/splanifold.js"() {
+    "use strict";
+  }
+});
+
 // packages/memory/dist/index.js
 var dist_exports3 = {};
 __export(dist_exports3, {
@@ -269018,7 +269464,13 @@ __export(dist_exports3, {
   initDb: () => initDb,
   linkEpisode: () => linkEpisode,
   personalizedPageRank: () => personalizedPageRank,
-  retrieveByPPR: () => retrieveByPPR
+  retrieveByPPR: () => retrieveByPPR,
+  splanifoldCosine: () => cosine,
+  splanifoldDeserialize: () => deserialize,
+  splanifoldEvaluate: () => evaluate,
+  splanifoldFit: () => fit,
+  splanifoldRmse: () => rmse,
+  splanifoldSerialize: () => serialize
 });
 var init_dist7 = __esm({
   "packages/memory/dist/index.js"() {
@@ -269038,6 +269490,7 @@ var init_dist7 = __esm({
     init_pprRetrieval();
     init_embeddings();
     init_proceduralMemoryStore();
+    init_splanifold();
   }
 });
 
@@ -269706,6 +270159,19 @@ function repairJson(raw) {
   }
   return found ? result : null;
 }
+function inferEpisodeModality(toolName) {
+  if (VISUAL_TOOLS.has(toolName))
+    return "visual";
+  if (AUDIO_TOOLS.has(toolName))
+    return "audio";
+  if (SOCIAL_TOOLS.has(toolName))
+    return "social";
+  if (SPATIAL_TOOLS.has(toolName))
+    return "spatial";
+  if (CODE_TOOLS.has(toolName))
+    return "code";
+  return "tool_result";
+}
 function getSystemPromptForTier(tier) {
   switch (tier) {
     case "small":
@@ -269746,7 +270212,7 @@ ${todoItems}
 </system-reminder>`;
   return { shouldInject: true, content, reason: "injected" };
 }
-var SYSTEM_PROMPT, SYSTEM_PROMPT_MEDIUM, SYSTEM_PROMPT_SMALL, AgenticRunner, OllamaAgenticBackend;
+var SYSTEM_PROMPT, SYSTEM_PROMPT_MEDIUM, SYSTEM_PROMPT_SMALL, VISUAL_TOOLS, AUDIO_TOOLS, SOCIAL_TOOLS, SPATIAL_TOOLS, CODE_TOOLS, AgenticRunner, OllamaAgenticBackend;
 var init_agenticRunner = __esm({
   "packages/orchestrator/dist/agenticRunner.js"() {
     "use strict";
@@ -269764,6 +270230,11 @@ var init_agenticRunner = __esm({
     SYSTEM_PROMPT = loadPrompt("agentic/system-large.md");
     SYSTEM_PROMPT_MEDIUM = loadPrompt("agentic/system-medium.md");
     SYSTEM_PROMPT_SMALL = loadPrompt("agentic/system-small.md");
+    VISUAL_TOOLS = /* @__PURE__ */ new Set(["vision", "camera_capture", "image_read", "screenshot", "ocr", "ocr_image_advanced", "visual_memory", "desktop_describe", "ocr_pdf", "generate_image"]);
+    AUDIO_TOOLS = /* @__PURE__ */ new Set(["audio_capture", "audio_analyze", "asr_listen", "transcribe_file", "transcribe_url", "audio_playback", "youtube_download"]);
+    SOCIAL_TOOLS = /* @__PURE__ */ new Set(["multimodal_memory", "send_message", "jibberlink"]);
+    SPATIAL_TOOLS = /* @__PURE__ */ new Set(["gps_location", "bluetooth_scan", "wifi_control", "sdr_scan", "meshtastic"]);
+    CODE_TOOLS = /* @__PURE__ */ new Set(["file_write", "file_edit", "file_patch", "batch_edit", "code_sandbox", "repl_exec", "notebook_edit"]);
     AgenticRunner = class {
       backend;
       tools = /* @__PURE__ */ new Map();
@@ -271582,17 +272053,33 @@ ${cachedEntry2.result.slice(0, 500)}` : `[BLOCKED — the observer confirmed thi
               if (this._episodeStore) {
                 try {
                   const episodeContent = result.success ? `${tc.name}: ${(result.output ?? "").slice(0, 500)}` : `${tc.name} ERROR: ${(result.error ?? result.output ?? "").slice(0, 500)}`;
+                  const episodeModality = inferEpisodeModality(tc.name);
+                  let episodeMetadata = {
+                    args_fingerprint: argsKey.slice(0, 200),
+                    success: result.success,
+                    duration_ms: performance.now() - toolStart
+                  };
+                  if (result.success && ["audio_analyze", "audio_capture"].includes(tc.name)) {
+                    try {
+                      const parsed = JSON.parse(result.output ?? "{}");
+                      if (parsed.rms_db !== void 0)
+                        episodeMetadata.rms_db = parsed.rms_db;
+                      if (parsed.classifications)
+                        episodeMetadata.sound_class = parsed.classifications[0]?.class;
+                      if (parsed.peak_frequencies)
+                        episodeMetadata.peak_frequencies = parsed.peak_frequencies.slice(0, 3);
+                      if (parsed.band_energy_db)
+                        episodeMetadata.band_energy = parsed.band_energy_db;
+                    } catch {
+                    }
+                  }
                   const episodeId = this._episodeStore.insert({
                     sessionId: this._sessionId,
                     turnNumber: turn,
-                    modality: "tool_result",
+                    modality: episodeModality,
                     toolName: tc.name,
                     content: episodeContent,
-                    metadata: {
-                      args_fingerprint: argsKey.slice(0, 200),
-                      success: result.success,
-                      duration_ms: performance.now() - toolStart
-                    }
+                    metadata: episodeMetadata
                   });
                   if (this._embeddingAvailable && episodeContent.length > 20) {
                     this._pendingEmbeddings.push({ id: episodeId, content: episodeContent.slice(0, 500) });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.187.344",
+  "version": "0.187.346",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) \u2014 interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",