onbuzz 4.9.0 → 4.9.5

package/src/services/embeddings/indexers/memoryIndexer.js

@@ -0,0 +1,358 @@
+/**
+ * MemoryIndexer — keeps the per-agent `memory` vector store in sync with
+ * the agent's declared memories, and serves semantic / hybrid search.
+ *
+ * Architecture:
+ *   - Tool-driven (not event-driven): memoryTool calls indexOne / deleteOne
+ *     synchronously after each add/update/delete. Best-effort —
+ *     embedding failures NEVER block the underlying memory operation.
+ *   - Per-agent scope: each agent gets its own vector store file under
+ *     `<userDataDir>/state/embeddings/agents/<id>/memory.vec.json`.
+ *   - Hybrid search by default: semantic + substring fused via Reciprocal
+ *     Rank Fusion. Preserves exact-match precision (memory IDs, file
+ *     paths, identifiers) while adding paraphrase recall.
+ *
+ * Why no separate event-bus subscription: memoryService is a plain CRUD
+ * service today. Adding a pub/sub layer just for indexing would be more
+ * surface area than calling the indexer directly from the tool's
+ * existing dispatch path. If we later route memory writes through a
+ * bus for other reasons, the indexer can subscribe; until then this is
+ * the simpler shape.
+ *
+ * Failure policy:
+ *   - All public methods are best-effort. Errors are logged at warn
+ *     and a structured result is returned, but never thrown to the
+ *     caller. A failing embed call must NOT break a memory write —
+ *     the user's memory is the source of truth; the index is a cache.
+ */
+
+import { makeScoringFn } from '../vectorStore/scoring.js';
+
+/**
+ * Pack memory fields into the text the embedding model sees. Order
+ * matters: models attend strongest to the front, so title goes first,
+ * then description, then content. Empty fields are dropped to keep
+ * embeddings dense.
+ */
+export function memoryToText(memory) {
+  if (!memory) return '';
+  const parts = [];
+  if (memory.title)       parts.push(memory.title);
+  if (memory.description) parts.push(memory.description);
+  if (memory.content)     parts.push(memory.content);
+  return parts.join('\n\n');
+}
+
+export class MemoryIndexer {
+  /**
+   * @param {object} opts
+   * @param {import('../embeddingService.js').EmbeddingService} opts.embeddingService
+   * @param {object} [opts.logger]
+   */
+  constructor({ embeddingService, logger = null }) {
+    this._svc = embeddingService;
+    this._logger = logger;
+  }
+
+  /** Are embeddings configured AND enabled? Cheap guard for consumers. */
+  get isEnabled() {
+    return !!(this._svc && this._svc.isEnabled);
+  }
+
+  /**
+   * Index a single memory (or re-index after an update). Best-effort —
+   * embedding failures are swallowed with a warn log so the caller's
+   * memory write isn't disturbed.
+   *
+   * @param {string} agentId
+   * @param {object} memory   - Must have { id, title?, description?, content?, ... }
+   * @returns {Promise<{indexed: boolean, reason?: string}>}
+   */
+  async indexOne(agentId, memory) {
+    if (!this.isEnabled) return { indexed: false, reason: 'disabled' };
+    if (!agentId || !memory?.id) return { indexed: false, reason: 'invalid-input' };
+
+    try {
+      const text = memoryToText(memory);
+      if (!text) return { indexed: false, reason: 'empty-text' };
+
+      const [vec] = await this._svc.embedDocuments([text], 'document');
+      const store = await this._svc.getStore({ agentId, surface: 'memory' });
+      await store.upsert(memory.id, vec, this._buildMetadata(memory, text));
+      return { indexed: true };
+    } catch (err) {
+      this._logger?.warn?.('[memoryIndexer] indexOne failed', {
+        agentId, id: memory?.id, error: err?.message,
+      });
+      return { indexed: false, reason: 'error', error: err?.message };
+    }
+  }
+
+  /**
+   * Remove a memory from the index. Best-effort — a failed delete still
+   * succeeds at the memory level; the orphan vector will eventually
+   * disappear on the next backfill.
+   */
+  async deleteOne(agentId, memoryId) {
+    if (!this.isEnabled) return { deleted: false, reason: 'disabled' };
+    if (!agentId || !memoryId) return { deleted: false, reason: 'invalid-input' };
+    try {
+      const store = await this._svc.getStore({ agentId, surface: 'memory' });
+      await store.delete(memoryId);
+      return { deleted: true };
+    } catch (err) {
+      this._logger?.warn?.('[memoryIndexer] deleteOne failed', {
+        agentId, id: memoryId, error: err?.message,
+      });
+      return { deleted: false, reason: 'error', error: err?.message };
+    }
+  }
+
+  /**
+   * Bulk index. Returns counts. Used by backfill and any other
+   * indexing-many-at-once path. The embed call batches under the
+   * provider's maxBatchSize automatically.
+   *
+   * Skips memories without an id or without any indexable text.
+   *
+   * @param {string} agentId
+   * @param {object[]} memories
+   * @returns {Promise<{indexed: number, skipped: number, failed: number, error?: string}>}
+   */
+  async indexMany(agentId, memories) {
+    if (!this.isEnabled) return { indexed: 0, skipped: 0, failed: 0, error: 'disabled' };
+    if (!agentId || !Array.isArray(memories) || memories.length === 0) {
+      return { indexed: 0, skipped: 0, failed: 0 };
+    }
+
+    // Filter to valid + non-empty memories. Track skipped count so the
+    // caller can show "N of M indexed" honestly.
+    const valid = [];
+    let skipped = 0;
+    for (const m of memories) {
+      const text = memoryToText(m);
+      if (!m?.id || !text) { skipped++; continue; }
+      valid.push({ memory: m, text });
+    }
+    if (valid.length === 0) return { indexed: 0, skipped, failed: 0 };
+
+    let vectors;
+    try {
+      vectors = await this._svc.embedDocuments(valid.map(v => v.text), 'document');
+    } catch (err) {
+      this._logger?.warn?.('[memoryIndexer] indexMany embed failed', {
+        agentId, count: valid.length, error: err?.message,
+      });
+      return { indexed: 0, skipped, failed: valid.length, error: err?.message };
+    }
+
+    const rows = valid.map((v, i) => ({
+      id: v.memory.id,
+      vector: vectors[i],
+      metadata: this._buildMetadata(v.memory, v.text),
+    }));
+
+    try {
+      const store = await this._svc.getStore({ agentId, surface: 'memory' });
+      await store.upsertBatch(rows);
+      return { indexed: rows.length, skipped, failed: 0 };
+    } catch (err) {
+      this._logger?.warn?.('[memoryIndexer] indexMany upsert failed', {
+        agentId, count: rows.length, error: err?.message,
+      });
+      return { indexed: 0, skipped, failed: rows.length, error: err?.message };
+    }
+  }
+
+  /**
+   * Drop the agent's memory index and rebuild from the provided memories.
+   * Used after first-enable, after a model swap, or whenever the user
+   * triggers an explicit re-index. Idempotent.
+   */
+  async backfill(agentId, memories) {
+    if (!this.isEnabled) return { ok: false, reason: 'disabled' };
+    const store = await this._svc.getStore({ agentId, surface: 'memory' });
+    try {
+      await store.drop();
+    } catch (err) {
+      this._logger?.warn?.('[memoryIndexer] backfill drop failed', {
+        agentId, error: err?.message,
+      });
+      // Best-effort: continue, indexMany will overwrite individual rows.
+    }
+    // After drop, the cached store handle is still usable (drop empties
+    // the in-memory rows and removes the file; subsequent upserts recreate it).
+    const result = await this.indexMany(agentId, memories);
+    return { ok: true, ...result };
+  }
+
+  /**
+   * Semantic (or hybrid) search over the agent's memory.
+   *
+   * Applies a multi-signal scoring layer by default (recency boost via
+   * sigmoid 1-week half-life — matches the production-tested Mem0 /
+   * ChatGPT memory shape). Disable by passing `boost: false`.
+   *
+   * @param {string} agentId
+   * @param {string} query
+   * @param {object} [opts]
+   * @param {number} [opts.topK=10]
+   * @param {boolean} [opts.hybrid=true] - Mix substring scoring via RRF.
+   *        Default true because exact matches (memory IDs, paths)
+   *        otherwise score 0 on pure semantic and get buried.
+   * @param {boolean} [opts.boost=true] - Apply recency boost. Set false
+   *        when the caller wants raw cosine (e.g. evaluation harness).
+   * @returns {Promise<{enabled: boolean, hits: Array<{id, score, metadata}>}>}
+   */
+  async search(agentId, query, { topK = 10, hybrid = true, boost = true } = {}) {
+    if (!this.isEnabled) return { enabled: false, hits: [] };
+    if (typeof query !== 'string' || !query.trim()) {
+      return { enabled: true, hits: [] };
+    }
+    try {
+      const store = await this._svc.getStore({ agentId, surface: 'memory' });
+      const queryVec = await this._svc.embedQuery(query);
+      const scoring = boost ? makeScoringFn({ recency: true }) : null;
+      const hits = hybrid
+        ? await store.hybridQuery(queryVec, query, { topK, scoring })
+        : await store.query(queryVec, { topK, scoring });
+      return { enabled: true, hits };
+    } catch (err) {
+      this._logger?.warn?.('[memoryIndexer] search failed', {
+        agentId, error: err?.message,
+      });
+      // On embed failure, surface enabled:true so the consumer knows
+      // embeddings ARE configured (but this call failed); they may
+      // choose to retry vs. fall back vs. show the error.
+      return { enabled: true, hits: [], error: err?.message };
+    }
+  }
+
+  /** @private */
+  _buildMetadata(memory, text) {
+    return {
+      memoryId: memory.id,
+      text,                              // for hybrid substring path + display
+      title: memory.title || '',
+      description: memory.description || '',
+      // ISO timestamps so VectorStore.stats().lastIndexedAt works.
+      createdAt: memory.createdAt || new Date().toISOString(),
+      updatedAt: memory.updatedAt || memory.createdAt || new Date().toISOString(),
+    };
+  }
+
+  // ──────────────────────────────────────────────────────────────────
+  // Phase 6c — Team-pool memory
+  //
+  // The same indexer API mirrored at team scope. The vector store path
+  // is `<stateDir>/embeddings/teams/<teamId>/memory.vec.json`. Same
+  // chunking shape, same metadata fields, plus a `publishedBy` field
+  // recording which agent originated the entry.
+  //
+  // Team pools answer Q1 from the design review: per-agent memory is
+  // private by default, team pools are the explicit publish channel
+  // for knowledge an agent wants to share with collaborators. ACL
+  // (agent must be member of team) is enforced upstream in memoryTool
+  // — this indexer is just storage / retrieval.
+  //
+  // v1 limitation: team pools live ONLY in the vector store. Disabling
+  // embeddings makes them invisible. That's intentional — team memory
+  // is fundamentally a semantic-shared-knowledge feature; without
+  // embeddings, "shared knowledge" with no search is much less useful.
+  // A separate JSON canonical store can land later if product demands.
+  // ──────────────────────────────────────────────────────────────────
+
+  /**
+   * Publish a memory into a team's shared pool.
+   *
+   * @param {string} teamId
+   * @param {object} memory - { id, title?, description?, content?, createdAt? }
+   * @param {string} publishedBy - The agent id doing the publish.
+   * @returns {Promise<{indexed: boolean, reason?: string, error?: string}>}
+   */
+  async publishToTeam(teamId, memory, publishedBy) {
+    if (!this.isEnabled) return { indexed: false, reason: 'disabled' };
+    if (!teamId || !memory?.id) return { indexed: false, reason: 'invalid-input' };
+    const text = memoryToText(memory);
+    if (!text) return { indexed: false, reason: 'empty-text' };
+    try {
+      const [vec] = await this._svc.embedDocuments([text], 'document');
+      const store = await this._svc.getStore({ teamId, surface: 'memory' });
+      await store.upsert(memory.id, vec, this._buildTeamMetadata(memory, text, publishedBy));
+      return { indexed: true };
+    } catch (err) {
+      this._logger?.warn?.('[memoryIndexer] publishToTeam failed', {
+        teamId, id: memory?.id, error: err?.message,
+      });
+      return { indexed: false, reason: 'error', error: err?.message };
+    }
+  }
+
+  /**
+   * Remove a memory from a team's shared pool. Anyone in the team can
+   * unpublish at the indexer level — ACL gating belongs upstream.
+   */
+  async unpublishFromTeam(teamId, memoryId) {
+    if (!this.isEnabled) return { deleted: false, reason: 'disabled' };
+    if (!teamId || !memoryId) return { deleted: false, reason: 'invalid-input' };
+    try {
+      const store = await this._svc.getStore({ teamId, surface: 'memory' });
+      await store.delete(memoryId);
+      return { deleted: true };
+    } catch (err) {
+      this._logger?.warn?.('[memoryIndexer] unpublishFromTeam failed', {
+        teamId, id: memoryId, error: err?.message,
+      });
+      return { deleted: false, reason: 'error', error: err?.message };
+    }
+  }
+
+  /**
+   * Semantic (or hybrid) search over a team's memory pool. Same shape
+   * as `search()` but scoped to a single team. Recency boost applies
+   * by default — team pools age the same way personal memory does.
+   */
+  async searchTeam(teamId, query, { topK = 10, hybrid = true, boost = true } = {}) {
+    if (!this.isEnabled) return { enabled: false, hits: [] };
+    if (!teamId || typeof query !== 'string' || !query.trim()) {
+      return { enabled: true, hits: [] };
+    }
+    try {
+      const store = await this._svc.getStore({ teamId, surface: 'memory' });
+      const queryVec = await this._svc.embedQuery(query);
+      const scoring = boost ? makeScoringFn({ recency: true }) : null;
+      const hits = hybrid
+        ? await store.hybridQuery(queryVec, query, { topK, scoring })
+        : await store.query(queryVec, { topK, scoring });
+      return { enabled: true, hits };
+    } catch (err) {
+      this._logger?.warn?.('[memoryIndexer] searchTeam failed', {
+        teamId, error: err?.message,
+      });
+      return { enabled: true, hits: [], error: err?.message };
+    }
+  }
+
+  /** Stats for a team pool — used by the UI cost / inventory dashboard. */
+  async teamStats(teamId) {
+    if (!this.isEnabled) return { count: 0, enabled: false };
+    if (!teamId) return { count: 0, enabled: true, error: 'invalid-input' };
+    try {
+      const store = await this._svc.getStore({ teamId, surface: 'memory' });
+      const s = await store.stats();
+      return { ...s, enabled: true };
+    } catch (err) {
+      return { count: 0, enabled: true, error: err?.message };
+    }
+  }
+
+  /** @private — team-pool metadata: includes publishedBy / publishedAt. */
+  _buildTeamMetadata(memory, text, publishedBy) {
+    const base = this._buildMetadata(memory, text);
+    return {
+      ...base,
+      publishedBy: publishedBy || 'unknown',
+      publishedAt: new Date().toISOString(),
+    };
+  }
+}