@h-rig/runtime 0.0.6-alpha.0

package/dist/src/control-plane/memory-sync/embed.js

@@ -0,0 +1,281 @@
+// @bun
+// packages/runtime/src/control-plane/memory-sync/embed.ts
+import { createHash } from "crypto";
+
+// packages/runtime/src/control-plane/memory-sync/db.ts
+function parseJsonRecord(value) {
+  if (typeof value !== "string" || value.length === 0) {
+    return null;
+  }
+  return JSON.parse(value);
+}
+function parseEmbedding(value) {
+  if (typeof value !== "string" || value.length === 0) {
+    return null;
+  }
+  return JSON.parse(value);
+}
+function rowString(row, key) {
+  const value = row[key];
+  return value == null ? null : String(value);
+}
+function toMemoryItemRow(row) {
+  return {
+    canonicalKey: String(row.canonical_key),
+    summary: String(row.summary),
+    kind: rowString(row, "kind"),
+    category: rowString(row, "category"),
+    status: String(row.status),
+    confidence: Number(row.confidence),
+    sourceRunId: rowString(row, "source_run_id"),
+    sourceTaskId: rowString(row, "source_task_id"),
+    branch: rowString(row, "branch"),
+    details: parseJsonRecord(row.details_json),
+    createdAt: String(row.created_at),
+    updatedAt: String(row.updated_at),
+    lastEventId: String(row.last_event_id),
+    supersededBy: rowString(row, "superseded_by"),
+    embedding: parseEmbedding(row.embedding)
+  };
+}
+async function setMemoryItemEmbedding(db, canonicalKey, embedding) {
+  await db.client.execute({
+    sql: `
+      UPDATE memory_items
+      SET embedding = ?
+      WHERE canonical_key = ?
+    `,
+    args: [JSON.stringify(embedding), canonicalKey]
+  });
+}
+async function getMemoryItemByKey(db, canonicalKey) {
+  const result = await db.client.execute({
+    sql: `
+      SELECT
+        canonical_key,
+        summary,
+        kind,
+        category,
+        status,
+        confidence,
+        source_run_id,
+        source_task_id,
+        branch,
+        details_json,
+        created_at,
+        updated_at,
+        last_event_id,
+        superseded_by,
+        embedding
+      FROM memory_items
+      WHERE canonical_key = ?
+      LIMIT 1
+    `,
+    args: [canonicalKey]
+  });
+  const row = result.rows[0];
+  return row ? toMemoryItemRow(row) : null;
+}
+
+// packages/runtime/src/control-plane/memory-sync/embed.ts
+var DEFAULT_EMBEDDING_API_BASE_URL = "https://api.openai.com/v1";
+var DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small";
+var DETERMINISTIC_EMBEDDER_MODE = "deterministic";
+var DETERMINISTIC_VECTOR_DIMS = 32;
+var MISSING_MEMORY_EMBEDDER_WARNING = "Shared memory embeddings disabled: OPENAI_API_KEY is not set. Continuing with lexical-only memory.";
+var missingMemoryEmbedderWarningShown = false;
+function embeddingTextForItem(item) {
+  const parts = [item.summary, item.kind ?? "", item.category ?? "", item.details ? JSON.stringify(item.details) : ""];
+  return parts.filter(Boolean).join(`
+`);
+}
+function tokenizeForDeterministicEmbedding(text) {
+  const tokens = text.toLowerCase().match(/[a-z0-9_./:-]+/g) ?? [];
+  return tokens.length > 0 ? tokens : [text.toLowerCase()];
+}
+function deterministicTokenVector(token) {
+  const hash = createHash("sha256").update(token).digest();
+  const vector = new Array(DETERMINISTIC_VECTOR_DIMS).fill(0);
+  for (let index = 0;index < 8; index += 1) {
+    const slot = (hash[index] ?? 0) % DETERMINISTIC_VECTOR_DIMS;
+    const sign = ((hash[index + 8] ?? 0) & 1) === 0 ? 1 : -1;
+    const magnitude = ((hash[index + 16] ?? 0) + 1) / 256;
+    vector[slot] = (vector[slot] ?? 0) + sign * magnitude;
+  }
+  return vector;
+}
+function normalizeVector(vector) {
+  const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
+  if (norm === 0) {
+    return vector.map(() => 0);
+  }
+  return vector.map((value) => Number((value / norm).toFixed(6)));
+}
+function createDeterministicMemoryEmbedder() {
+  return {
+    async embed(texts) {
+      return texts.map((text) => {
+        const vector = new Array(DETERMINISTIC_VECTOR_DIMS).fill(0);
+        for (const token of tokenizeForDeterministicEmbedding(text)) {
+          const tokenVector = deterministicTokenVector(token);
+          for (let index = 0;index < tokenVector.length; index += 1) {
+            vector[index] = (vector[index] ?? 0) + (tokenVector[index] ?? 0);
+          }
+        }
+        return normalizeVector(vector);
+      });
+    }
+  };
+}
+function embeddingResponseMessage(payload, status) {
+  if (typeof payload === "object" && payload !== null) {
+    const maybeMessage = payload.error?.message;
+    if (maybeMessage) {
+      return maybeMessage;
+    }
+  }
+  return `memory embedding request failed with status ${status}`;
+}
+function createOpenAiMemoryEmbedder(options) {
+  const apiKey = options.apiKey.trim();
+  const model = options.model?.trim() || DEFAULT_EMBEDDING_MODEL;
+  const apiBaseUrl = (options.apiBaseUrl?.trim() || DEFAULT_EMBEDDING_API_BASE_URL).replace(/\/+$/, "");
+  const fetchImpl = options.fetchImpl ?? fetch;
+  if (!apiKey) {
+    throw new Error("memory embedding api key must be non-empty");
+  }
+  return {
+    async embed(texts) {
+      if (texts.length === 0) {
+        return [];
+      }
+      const response = await fetchImpl(`${apiBaseUrl}/embeddings`, {
+        method: "POST",
+        headers: {
+          authorization: `Bearer ${apiKey}`,
+          "content-type": "application/json"
+        },
+        body: JSON.stringify({
+          input: texts,
+          model
+        })
+      });
+      const payload = await response.json();
+      if (!response.ok) {
+        throw new Error(embeddingResponseMessage(payload, response.status));
+      }
+      if (!Array.isArray(payload.data) || payload.data.length !== texts.length) {
+        throw new Error(`memory embedding provider returned ${payload.data?.length ?? 0} embeddings for ${texts.length} inputs`);
+      }
+      const byIndex = new Map;
+      for (const row of payload.data) {
+        if (typeof row.index !== "number" || !Array.isArray(row.embedding)) {
+          throw new Error("memory embedding provider returned an invalid response payload");
+        }
+        byIndex.set(row.index, row.embedding);
+      }
+      return texts.map((_text, index) => {
+        const embedding = byIndex.get(index);
+        if (!embedding) {
+          throw new Error(`memory embedding provider omitted embedding ${index}`);
+        }
+        return embedding;
+      });
+    }
+  };
+}
+function createConfiguredMemoryEmbedder(options = {}) {
+  const env = options.env ?? process.env;
+  const mode = env.RIG_MEMORY_EMBEDDER?.trim();
+  if (mode === DETERMINISTIC_EMBEDDER_MODE) {
+    return createDeterministicMemoryEmbedder();
+  }
+  const apiKey = env.OPENAI_API_KEY?.trim();
+  if (!apiKey) {
+    throw new Error("memory embeddings require OPENAI_API_KEY or RIG_MEMORY_EMBEDDER=deterministic");
+  }
+  return createOpenAiMemoryEmbedder({
+    apiKey,
+    model: env.RIG_MEMORY_EMBEDDING_MODEL?.trim() || DEFAULT_EMBEDDING_MODEL,
+    apiBaseUrl: options.apiBaseUrl ?? env.RIG_MEMORY_EMBEDDING_API_BASE_URL?.trim() ?? DEFAULT_EMBEDDING_API_BASE_URL,
+    fetchImpl: options.fetchImpl
+  });
+}
+function warnMissingMemoryEmbedderOnce(warn) {
+  if (missingMemoryEmbedderWarningShown) {
+    return;
+  }
+  missingMemoryEmbedderWarningShown = true;
+  warn(MISSING_MEMORY_EMBEDDER_WARNING);
+}
+function resetMemoryEmbeddingWarningsForTests() {
+  missingMemoryEmbedderWarningShown = false;
+}
+function embeddingTargetsForEvent(event) {
+  switch (event.eventType) {
+    case "observed":
+    case "promoted":
+    case "corrected":
+    case "reinforced":
+      return [event.canonicalKey];
+    case "superseded":
+    case "retracted":
+    case "retrieved":
+    case "feedback_recorded":
+      return [];
+  }
+}
+function memoryEventRequiresEmbedding(event) {
+  return embeddingTargetsForEvent(event).length > 0;
+}
+function maybeCreateConfiguredMemoryEmbedder(options = {}, warn = (message) => {
+  console.warn(message);
+}) {
+  const env = options.env ?? process.env;
+  const mode = env.RIG_MEMORY_EMBEDDER?.trim();
+  if (mode === DETERMINISTIC_EMBEDDER_MODE) {
+    return createConfiguredMemoryEmbedder(options);
+  }
+  const apiKey = env.OPENAI_API_KEY?.trim();
+  if (!apiKey) {
+    warnMissingMemoryEmbedderOnce(warn);
+    return null;
+  }
+  return createConfiguredMemoryEmbedder(options);
+}
+async function embedChangedMemoryItems(db, event, embedder) {
+  const targets = [...new Set(embeddingTargetsForEvent(event))];
+  if (targets.length === 0) {
+    return [];
+  }
+  const items = [];
+  for (const canonicalKey of targets) {
+    const item = await getMemoryItemByKey(db, canonicalKey);
+    if (item && item.status === "active") {
+      items.push(item);
+    }
+  }
+  if (items.length === 0) {
+    return [];
+  }
+  const texts = items.map(embeddingTextForItem);
+  const embeddings = await embedder.embed(texts);
+  for (const [index, item] of items.entries()) {
+    const embedding = embeddings[index];
+    if (!embedding) {
+      throw new Error(`missing embedding for ${item.canonicalKey}`);
+    }
+    await setMemoryItemEmbedding(db, item.canonicalKey, embedding);
+  }
+  return texts;
+}
+export {
+  resetMemoryEmbeddingWarningsForTests,
+  memoryEventRequiresEmbedding,
+  maybeCreateConfiguredMemoryEmbedder,
+  embedChangedMemoryItems,
+  createOpenAiMemoryEmbedder,
+  createDeterministicMemoryEmbedder,
+  createConfiguredMemoryEmbedder,
+  MISSING_MEMORY_EMBEDDER_WARNING
+};