@openenthrium/oe-runtime-sdk 1.7.5

package/src/utils/rag/chroma.js

@@ -0,0 +1,82 @@
+const { embed } = require("../../providers/embedding");
+const getSetting = require("./_settings");
+
+let _client = null;
+let _clientKey = null;
+
+async function getClient(cfg = {}) {
+  const url    = cfg.url    || (await getSetting("vector_db_url"))     || "http://localhost:8000";
+  const apiKey = cfg.apiKey || (await getSetting("vector_db_api_key")) || "";
+  const key = `${url}:${apiKey}`;
+  if (!_client || _clientKey !== key) {
+    const { ChromaClient } = await import("chromadb");
+    _client = new ChromaClient({
+      path: url,
+      ...(apiKey ? { auth: { provider: "token", credentials: apiKey } } : {}),
+    });
+    _clientKey = key;
+  }
+  return _client;
+}
+
+function colName(workspaceSlug) {
+  return `ws_${workspaceSlug.replace(/-/g, "_")}`;
+}
+
+async function upsertChunksBatched(workspaceSlug, documentUid, chunks, options = {}, config = {}) {
+  const { batchSize = 50, onProgress, shouldCancel } = options;
+  const client     = await getClient(config);
+  const collection = await client.getOrCreateCollection({ name: colName(workspaceSlug) });
+
+  let chunksProcessed = 0, embeddingTokens = 0, embeddingModel = null;
+
+  for (let i = 0; i < chunks.length; i += batchSize) {
+    if (shouldCancel && await shouldCancel()) return { chunksProcessed, embeddingTokens, embeddingModel };
+
+    const batch = chunks.slice(i, i + batchSize);
+    const { embeddings, tokensUsed, model } = await embed(batch.map(c => c.text));
+    embeddingTokens += tokensUsed;
+    if (!embeddingModel) embeddingModel = model;
+
+    await collection.upsert({
+      ids:        batch.map((_, j) => `${documentUid}_${i + j}`),
+      documents:  batch.map(c => c.text),
+      embeddings,
+      metadatas:  batch.map(c => ({ documentUid, ...(c.metadata || {}) })),
+    });
+
+    chunksProcessed += batch.length;
+    if (onProgress) await onProgress(chunksProcessed);
+  }
+  return { chunksProcessed, embeddingTokens, embeddingModel };
+}
+
+async function similaritySearch(workspaceSlug, query, topK = 5, config = {}) {
+  try {
+    const client     = await getClient(config);
+    const collection = await client.getOrCreateCollection({ name: colName(workspaceSlug) });
+    const { embeddings } = await embed([query]);
+    const results = await collection.query({
+      queryEmbeddings: [embeddings[0]],
+      nResults: topK,
+      include: ["documents", "metadatas", "distances"],
+    });
+    return (results.ids[0] || []).map((_, i) => ({
+      text:     results.documents[0][i],
+      metadata: results.metadatas[0][i] || {},
+      score:    1 - (results.distances[0][i] || 0),
+    }));
+  } catch {
+    return [];
+  }
+}
+
+async function deleteDocumentChunks(workspaceSlug, documentUid, config = {}) {
+  try {
+    const client     = await getClient(config);
+    const collection = await client.getOrCreateCollection({ name: colName(workspaceSlug) });
+    await collection.delete({ where: { documentUid } });
+  } catch {}
+}
+
+module.exports = { upsertChunksBatched, similaritySearch, deleteDocumentChunks };

package/src/utils/rag/lancedb.js

@@ -0,0 +1,103 @@
+const path = require("path");
+const { embed } = require("../../providers/embedding");
+
+let lancedb;
+async function getDb() {
+  if (!lancedb) lancedb = await import("@lancedb/lancedb");
+  const dbPath = path.join(__dirname, "../../../storage/lancedb");
+  return lancedb.connect(dbPath);
+}
+
+async function ensureFtsIndex(table) {
+  try {
+    const indices = await table.listIndices();
+    if (!indices.some(idx => idx.indexType === "FTS")) {
+      await table.createIndex("text", { config: lancedb.Index.fts() });
+    }
+  } catch {}
+}
+
+async function upsertChunksBatched(workspaceSlug, documentUid, chunks, options = {}) {
+  const { batchSize = 10, onProgress, shouldCancel } = options;
+  const db = await getDb();
+  const tableName = `ws_${workspaceSlug.replace(/-/g, "_")}`;
+
+  let chunksProcessed = 0;
+  let embeddingTokens  = 0;
+  let embeddingModel   = null;
+
+  for (let i = 0; i < chunks.length; i += batchSize) {
+    if (shouldCancel && await shouldCancel()) return { chunksProcessed, embeddingTokens, embeddingModel };
+
+    const batch = chunks.slice(i, i + batchSize);
+    const { embeddings, tokensUsed, model } = await embed(batch.map(c => c.text));
+
+    embeddingTokens += tokensUsed;
+    if (!embeddingModel) embeddingModel = model;
+
+    const records = batch.map((c, j) => ({
+      id:          `${documentUid}_${i + j}`,
+      documentUid,
+      text:        c.text,
+      metadata:    JSON.stringify(c.metadata || {}),
+      vector:      embeddings[j],
+    }));
+
+    try {
+      const table = await db.openTable(tableName);
+      await table.add(records);
+    } catch {
+      const table = await db.createTable(tableName, records);
+      await ensureFtsIndex(table);
+    }
+
+    chunksProcessed += batch.length;
+    if (onProgress) await onProgress(chunksProcessed);
+  }
+
+  try {
+    const table = await db.openTable(tableName);
+    await ensureFtsIndex(table);
+  } catch {}
+
+  return { chunksProcessed, embeddingTokens, embeddingModel };
+}
+
+async function similaritySearch(workspaceSlug, query, topK = 5) {
+  const db = await getDb();
+  const tableName = `ws_${workspaceSlug.replace(/-/g, "_")}`;
+  try {
+    const { embeddings } = await embed([query]);
+    const table = await db.openTable(tableName);
+    const fetchK = Math.max(topK * 3, 20);
+
+    const [vectorResults, ftsResults] = await Promise.all([
+      table.search(embeddings[0]).limit(fetchK).toArray(),
+      table.search(query, "fts").limit(fetchK).toArray().catch(() => []),
+    ]);
+
+    const RRF_K = 60;
+    const scores = {}, texts = {}, metas = {};
+    vectorResults.forEach((r, i) => { scores[r.id] = (scores[r.id] || 0) + 1 / (RRF_K + i + 1); texts[r.id] = r.text; metas[r.id] = r.metadata; });
+    ftsResults.forEach((r, i)    => { scores[r.id] = (scores[r.id] || 0) + 1 / (RRF_K + i + 1); texts[r.id] = r.text; metas[r.id] = r.metadata; });
+
+    return Object.entries(scores)
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, topK)
+      .map(([id, score]) => ({ text: texts[id], metadata: JSON.parse(metas[id] || "{}"), score }));
+  } catch (err) {
+    console.error(`[lancedb] similaritySearch failed for workspace "${workspaceSlug}":`, err?.message || err);
+    return [];
+  }
+}
+
+async function deleteDocumentChunks(workspaceSlug, documentUid) {
+  const db = await getDb();
+  const tableName = `ws_${workspaceSlug.replace(/-/g, "_")}`;
+  try {
+    const table = await db.openTable(tableName);
+    await table.delete(`documentUid = '${documentUid}'`);
+  } catch {}
+}
+
+module.exports = { upsertChunksBatched, similaritySearch, deleteDocumentChunks };

package/src/utils/rag/milvus.js

@@ -0,0 +1,116 @@
+const { embed } = require("../../providers/embedding");
+const getSetting = require("./_settings");
+
+let _client = null, _clientKey = null;
+
+async function getClient(cfg = {}) {
+  const address = cfg.url    || (await getSetting("vector_db_url"))     || "localhost:19530";
+  const token   = cfg.apiKey || (await getSetting("vector_db_api_key")) || "";
+  const key = `${address}:${token}`;
+  if (!_client || _clientKey !== key) {
+    const { MilvusClient } = require("@zilliz/milvus2-sdk-node");
+    _client    = new MilvusClient({ address, ...(token ? { token } : {}) });
+    _clientKey = key;
+  }
+  return _client;
+}
+
+function colName(workspaceSlug) {
+  return `ws_${workspaceSlug.replace(/[-]/g, "_")}`;
+}
+
+async function ensureCollection(client, name, dim) {
+  const exists = await client.hasCollection({ collection_name: name });
+  if (!exists.value) {
+    await client.createCollection({
+      collection_name: name,
+      fields: [
+        { name: "id",           data_type: "VarChar", max_length: 128, is_primary_key: true },
+        { name: "documentUid", data_type: "VarChar", max_length: 128 },
+        { name: "text",         data_type: "VarChar", max_length: 65535 },
+        { name: "metadata",     data_type: "VarChar", max_length: 65535 },
+        { name: "embedding",    data_type: "FloatVector", dim },
+      ],
+    });
+    await client.createIndex({
+      collection_name: name,
+      field_name:      "embedding",
+      index_type:      "HNSW",
+      metric_type:     "COSINE",
+      params:          { M: 16, efConstruction: 200 },
+    });
+    await client.loadCollection({ collection_name: name });
+  }
+}
+
+async function upsertChunksBatched(workspaceSlug, documentUid, chunks, options = {}, config = {}) {
+  const { batchSize = 50, onProgress, shouldCancel } = options;
+  const client = await getClient(config);
+  const col    = colName(workspaceSlug);
+
+  let chunksProcessed = 0, embeddingTokens = 0, embeddingModel = null;
+  let collectionReady = false;
+
+  for (let i = 0; i < chunks.length; i += batchSize) {
+    if (shouldCancel && await shouldCancel()) return { chunksProcessed, embeddingTokens, embeddingModel };
+
+    const batch = chunks.slice(i, i + batchSize);
+    const { embeddings, tokensUsed, model } = await embed(batch.map(c => c.text));
+    embeddingTokens += tokensUsed;
+    if (!embeddingModel) embeddingModel = model;
+
+    if (!collectionReady) {
+      await ensureCollection(client, col, embeddings[0].length);
+      collectionReady = true;
+    }
+
+    await client.insert({
+      collection_name: col,
+      data: batch.map((c, j) => ({
+        id:          `${documentUid}_${i + j}`.slice(0, 128),
+        documentUid: documentUid.slice(0, 128),
+        text:        (c.text || "").slice(0, 65535),
+        metadata:    JSON.stringify(c.metadata || {}).slice(0, 65535),
+        embedding:   embeddings[j],
+      })),
+    });
+
+    chunksProcessed += batch.length;
+    if (onProgress) await onProgress(chunksProcessed);
+  }
+  return { chunksProcessed, embeddingTokens, embeddingModel };
+}
+
+async function similaritySearch(workspaceSlug, query, topK = 5, config = {}) {
+  try {
+    const client = await getClient(config);
+    const { embeddings } = await embed([query]);
+    const results = await client.search({
+      collection_name: colName(workspaceSlug),
+      vectors:         [embeddings[0]],
+      vector_type:     "FloatVector",
+      limit:           topK,
+      output_fields:   ["text", "metadata", "documentUid"],
+      metric_type:     "COSINE",
+    });
+    return (results.results || []).map(r => ({
+      text:     r.text || "",
+      metadata: (() => { try { return JSON.parse(r.metadata || "{}"); } catch { return {}; } })(),
+      score:    r.score,
+    }));
+  } catch {
+    return [];
+  }
+}
+
+async function deleteDocumentChunks(workspaceSlug, documentUid, config = {}) {
+  try {
+    const client = await getClient(config);
+    await client.deleteEntities({
+      collection_name: colName(workspaceSlug),
+      expr: `documentUid == "${documentUid}"`,
+    });
+  } catch {}
+}
+
+module.exports = { upsertChunksBatched, similaritySearch, deleteDocumentChunks };

package/src/utils/rag/pgvector.js

@@ -0,0 +1,104 @@
+const { embed } = require("../../providers/embedding");
+const getSetting = require("./_settings");
+const { v5: uuidv5 } = require("uuid");
+
+const UUID_NS = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
+
+let _pool = null, _poolKey = null;
+
+async function getPool(cfg = {}) {
+  const connStr = cfg.url || (await getSetting("vector_db_url")) || "";
+  if (!_pool || _poolKey !== connStr) {
+    const { Pool } = require("pg");
+    const pgvector = require("pgvector/pg");
+    _pool = new Pool({ connectionString: connStr });
+    await pgvector.registerTypes(_pool);
+    _poolKey = connStr;
+  }
+  return _pool;
+}
+
+async function ensureTable(pool, dim) {
+  await pool.query("CREATE EXTENSION IF NOT EXISTS vector");
+  await pool.query(`
+    CREATE TABLE IF NOT EXISTS embeddings (
+      id            TEXT PRIMARY KEY,
+      workspace_slug TEXT NOT NULL,
+      document_uid  TEXT NOT NULL,
+      text          TEXT NOT NULL,
+      metadata      JSONB,
+      embedding     vector(${dim})
+    )
+  `);
+  await pool.query("CREATE INDEX IF NOT EXISTS idx_emb_workspace ON embeddings(workspace_slug)");
+  await pool.query("CREATE INDEX IF NOT EXISTS idx_emb_document  ON embeddings(document_uid)");
+}
+
+let _tableReady = false;
+
+async function upsertChunksBatched(workspaceSlug, documentUid, chunks, options = {}, config = {}) {
+  const { batchSize = 50, onProgress, shouldCancel } = options;
+  const pool = await getPool(config);
+
+  let chunksProcessed = 0, embeddingTokens = 0, embeddingModel = null;
+
+  for (let i = 0; i < chunks.length; i += batchSize) {
+    if (shouldCancel && await shouldCancel()) return { chunksProcessed, embeddingTokens, embeddingModel };
+
+    const batch = chunks.slice(i, i + batchSize);
+    const { embeddings, tokensUsed, model } = await embed(batch.map(c => c.text));
+    embeddingTokens += tokensUsed;
+    if (!embeddingModel) embeddingModel = model;
+
+    if (!_tableReady) {
+      await ensureTable(pool, embeddings[0].length);
+      _tableReady = true;
+    }
+
+    const pgvector = require("pgvector/pg");
+    for (let j = 0; j < batch.length; j++) {
+      const id = uuidv5(`${documentUid}_${i + j}`, UUID_NS);
+      await pool.query(
+        `INSERT INTO embeddings (id, workspace_slug, document_uid, text, metadata, embedding)
+         VALUES ($1, $2, $3, $4, $5, $6)
+         ON CONFLICT (id) DO UPDATE SET embedding = EXCLUDED.embedding, text = EXCLUDED.text`,
+        [id, workspaceSlug, documentUid, batch[j].text, batch[j].metadata || {}, pgvector.toSql(embeddings[j])]
+      );
+    }
+
+    chunksProcessed += batch.length;
+    if (onProgress) await onProgress(chunksProcessed);
+  }
+  return { chunksProcessed, embeddingTokens, embeddingModel };
+}
+
+async function similaritySearch(workspaceSlug, query, topK = 5, config = {}) {
+  try {
+    const pool = await getPool(config);
+    const pgvector = require("pgvector/pg");
+    const { embeddings } = await embed([query]);
+    const { rows } = await pool.query(
+      `SELECT text, metadata, 1 - (embedding <=> $1) AS score
+       FROM embeddings
+       WHERE workspace_slug = $2
+       ORDER BY embedding <=> $1
+       LIMIT $3`,
+      [pgvector.toSql(embeddings[0]), workspaceSlug, topK]
+    );
+    return rows.map(r => ({ text: r.text, metadata: r.metadata || {}, score: parseFloat(r.score) }));
+  } catch {
+    return [];
+  }
+}
+
+async function deleteDocumentChunks(workspaceSlug, documentUid, config = {}) {
+  try {
+    const pool = await getPool(config);
+    await pool.query(
+      "DELETE FROM embeddings WHERE workspace_slug = $1 AND document_uid = $2",
+      [workspaceSlug, documentUid]
+    );
+  } catch {}
+}
+
+module.exports = { upsertChunksBatched, similaritySearch, deleteDocumentChunks };

package/src/utils/rag/pinecone.js

@@ -0,0 +1,71 @@
+const { embed } = require("../../providers/embedding");
+const getSetting = require("./_settings");
+
+let _pinecone = null;
+let _pineconeKey = null;
+
+async function getIndex(cfg = {}) {
+  const apiKey    = cfg.apiKey || (await getSetting("vector_db_api_key")) || "";
+  const indexName = cfg.index  || (await getSetting("vector_db_index"))   || "open-enthrium";
+  if (!_pinecone || _pineconeKey !== apiKey) {
+    const { Pinecone } = await import("@pinecone-database/pinecone");
+    _pinecone    = new Pinecone({ apiKey });
+    _pineconeKey = apiKey;
+  }
+  return _pinecone.index(indexName);
+}
+
+async function upsertChunksBatched(workspaceSlug, documentUid, chunks, options = {}, config = {}) {
+  const { batchSize = 100, onProgress, shouldCancel } = options;
+  const index = await getIndex(config);
+  const ns    = index.namespace(workspaceSlug);
+
+  let chunksProcessed = 0, embeddingTokens = 0, embeddingModel = null;
+
+  for (let i = 0; i < chunks.length; i += batchSize) {
+    if (shouldCancel && await shouldCancel()) return { chunksProcessed, embeddingTokens, embeddingModel };
+
+    const batch = chunks.slice(i, i + batchSize);
+    const { embeddings, tokensUsed, model } = await embed(batch.map(c => c.text));
+    embeddingTokens += tokensUsed;
+    if (!embeddingModel) embeddingModel = model;
+
+    await ns.upsert(batch.map((c, j) => ({
+      id:       `${documentUid}_${i + j}`,
+      values:   embeddings[j],
+      metadata: { documentUid, text: c.text, ...(c.metadata || {}) },
+    })));
+
+    chunksProcessed += batch.length;
+    if (onProgress) await onProgress(chunksProcessed);
+  }
+  return { chunksProcessed, embeddingTokens, embeddingModel };
+}
+
+async function similaritySearch(workspaceSlug, query, topK = 5, config = {}) {
+  try {
+    const index = await getIndex(config);
+    const ns    = index.namespace(workspaceSlug);
+    const { embeddings } = await embed([query]);
+    const results = await ns.query({ vector: embeddings[0], topK, includeMetadata: true });
+    return (results.matches || []).map(m => ({
+      text:     m.metadata?.text || "",
+      metadata: Object.fromEntries(
+        Object.entries(m.metadata || {}).filter(([k]) => k !== "text" && k !== "documentUid")
+      ),
+      score: m.score,
+    }));
+  } catch {
+    return [];
+  }
+}
+
+async function deleteDocumentChunks(workspaceSlug, documentUid, config = {}) {
+  try {
+    const index = await getIndex(config);
+    const ns    = index.namespace(workspaceSlug);
+    await ns.deleteMany({ documentUid });
+  } catch {}
+}
+
+module.exports = { upsertChunksBatched, similaritySearch, deleteDocumentChunks };

package/src/utils/rag/qdrant.js

@@ -0,0 +1,106 @@
+const { embed } = require("../../providers/embedding");
+const getSetting = require("./_settings");
+const { v5: uuidv5 } = require("uuid");
+
+const UUID_NS = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
+
+let _client = null, _clientKey = null;
+
+async function getClient(cfg = {}) {
+  const url    = cfg.url    || (await getSetting("vector_db_url"))     || "http://localhost:6333";
+  const apiKey = cfg.apiKey || (await getSetting("vector_db_api_key")) || "";
+  const key = `${url}:${apiKey}`;
+  if (!_client || _clientKey !== key) {
+    const { QdrantClient } = await import("@qdrant/js-client-rest");
+    _client    = new QdrantClient({ url, ...(apiKey ? { apiKey } : {}) });
+    _clientKey = key;
+  }
+  return _client;
+}
+
+function colName(workspaceSlug) {
+  return `ws_${workspaceSlug.replace(/-/g, "_")}`;
+}
+
+async function ensureCollection(client, name, vectorSize) {
+  try {
+    await client.getCollection(name);
+  } catch {
+    await client.createCollection(name, {
+      vectors: { size: vectorSize, distance: "Cosine" },
+    });
+    await client.createPayloadIndex(name, {
+      field_name: "documentUid",
+      field_schema: "keyword",
+    });
+  }
+}
+
+async function upsertChunksBatched(workspaceSlug, documentUid, chunks, options = {}, config = {}) {
+  const { batchSize = 50, onProgress, shouldCancel } = options;
+  const client = await getClient(config);
+  const col    = colName(workspaceSlug);
+
+  let chunksProcessed = 0, embeddingTokens = 0, embeddingModel = null;
+  let collectionReady = false;
+
+  for (let i = 0; i < chunks.length; i += batchSize) {
+    if (shouldCancel && await shouldCancel()) return { chunksProcessed, embeddingTokens, embeddingModel };
+
+    const batch = chunks.slice(i, i + batchSize);
+    const { embeddings, tokensUsed, model } = await embed(batch.map(c => c.text));
+    embeddingTokens += tokensUsed;
+    if (!embeddingModel) embeddingModel = model;
+
+    if (!collectionReady) {
+      await ensureCollection(client, col, embeddings[0].length);
+      collectionReady = true;
+    }
+
+    await client.upsert(col, {
+      wait: true,
+      points: batch.map((c, j) => ({
+        id:      uuidv5(`${documentUid}_${i + j}`, UUID_NS),
+        vector:  embeddings[j],
+        payload: { documentUid, text: c.text, ...(c.metadata || {}) },
+      })),
+    });
+
+    chunksProcessed += batch.length;
+    if (onProgress) await onProgress(chunksProcessed);
+  }
+  return { chunksProcessed, embeddingTokens, embeddingModel };
+}
+
+async function similaritySearch(workspaceSlug, query, topK = 5, config = {}) {
+  try {
+    const client = await getClient(config);
+    const { embeddings } = await embed([query]);
+    const results = await client.search(colName(workspaceSlug), {
+      vector: embeddings[0],
+      limit: topK,
+      with_payload: true,
+    });
+    return results.map(r => ({
+      text:     r.payload?.text || "",
+      metadata: Object.fromEntries(
+        Object.entries(r.payload || {}).filter(([k]) => k !== "text" && k !== "documentUid")
+      ),
+      score: r.score,
+    }));
+  } catch {
+    return [];
+  }
+}
+
+async function deleteDocumentChunks(workspaceSlug, documentUid, config = {}) {
+  try {
+    const client = await getClient(config);
+    await client.delete(colName(workspaceSlug), {
+      wait: true,
+      filter: { must: [{ key: "documentUid", match: { value: documentUid } }] },
+    });
+  } catch {}
+}
+
+module.exports = { upsertChunksBatched, similaritySearch, deleteDocumentChunks };