@lov3kaizen/agentsea-embeddings 0.5.1

package/dist/chunk-TER262ST.mjs

@@ -0,0 +1,877 @@
+import {
+  batch
+} from "./chunk-3KM32UQK.mjs";
+import {
+  EmbeddingModel
+} from "./chunk-QAITLJ2E.mjs";
+
+// src/stores/BaseStore.ts
+var BaseStore = class {
+  /** Store configuration */
+  config;
+  constructor(config) {
+    this.config = {
+      namespace: config.namespace ?? "default",
+      dimensions: config.dimensions,
+      metric: config.metric ?? "cosine",
+      ...config
+    };
+  }
+  /**
+   * Get namespace
+   */
+  get namespace() {
+    return this.config.namespace ?? "default";
+  }
+  /**
+   * Get dimensions
+   */
+  get dimensions() {
+    return this.config.dimensions;
+  }
+  /**
+   * Get distance metric
+   */
+  get metric() {
+    return this.config.metric ?? "cosine";
+  }
+  /**
+   * Calculate similarity/distance between vectors
+   */
+  calculateScore(a, b) {
+    switch (this.metric) {
+      case "cosine":
+        return EmbeddingModel.cosineSimilarity(a, b);
+      case "euclidean": {
+        const dist = EmbeddingModel.euclideanDistance(a, b);
+        return 1 / (1 + dist);
+      }
+      case "dot_product":
+        return EmbeddingModel.dotProduct(a, b);
+      default:
+        return EmbeddingModel.cosineSimilarity(a, b);
+    }
+  }
+  /**
+   * Filter records by metadata
+   */
+  filterByMetadata(records, filter) {
+    if (!filter) return records;
+    return records.filter((record) => {
+      if (!record.metadata) return false;
+      for (const [key, value] of Object.entries(filter)) {
+        if (record.metadata[key] !== value) {
+          return false;
+        }
+      }
+      return true;
+    });
+  }
+  /**
+   * Convert records to search results
+   */
+  toSearchResults(records, options) {
+    return records.map((record) => ({
+      id: record.id,
+      text: options?.includeText !== false ? record.text ?? "" : "",
+      score: record.score,
+      metadata: options?.includeMetadata !== false ? record.metadata ?? {} : {},
+      distance: this.metric !== "cosine" ? 1 - record.score : void 0
+    }));
+  }
+};
+
+// src/stores/MemoryStore.ts
+import * as fs from "fs/promises";
+var MemoryStore = class extends BaseStore {
+  storeType = "memory";
+  vectors = /* @__PURE__ */ new Map();
+  namespaces = /* @__PURE__ */ new Map();
+  persistPath;
+  persistInterval;
+  maxVectors;
+  constructor(config = { type: "memory" }) {
+    super(config);
+    this.maxVectors = config.maxVectors ?? 1e5;
+    this.persistPath = config.persistPath;
+    if (config.persistInterval && config.persistPath) {
+      this.persistInterval = setInterval(
+        () => void this.persist().catch(() => {
+        }),
+        config.persistInterval
+      );
+    }
+  }
+  async upsert(records, options) {
+    const startTime = performance.now();
+    const namespace = options?.namespace ?? this.namespace;
+    const upsertedIds = [];
+    const errors = [];
+    if (!this.namespaces.has(namespace)) {
+      this.namespaces.set(namespace, /* @__PURE__ */ new Set());
+    }
+    const nsIds = this.namespaces.get(namespace);
+    for (const record of records) {
+      try {
+        if (this.vectors.size >= this.maxVectors && !this.vectors.has(record.id)) {
+          const oldestId = this.vectors.keys().next().value;
+          if (oldestId) {
+            this.vectors.delete(oldestId);
+            for (const ns of this.namespaces.values()) {
+              ns.delete(oldestId);
+            }
+          }
+        }
+        this.vectors.set(record.id, record);
+        nsIds.add(record.id);
+        upsertedIds.push(record.id);
+      } catch (error) {
+        errors.push({ id: record.id, error: error.message });
+      }
+    }
+    return Promise.resolve({
+      upsertedIds,
+      upsertedCount: upsertedIds.length,
+      errors,
+      durationMs: performance.now() - startTime
+    });
+  }
+  async query(vector, options) {
+    const startTime = performance.now();
+    const namespace = options?.namespace ?? this.namespace;
+    const topK = options?.topK ?? 10;
+    const minScore = options?.minScore ?? 0;
+    const nsIds = this.namespaces.get(namespace);
+    if (!nsIds || nsIds.size === 0) {
+      return {
+        matches: [],
+        namespace,
+        durationMs: performance.now() - startTime
+      };
+    }
+    let scoredRecords = [];
+    for (const id of nsIds) {
+      const record = this.vectors.get(id);
+      if (!record) continue;
+      const score = this.calculateScore(vector, record.vector);
+      if (score >= minScore) {
+        scoredRecords.push({ ...record, score });
+      }
+    }
+    if (options?.filter) {
+      scoredRecords = this.filterByMetadata(
+        scoredRecords,
+        options.filter
+      );
+    }
+    scoredRecords.sort((a, b) => b.score - a.score);
+    const topResults = scoredRecords.slice(0, topK);
+    return Promise.resolve({
+      matches: this.toSearchResults(topResults, options),
+      namespace,
+      durationMs: performance.now() - startTime
+    });
+  }
+  async delete(ids, options) {
+    const startTime = performance.now();
+    const namespace = options?.namespace ?? this.namespace;
+    let deletedCount = 0;
+    const nsIds = this.namespaces.get(namespace);
+    for (const id of ids) {
+      if (this.vectors.has(id)) {
+        this.vectors.delete(id);
+        nsIds?.delete(id);
+        deletedCount++;
+      }
+    }
+    return Promise.resolve({
+      deletedCount,
+      durationMs: performance.now() - startTime
+    });
+  }
+  async deleteAll(options) {
+    const startTime = performance.now();
+    const namespace = options?.namespace ?? this.namespace;
+    if (options?.deleteAll) {
+      const count2 = this.vectors.size;
+      this.vectors.clear();
+      this.namespaces.clear();
+      return Promise.resolve({
+        deletedCount: count2,
+        durationMs: performance.now() - startTime
+      });
+    }
+    const nsIds = this.namespaces.get(namespace);
+    if (!nsIds) {
+      return Promise.resolve({
+        deletedCount: 0,
+        durationMs: performance.now() - startTime
+      });
+    }
+    const count = nsIds.size;
+    for (const id of nsIds) {
+      this.vectors.delete(id);
+    }
+    this.namespaces.delete(namespace);
+    return Promise.resolve({
+      deletedCount: count,
+      durationMs: performance.now() - startTime
+    });
+  }
+  getStats() {
+    return Promise.resolve({
+      type: this.storeType,
+      vectorCount: this.vectors.size,
+      namespaceCount: this.namespaces.size,
+      dimensions: this.dimensions ?? 0,
+      metric: this.metric,
+      lastUpdated: Date.now()
+    });
+  }
+  checkHealth() {
+    return Promise.resolve({
+      healthy: true,
+      latencyMs: 0,
+      lastCheck: Date.now()
+    });
+  }
+  async close() {
+    if (this.persistInterval) {
+      clearInterval(this.persistInterval);
+    }
+    if (this.persistPath) {
+      await this.persist();
+    }
+  }
+  /**
+   * Persist store to file
+   */
+  async persist() {
+    if (!this.persistPath) return;
+    const data = {
+      vectors: Array.from(this.vectors.entries()),
+      namespaces: Array.from(this.namespaces.entries()).map(([k, v]) => [
+        k,
+        Array.from(v)
+      ])
+    };
+    await fs.writeFile(this.persistPath, JSON.stringify(data), "utf-8");
+  }
+  /**
+   * Load store from file
+   */
+  async load() {
+    if (!this.persistPath) return;
+    try {
+      const data = JSON.parse(await fs.readFile(this.persistPath, "utf-8"));
+      this.vectors = new Map(data.vectors);
+      this.namespaces = new Map(
+        data.namespaces.map(([k, v]) => [k, new Set(v)])
+      );
+    } catch {
+    }
+  }
+  /**
+   * Get all vectors
+   */
+  getAll() {
+    return Array.from(this.vectors.values());
+  }
+  /**
+   * Get vector by ID
+   */
+  getById(id) {
+    return this.vectors.get(id);
+  }
+};
+function createMemoryStore(config) {
+  return new MemoryStore(config);
+}
+
+// src/stores/PineconeStore.ts
+var PineconeStore = class extends BaseStore {
+  storeType = "pinecone";
+  client;
+  index;
+  apiKey;
+  indexName;
+  initialized = false;
+  constructor(config) {
+    super(config);
+    if (!config.apiKey) {
+      throw new Error("Pinecone API key is required");
+    }
+    if (!config.indexName) {
+      throw new Error("Pinecone index name is required");
+    }
+    this.apiKey = config.apiKey;
+    this.indexName = config.indexName;
+  }
+  /**
+   * Initialize Pinecone client
+   */
+  async init() {
+    if (this.initialized) return;
+    try {
+      const { Pinecone } = await import("@pinecone-database/pinecone");
+      this.client = new Pinecone({ apiKey: this.apiKey });
+      this.index = this.client.index(
+        this.indexName
+      );
+      this.initialized = true;
+    } catch (error) {
+      throw new Error(
+        `Failed to initialize Pinecone: ${error.message}`
+      );
+    }
+  }
+  async ensureInitialized() {
+    if (!this.initialized) {
+      await this.init();
+    }
+  }
+  async upsert(records, options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    const namespace = options?.namespace ?? this.namespace;
+    const batchSize = options?.batchSize ?? 100;
+    const upsertedIds = [];
+    const errors = [];
+    const vectors = records.map((record) => ({
+      id: record.id,
+      values: record.vector,
+      metadata: {
+        ...record.metadata,
+        text: record.text
+      }
+    }));
+    const batches = batch(vectors, batchSize);
+    let completed = 0;
+    for (const batchVectors of batches) {
+      try {
+        const ns = this.index.namespace(namespace);
+        await ns.upsert(
+          batchVectors
+        );
+        upsertedIds.push(...batchVectors.map((v) => v.id));
+      } catch (error) {
+        for (const v of batchVectors) {
+          errors.push({ id: v.id, error: error.message });
+        }
+      }
+      completed += batchVectors.length;
+      options?.onProgress?.({ completed, total: records.length });
+    }
+    return {
+      upsertedIds,
+      upsertedCount: upsertedIds.length,
+      errors,
+      durationMs: performance.now() - startTime
+    };
+  }
+  async query(vector, options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    const namespace = options?.namespace ?? this.namespace;
+    const topK = options?.topK ?? 10;
+    const ns = this.index.namespace(
+      namespace
+    );
+    const result = await ns.query({
+      vector,
+      topK,
+      filter: options?.filter,
+      includeValues: options?.includeVectors ?? false,
+      includeMetadata: options?.includeMetadata ?? true
+    });
+    const matches = result.matches.map((match) => ({
+      id: match.id,
+      text: match.metadata?.text ?? "",
+      score: match.score,
+      metadata: match.metadata ?? {}
+    }));
+    const filtered = options?.minScore ? matches.filter((m) => m.score >= options.minScore) : matches;
+    return {
+      matches: filtered,
+      namespace,
+      durationMs: performance.now() - startTime
+    };
+  }
+  async delete(ids, options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    const namespace = options?.namespace ?? this.namespace;
+    const ns = this.index.namespace(
+      namespace
+    );
+    await ns.deleteMany(
+      ids
+    );
+    return {
+      deletedCount: ids.length,
+      durationMs: performance.now() - startTime
+    };
+  }
+  async deleteAll(options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    const namespace = options?.namespace ?? this.namespace;
+    const ns = this.index.namespace(
+      namespace
+    );
+    await ns.deleteAll();
+    return {
+      deletedCount: -1,
+      // Unknown count
+      durationMs: performance.now() - startTime
+    };
+  }
+  async getStats() {
+    await this.ensureInitialized();
+    const stats = await this.index.describeIndexStats();
+    return {
+      type: this.storeType,
+      vectorCount: stats.totalVectorCount ?? 0,
+      namespaceCount: Object.keys(stats.namespaces ?? {}).length,
+      dimensions: stats.dimension,
+      metric: this.metric,
+      lastUpdated: Date.now()
+    };
+  }
+  async checkHealth() {
+    const startTime = performance.now();
+    try {
+      await this.ensureInitialized();
+      await this.index.describeIndexStats();
+      return {
+        healthy: true,
+        latencyMs: performance.now() - startTime,
+        lastCheck: Date.now()
+      };
+    } catch (error) {
+      return {
+        healthy: false,
+        latencyMs: performance.now() - startTime,
+        lastCheck: Date.now(),
+        error: error.message
+      };
+    }
+  }
+  async close() {
+    this.initialized = false;
+    return Promise.resolve();
+  }
+};
+function createPineconeStore(config) {
+  return new PineconeStore(config);
+}
+
+// src/stores/ChromaStore.ts
+var ChromaStore = class extends BaseStore {
+  storeType = "chroma";
+  client;
+  collection;
+  collectionName;
+  url;
+  initialized = false;
+  constructor(config) {
+    super(config);
+    if (!config.collectionName) {
+      throw new Error("Chroma collection name is required");
+    }
+    this.collectionName = config.collectionName;
+    this.url = config.url;
+  }
+  /**
+   * Initialize ChromaDB client
+   */
+  async init() {
+    if (this.initialized) return;
+    try {
+      const { ChromaClient } = await import("chromadb");
+      this.client = this.url ? new ChromaClient({ path: this.url }) : new ChromaClient();
+      this.collection = await this.client.getOrCreateCollection({
+        name: this.collectionName,
+        metadata: {
+          "hnsw:space": this.metricToChroma(this.metric)
+        }
+      });
+      this.initialized = true;
+    } catch (error) {
+      throw new Error(
+        `Failed to initialize ChromaDB: ${error.message}`
+      );
+    }
+  }
+  metricToChroma(metric) {
+    switch (metric) {
+      case "cosine":
+        return "cosine";
+      case "euclidean":
+        return "l2";
+      case "dot_product":
+        return "ip";
+      default:
+        return "cosine";
+    }
+  }
+  async ensureInitialized() {
+    if (!this.initialized) {
+      await this.init();
+    }
+  }
+  async upsert(records, _options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    const ids = records.map((r) => r.id);
+    const embeddings = records.map((r) => r.vector);
+    const documents = records.map((r) => r.text ?? "");
+    const metadatas = records.map((r) => r.metadata ?? {});
+    const upsertedIds = [];
+    const errors = [];
+    try {
+      await this.collection.upsert({
+        ids,
+        embeddings,
+        documents,
+        metadatas
+      });
+      upsertedIds.push(...ids);
+    } catch (error) {
+      for (const id of ids) {
+        errors.push({ id, error: error.message });
+      }
+    }
+    return {
+      upsertedIds,
+      upsertedCount: upsertedIds.length,
+      errors,
+      durationMs: performance.now() - startTime
+    };
+  }
+  async query(vector, options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    const topK = options?.topK ?? 10;
+    const result = await this.collection.query({
+      queryEmbeddings: [vector],
+      nResults: topK,
+      where: options?.filter,
+      include: ["documents", "metadatas", "distances"]
+    });
+    const matches = (result.ids[0] ?? []).map((id, i) => {
+      const distance = result.distances?.[0]?.[i] ?? 0;
+      const score = 1 / (1 + distance);
+      return {
+        id,
+        text: result.documents?.[0]?.[i] ?? "",
+        score,
+        metadata: result.metadatas?.[0]?.[i] ?? {},
+        distance
+      };
+    });
+    const filtered = options?.minScore ? matches.filter((m) => m.score >= options.minScore) : matches;
+    return {
+      matches: filtered,
+      namespace: this.collectionName,
+      durationMs: performance.now() - startTime
+    };
+  }
+  async delete(ids, _options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    await this.collection.delete({ ids });
+    return {
+      deletedCount: ids.length,
+      durationMs: performance.now() - startTime
+    };
+  }
+  async deleteAll(_options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    await this.client.deleteCollection({ name: this.collectionName });
+    this.collection = await this.client.createCollection({
+      name: this.collectionName,
+      metadata: {
+        "hnsw:space": this.metricToChroma(this.metric)
+      }
+    });
+    return {
+      deletedCount: -1,
+      durationMs: performance.now() - startTime
+    };
+  }
+  async getStats() {
+    await this.ensureInitialized();
+    const count = await this.collection.count();
+    return {
+      type: this.storeType,
+      vectorCount: count,
+      namespaceCount: 1,
+      dimensions: this.dimensions ?? 0,
+      metric: this.metric,
+      lastUpdated: Date.now()
+    };
+  }
+  async checkHealth() {
+    const startTime = performance.now();
+    try {
+      await this.ensureInitialized();
+      await this.collection.count();
+      return {
+        healthy: true,
+        latencyMs: performance.now() - startTime,
+        lastCheck: Date.now()
+      };
+    } catch (error) {
+      return {
+        healthy: false,
+        latencyMs: performance.now() - startTime,
+        lastCheck: Date.now(),
+        error: error.message
+      };
+    }
+  }
+  async close() {
+    this.initialized = false;
+    return Promise.resolve();
+  }
+};
+function createChromaStore(config) {
+  return new ChromaStore(config);
+}
+
+// src/stores/QdrantStore.ts
+var QdrantStore = class extends BaseStore {
+  storeType = "qdrant";
+  client;
+  collectionName;
+  url;
+  apiKey;
+  initialized = false;
+  constructor(config) {
+    super(config);
+    if (!config.url) {
+      throw new Error("Qdrant URL is required");
+    }
+    if (!config.collectionName) {
+      throw new Error("Qdrant collection name is required");
+    }
+    this.url = config.url;
+    this.collectionName = config.collectionName;
+    this.apiKey = config.apiKey;
+  }
+  /**
+   * Initialize Qdrant client
+   */
+  async init() {
+    if (this.initialized) return;
+    try {
+      const { QdrantClient } = await import("@qdrant/js-client-rest");
+      this.client = new QdrantClient({
+        url: this.url,
+        apiKey: this.apiKey
+      });
+      const collections = await this.client.getCollections();
+      const exists = collections.collections.some(
+        (c) => c.name === this.collectionName
+      );
+      if (!exists && this.dimensions) {
+        await this.client.createCollection(this.collectionName, {
+          vectors: {
+            size: this.dimensions,
+            distance: this.metricToQdrant(this.metric)
+          }
+        });
+      }
+      this.initialized = true;
+    } catch (error) {
+      throw new Error(
+        `Failed to initialize Qdrant: ${error.message}`
+      );
+    }
+  }
+  metricToQdrant(metric) {
+    switch (metric) {
+      case "cosine":
+        return "Cosine";
+      case "euclidean":
+        return "Euclid";
+      case "dot_product":
+        return "Dot";
+      default:
+        return "Cosine";
+    }
+  }
+  async ensureInitialized() {
+    if (!this.initialized) {
+      await this.init();
+    }
+  }
+  async upsert(records, options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    const batchSize = options?.batchSize ?? 100;
+    const upsertedIds = [];
+    const errors = [];
+    const points = records.map((record) => ({
+      id: record.id,
+      vector: record.vector,
+      payload: {
+        ...record.metadata,
+        text: record.text
+      }
+    }));
+    const batches = batch(points, batchSize);
+    let completed = 0;
+    for (const batchPoints of batches) {
+      try {
+        await this.client.upsert(this.collectionName, {
+          points: batchPoints
+        });
+        upsertedIds.push(...batchPoints.map((p) => p.id));
+      } catch (error) {
+        for (const p of batchPoints) {
+          errors.push({ id: p.id, error: error.message });
+        }
+      }
+      completed += batchPoints.length;
+      options?.onProgress?.({ completed, total: records.length });
+    }
+    return {
+      upsertedIds,
+      upsertedCount: upsertedIds.length,
+      errors,
+      durationMs: performance.now() - startTime
+    };
+  }
+  async query(vector, options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    const topK = options?.topK ?? 10;
+    const result = await this.client.search(this.collectionName, {
+      vector,
+      limit: topK,
+      filter: options?.filter ? this.buildQdrantFilter(options.filter) : void 0,
+      with_payload: options?.includeMetadata ?? true,
+      with_vector: options?.includeVectors ?? false,
+      score_threshold: options?.minScore
+    });
+    const matches = result.map((point) => ({
+      id: point.id.toString(),
+      text: point.payload?.text ?? "",
+      score: point.score,
+      metadata: point.payload ?? {}
+    }));
+    return {
+      matches,
+      namespace: this.collectionName,
+      durationMs: performance.now() - startTime
+    };
+  }
+  buildQdrantFilter(filter) {
+    const must = [];
+    for (const [key, value] of Object.entries(filter)) {
+      must.push({
+        key,
+        match: { value }
+      });
+    }
+    return { must };
+  }
+  async delete(ids, _options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    await this.client.delete(this.collectionName, {
+      points: ids
+    });
+    return {
+      deletedCount: ids.length,
+      durationMs: performance.now() - startTime
+    };
+  }
+  async deleteAll(_options) {
+    await this.ensureInitialized();
+    const startTime = performance.now();
+    await this.client.deleteCollection(this.collectionName);
+    if (this.dimensions) {
+      await this.client.createCollection(this.collectionName, {
+        vectors: {
+          size: this.dimensions,
+          distance: this.metricToQdrant(this.metric)
+        }
+      });
+    }
+    return {
+      deletedCount: -1,
+      durationMs: performance.now() - startTime
+    };
+  }
+  async getStats() {
+    await this.ensureInitialized();
+    const info = await this.client.getCollection(this.collectionName);
+    return {
+      type: this.storeType,
+      vectorCount: info.vectors_count,
+      namespaceCount: 1,
+      dimensions: info.config?.params?.vectors?.size,
+      metric: this.metric,
+      lastUpdated: Date.now()
+    };
+  }
+  async checkHealth() {
+    const startTime = performance.now();
+    try {
+      await this.ensureInitialized();
+      await this.client.getCollection(this.collectionName);
+      return {
+        healthy: true,
+        latencyMs: performance.now() - startTime,
+        lastCheck: Date.now()
+      };
+    } catch (error) {
+      return {
+        healthy: false,
+        latencyMs: performance.now() - startTime,
+        lastCheck: Date.now(),
+        error: error.message
+      };
+    }
+  }
+  async close() {
+    this.initialized = false;
+    return Promise.resolve();
+  }
+};
+function createQdrantStore(config) {
+  return new QdrantStore(config);
+}
+
+// src/stores/index.ts
+function createStore(type, config) {
+  switch (type) {
+    case "memory":
+      return new MemoryStore(config);
+    case "pinecone":
+      return new PineconeStore(config);
+    case "chroma":
+      return new ChromaStore(config);
+    case "qdrant":
+      return new QdrantStore(config);
+    default:
+      return new MemoryStore(config);
+  }
+}
+
+export {
+  BaseStore,
+  MemoryStore,
+  createMemoryStore,
+  PineconeStore,
+  createPineconeStore,
+  ChromaStore,
+  createChromaStore,
+  QdrantStore,
+  createQdrantStore,
+  createStore
+};