@agentionai/agents 0.3.0-beta

package/dist/vectorstore/LanceDBVectorStore.js

@@ -0,0 +1,338 @@
+"use strict";
+/**
+ * LanceDB implementation of the VectorStore interface.
+ *
+ * LanceDB is an embedded vector database that runs locally or can connect
+ * to cloud storage. It provides fast vector search with automatic indexing.
+ *
+ * @requires @lancedb/lancedb - Install with: npm install @lancedb/lancedb
+ * @requires apache-arrow - Install with: npm install apache-arrow
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LanceDBVectorStore = void 0;
+const VectorStore_1 = require("./VectorStore");
+/**
+ * LanceDB implementation of the VectorStore interface.
+ *
+ * @example
+ * ```typescript
+ * import { LanceDBVectorStore, OpenAIEmbeddings } from "@agentionai/agents";
+ *
+ * // Create with OpenAI embeddings
+ * const embeddings = new OpenAIEmbeddings({
+ *   model: "text-embedding-3-small",
+ * });
+ *
+ * const store = await LanceDBVectorStore.create({
+ *   name: "knowledge_base",
+ *   uri: "./my-database",
+ *   tableName: "documents",
+ *   embeddings,
+ * });
+ *
+ * // Add documents (embeddings generated automatically)
+ * await store.addDocuments([
+ *   { id: "1", content: "LanceDB is a vector database" },
+ *   { id: "2", content: "Vector search enables semantic queries" },
+ * ]);
+ *
+ * // Search
+ * const results = await store.search("What is LanceDB?", { limit: 5 });
+ *
+ * // Create a tool for agents
+ * const searchTool = store.toRetrievalTool("Search the knowledge base");
+ * ```
+ */
+class LanceDBVectorStore extends VectorStore_1.VectorStore {
+    constructor(config, connection, table) {
+        super();
+        this.name = config.name;
+        this.connection = connection;
+        this.table = table;
+        this.embeddings = config.embeddings;
+        this.tableName = config.tableName;
+        this.dimensions =
+            config.dimensions ?? config.embeddings?.dimensions ?? 1536;
+    }
+    /**
+     * Create a new LanceDBVectorStore instance.
+     *
+     * This is an async factory method since LanceDB connection is asynchronous.
+     *
+     * @param config - Configuration for the store
+     * @returns A configured LanceDBVectorStore instance
+     *
+     * @throws Error if @lancedb/lancedb is not installed
+     */
+    static async create(config) {
+        // Dynamic import to make lancedb an optional dependency
+        let lancedb;
+        try {
+            lancedb = await Promise.resolve().then(() => __importStar(require("@lancedb/lancedb")));
+        }
+        catch {
+            throw new Error("LanceDB is not installed. Install it with: npm install @lancedb/lancedb apache-arrow");
+        }
+        const connection = await lancedb.connect(config.uri, config.connectionOptions);
+        const tableNames = await connection.tableNames();
+        let table;
+        const dimensions = config.dimensions ?? config.embeddings?.dimensions ?? 1536;
+        if (tableNames.includes(config.tableName)) {
+            table = await connection.openTable(config.tableName);
+        }
+        else {
+            // Create table with schema
+            let arrow;
+            try {
+                arrow = await Promise.resolve().then(() => __importStar(require("apache-arrow")));
+            }
+            catch {
+                throw new Error("apache-arrow is not installed. Install it with: npm install apache-arrow");
+            }
+            const schema = new arrow.Schema([
+                new arrow.Field("id", new arrow.Utf8(), false),
+                new arrow.Field("text", new arrow.Utf8(), false),
+                new arrow.Field("vector", new arrow.FixedSizeList(dimensions, new arrow.Field("item", new arrow.Float32(), true)), false),
+                new arrow.Field("metadata", new arrow.Utf8(), true),
+            ]);
+            table = await connection.createEmptyTable(config.tableName, schema);
+        }
+        return new LanceDBVectorStore(config, connection, table);
+    }
+    /**
+     * Add documents to the vector store.
+     * If an embeddings provider is configured, embeddings are generated automatically.
+     */
+    async addDocuments(documents, _options) {
+        if (!this.embeddings) {
+            throw new Error("No embeddings provider configured. Use addEmbeddedDocuments() with pre-computed embeddings, or configure an embeddings provider.");
+        }
+        // Generate embeddings for all documents
+        const texts = documents.map((doc) => doc.content);
+        const vectors = await this.embeddings.embed(texts);
+        // Convert to embedded documents
+        const embeddedDocs = documents.map((doc, i) => ({
+            ...doc,
+            embedding: vectors[i],
+        }));
+        return this.addEmbeddedDocuments(embeddedDocs, _options);
+    }
+    /**
+     * Add documents with pre-computed embeddings.
+     */
+    async addEmbeddedDocuments(documents, _options) {
+        const records = documents.map((doc) => ({
+            id: doc.id,
+            text: doc.content,
+            vector: doc.embedding,
+            metadata: doc.metadata ? JSON.stringify(doc.metadata) : undefined,
+        }));
+        await this.table.add(records);
+        return documents.map((d) => d.id);
+    }
+    /**
+     * Search for documents similar to the query.
+     */
+    async search(query, options) {
+        if (!this.embeddings) {
+            throw new Error("No embeddings provider configured. Use searchByVector() with a pre-computed query embedding, or configure an embeddings provider.");
+        }
+        const queryVector = await this.embeddings.embedQuery(query);
+        return this.searchByVector(queryVector, options);
+    }
+    /**
+     * Search using a pre-computed embedding vector.
+     */
+    async searchByVector(embedding, options) {
+        const limit = options?.limit ?? 10;
+        const scoreThreshold = options?.scoreThreshold;
+        let queryBuilder = this.table.vectorSearch(embedding).limit(limit);
+        if (options?.filter) {
+            const filterStr = this.buildFilterString(options.filter);
+            if (filterStr) {
+                queryBuilder = queryBuilder.where(filterStr);
+            }
+        }
+        const results = await queryBuilder.toArray();
+        return this.processResults(results, scoreThreshold);
+    }
+    /**
+     * Delete documents by their IDs.
+     */
+    async delete(ids, _options) {
+        const idList = ids.map((id) => `'${id}'`).join(", ");
+        const filter = `id IN (${idList})`;
+        const countBefore = await this.table.countRows();
+        await this.table.delete(filter);
+        const countAfter = await this.table.countRows();
+        return countBefore - countAfter;
+    }
+    /**
+     * Delete all documents.
+     */
+    async clear(_options) {
+        await this.table.delete("id IS NOT NULL");
+    }
+    /**
+     * Get a document by its ID.
+     */
+    async getById(id, _options) {
+        const results = await this.table
+            .query()
+            .where(`id = '${id}'`)
+            .limit(1)
+            .toArray();
+        if (results.length === 0) {
+            return null;
+        }
+        const row = results[0];
+        return {
+            id: row.id,
+            content: row.text,
+            metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
+        };
+    }
+    /**
+     * Get existing documents by their content hashes.
+     * Used for deduplication during ingestion.
+     */
+    async getByHashes(hashes, _options) {
+        const hashMap = new Map();
+        if (hashes.length === 0) {
+            return hashMap;
+        }
+        // Query for documents with matching hashes
+        // Since hash is stored in the metadata JSON, we need to check each hash
+        for (const hash of hashes) {
+            // LanceDB doesn't support JSON path queries, so we search for the hash string
+            // in the metadata field. This works because the hash is a unique string.
+            const results = await this.table
+                .query()
+                .where(`metadata LIKE '%${hash}%'`)
+                .limit(1)
+                .toArray();
+            if (results.length > 0) {
+                const record = results[0];
+                hashMap.set(hash, record.id);
+            }
+        }
+        return hashMap;
+    }
+    /**
+     * Get the underlying LanceDB connection.
+     */
+    getConnection() {
+        return this.connection;
+    }
+    /**
+     * Get the underlying LanceDB table.
+     */
+    getTable() {
+        return this.table;
+    }
+    /**
+     * Get the configured embeddings provider.
+     */
+    getEmbeddings() {
+        return this.embeddings;
+    }
+    /**
+     * Get the vector dimensions.
+     */
+    getDimensions() {
+        return this.dimensions;
+    }
+    /**
+     * Create an index on the vector column for faster searches.
+     * Recommended for tables with more than 10,000 rows.
+     */
+    async createIndex() {
+        await this.table.createIndex("vector");
+    }
+    /**
+     * Optimize the table for better performance.
+     */
+    async optimize() {
+        await this.table.optimize();
+    }
+    /**
+     * Build a SQL filter string from a filter object.
+     */
+    buildFilterString(filter) {
+        const conditions = [];
+        for (const [key, value] of Object.entries(filter)) {
+            if (typeof value === "string") {
+                conditions.push(`${key} = '${value}'`);
+            }
+            else if (typeof value === "number") {
+                conditions.push(`${key} = ${value}`);
+            }
+            else if (typeof value === "boolean") {
+                conditions.push(`${key} = ${value}`);
+            }
+        }
+        return conditions.join(" AND ");
+    }
+    /**
+     * Process raw LanceDB results into SearchResult format.
+     */
+    processResults(results, scoreThreshold) {
+        const searchResults = [];
+        for (const row of results) {
+            // LanceDB returns _distance for vector search
+            const distance = row._distance ?? 0;
+            // Convert distance to similarity score (lower distance = higher similarity)
+            const score = 1 / (1 + distance);
+            if (scoreThreshold !== undefined && score < scoreThreshold) {
+                continue;
+            }
+            const metadata = row.metadata
+                ? JSON.parse(row.metadata)
+                : undefined;
+            searchResults.push({
+                document: {
+                    id: row.id,
+                    content: row.text,
+                    metadata,
+                },
+                score,
+            });
+        }
+        return searchResults;
+    }
+}
+exports.LanceDBVectorStore = LanceDBVectorStore;
+//# sourceMappingURL=LanceDBVectorStore.js.map

package/dist/vectorstore/OpenAIEmbeddings.d.ts

@@ -0,0 +1,45 @@
+/**
+ * OpenAI embeddings implementation.
+ *
+ * @requires openai - Uses the OpenAI SDK (already a peer dependency)
+ */
+import { Embeddings } from "./Embeddings";
+/**
+ * Configuration for OpenAI embeddings.
+ */
+export interface OpenAIEmbeddingsConfig {
+    /** OpenAI API key (defaults to OPENAI_API_KEY env var) */
+    apiKey?: string;
+    /** Model to use for embeddings */
+    model?: "text-embedding-3-small" | "text-embedding-3-large" | "text-embedding-ada-002" | string;
+    /** Number of dimensions (only for text-embedding-3-* models) */
+    dimensions?: number;
+    /** Base URL for API (for proxies or compatible APIs) */
+    baseURL?: string;
+}
+/**
+ * OpenAI embeddings provider.
+ *
+ * @example
+ * ```typescript
+ * const embeddings = new OpenAIEmbeddings({
+ *   model: 'text-embedding-3-small',
+ * });
+ *
+ * const vectors = await embeddings.embed(['Hello world', 'Goodbye world']);
+ * ```
+ */
+export declare class OpenAIEmbeddings extends Embeddings {
+    readonly name = "openai";
+    readonly model: string;
+    readonly dimensions: number;
+    private apiKey;
+    private baseURL?;
+    private requestedDimensions?;
+    constructor(config?: OpenAIEmbeddingsConfig);
+    /**
+     * Generate embeddings for multiple texts using OpenAI API.
+     */
+    embed(texts: string[]): Promise<number[][]>;
+}
+//# sourceMappingURL=OpenAIEmbeddings.d.ts.map

package/dist/vectorstore/OpenAIEmbeddings.js

@@ -0,0 +1,109 @@
+"use strict";
+/**
+ * OpenAI embeddings implementation.
+ *
+ * @requires openai - Uses the OpenAI SDK (already a peer dependency)
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIEmbeddings = void 0;
+const Embeddings_1 = require("./Embeddings");
+/** Default dimensions for each model */
+const MODEL_DIMENSIONS = {
+    "text-embedding-3-small": 1536,
+    "text-embedding-3-large": 3072,
+    "text-embedding-ada-002": 1536,
+};
+/**
+ * OpenAI embeddings provider.
+ *
+ * @example
+ * ```typescript
+ * const embeddings = new OpenAIEmbeddings({
+ *   model: 'text-embedding-3-small',
+ * });
+ *
+ * const vectors = await embeddings.embed(['Hello world', 'Goodbye world']);
+ * ```
+ */
+class OpenAIEmbeddings extends Embeddings_1.Embeddings {
+    constructor(config = {}) {
+        super();
+        this.name = "openai";
+        this.model = config.model ?? "text-embedding-3-small";
+        this.apiKey = config.apiKey ?? process.env.OPENAI_API_KEY ?? "";
+        this.baseURL = config.baseURL;
+        this.requestedDimensions = config.dimensions;
+        // Determine dimensions
+        if (config.dimensions) {
+            this.dimensions = config.dimensions;
+        }
+        else {
+            this.dimensions = MODEL_DIMENSIONS[this.model] ?? 1536;
+        }
+        if (!this.apiKey) {
+            throw new Error("OpenAI API key is required. Set OPENAI_API_KEY env var or pass apiKey in config.");
+        }
+    }
+    /**
+     * Generate embeddings for multiple texts using OpenAI API.
+     */
+    async embed(texts) {
+        if (texts.length === 0) {
+            return [];
+        }
+        // Dynamic import to keep openai optional at module load time
+        const { default: OpenAI } = await Promise.resolve().then(() => __importStar(require("openai")));
+        const client = new OpenAI({
+            apiKey: this.apiKey,
+            baseURL: this.baseURL,
+        });
+        const params = {
+            model: this.model,
+            input: texts,
+        };
+        // Only text-embedding-3-* models support custom dimensions
+        if (this.requestedDimensions &&
+            this.model.startsWith("text-embedding-3-")) {
+            params.dimensions = this.requestedDimensions;
+        }
+        const response = await client.embeddings.create(params);
+        // Sort by index to ensure order matches input
+        const sorted = response.data.sort((a, b) => a.index - b.index);
+        return sorted.map((item) => item.embedding);
+    }
+}
+exports.OpenAIEmbeddings = OpenAIEmbeddings;
+//# sourceMappingURL=OpenAIEmbeddings.js.map