@agentionai/agents 0.10.2 → 0.12.0-beta

package/dist/vectorstore/OpenSearchVectorStore.js

@@ -0,0 +1,481 @@
+"use strict";
+/**
+ * OpenSearch implementation of the VectorStore interface.
+ *
+ * Uses the OpenSearch k-NN plugin for approximate nearest-neighbour search
+ * via HNSW indexing. Supports cosine similarity, L2, and inner product
+ * space types.
+ *
+ * @requires @opensearch-project/opensearch - Install with: npm install @opensearch-project/opensearch
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenSearchVectorStore = void 0;
+const VectorStore_1 = require("./VectorStore");
+// ---------------------------------------------------------------------------
+// Score normalisation helpers
+// ---------------------------------------------------------------------------
+/**
+ * Raw OpenSearch k-NN scores are not in the [0, 1] range for all space types.
+ * This function normalises them:
+ * - cosinesimil: OpenSearch returns `1 + cos(q, d)` → range [0, 2]; divide by 2.
+ * - l2:          OpenSearch returns `1 / (1 + l2_dist)` → already in (0, 1].
+ * - innerproduct: pass through as-is (application-defined interpretation).
+ */
+function normalizeScore(rawScore, spaceType) {
+    if (spaceType === "cosinesimil") {
+        return rawScore / 2;
+    }
+    return rawScore;
+}
+// ---------------------------------------------------------------------------
+// OpenSearchVectorStore
+// ---------------------------------------------------------------------------
+/**
+ * OpenSearch implementation of the VectorStore interface.
+ *
+ * Stores documents in an OpenSearch index with a `knn_vector` field and
+ * performs approximate nearest-neighbour search using the k-NN plugin (HNSW).
+ *
+ * **Namespace support**: namespaces are stored as a top-level `namespace`
+ * keyword field. All search / delete operations that receive a namespace
+ * automatically add a term filter on this field.
+ *
+ * **Metadata**: stored as a nested `metadata` object with dynamic mapping.
+ * Chunk metadata fields produced by the library's chunkers (e.g. `hash`,
+ * `prev_id`, `next_id`) live inside `metadata` and are searchable via
+ * `metadata.<field>` queries.
+ *
+ * @example Basic setup with OpenAI embeddings
+ * ```typescript
+ * import { OpenSearchVectorStore } from "@agentionai/agents/vectorstore";
+ * import { OpenAIEmbeddings } from "@agentionai/agents/embeddings";
+ *
+ * const embeddings = new OpenAIEmbeddings({ model: "text-embedding-3-small" });
+ *
+ * const store = await OpenSearchVectorStore.create({
+ *   name: "my_store",
+ *   node: "https://localhost:9200",
+ *   auth: { username: "admin", password: "admin" },
+ *   ssl: { rejectUnauthorized: false },
+ *   indexName: "knowledge_base",
+ *   embeddings,
+ * });
+ *
+ * await store.addDocuments([
+ *   { id: "1", content: "OpenSearch is a distributed search engine.", metadata: { source: "docs" } },
+ * ]);
+ *
+ * const results = await store.search("distributed search", { limit: 5 });
+ * ```
+ *
+ * @example Use as an agent retrieval tool
+ * ```typescript
+ * const searchTool = store.toRetrievalTool("Search product documentation");
+ * const agent = new ClaudeAgent({ tools: [searchTool], ... });
+ * ```
+ */
+/**
+ * Chunk metadata field names always declared explicitly in the mapping.
+ * These are produced by the library's chunkers and used internally for
+ * deduplication (hash) and chunk navigation (prev_id, next_id).
+ */
+const CHUNK_KEYWORD_FIELDS = new Set([
+    "hash", "prev_id", "next_id", "source_id", "source_path", "section",
+]);
+const CHUNK_NUMERIC_FIELDS = new Set([
+    "index", "total", "start", "end", "char_count", "token_count", "page",
+]);
+class OpenSearchVectorStore extends VectorStore_1.VectorStore {
+    constructor(config, client) {
+        super();
+        this.name = config.name;
+        this.client = client;
+        this.indexName = config.indexName;
+        this.embeddings = config.embeddings;
+        this.dimensions =
+            config.dimensions ?? config.embeddings?.dimensions ?? 1536;
+        this.spaceType = config.spaceType ?? "cosinesimil";
+        this.engine = config.engine ?? "lucene";
+        this.efSearch = config.efSearch ?? 512;
+        this.efConstruction = config.efConstruction ?? 512;
+        this.m = config.m ?? 16;
+        this.metadataFields = config.metadataFields;
+        // Build the set of field names that are explicitly mapped as keyword.
+        // Used by the filter builder to decide whether to append ".keyword".
+        this.keywordFields = new Set(CHUNK_KEYWORD_FIELDS);
+        for (const field of config.metadataFields ?? []) {
+            if (field.type === "string") {
+                this.keywordFields.add(field.name);
+            }
+        }
+    }
+    /**
+     * Create a new OpenSearchVectorStore instance.
+     *
+     * Connects to the given OpenSearch node and creates the index (with k-NN
+     * mapping) if it does not already exist.
+     *
+     * @param config - Store configuration
+     * @returns A ready-to-use OpenSearchVectorStore instance
+     * @throws Error if `@opensearch-project/opensearch` is not installed
+     */
+    static async create(config) {
+        let ClientCtor;
+        try {
+            // Use a variable so TypeScript does not attempt static module resolution
+            // for this optional peer dependency.
+            const pkgName = "@opensearch-project/opensearch";
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const mod = (await Promise.resolve(`${pkgName}`).then(s => __importStar(require(s))));
+            ClientCtor = mod.Client;
+        }
+        catch {
+            throw new Error("@opensearch-project/opensearch is not installed. " +
+                "Install it with: npm install @opensearch-project/opensearch");
+        }
+        const clientCfg = { node: config.node };
+        if (config.auth)
+            clientCfg.auth = config.auth;
+        if (config.ssl)
+            clientCfg.ssl = config.ssl;
+        const client = new ClientCtor(clientCfg);
+        const store = new OpenSearchVectorStore(config, client);
+        await store.ensureIndex();
+        return store;
+    }
+    // -------------------------------------------------------------------------
+    // Index management
+    // -------------------------------------------------------------------------
+    /**
+     * Create the k-NN index if it does not already exist.
+     *
+     * The `metadata` object always includes explicit mappings for chunk metadata
+     * fields (hash, prev_id, etc.) so they work correctly in term queries.
+     * Any user-declared `metadataFields` are also mapped with proper types.
+     * All other metadata fields fall back to dynamic mapping.
+     */
+    async ensureIndex() {
+        const { body: exists } = await this.client.indices.exists({
+            index: this.indexName,
+        });
+        if (exists)
+            return;
+        // Build explicit sub-properties for the metadata object.
+        const metadataProperties = {};
+        // Chunk metadata fields — always declared with correct types.
+        for (const field of CHUNK_KEYWORD_FIELDS) {
+            metadataProperties[field] = { type: "keyword" };
+        }
+        for (const field of CHUNK_NUMERIC_FIELDS) {
+            metadataProperties[field] = { type: "integer" };
+        }
+        // User-declared metadata fields.
+        for (const field of this.metadataFields ?? []) {
+            if (field.type === "number") {
+                metadataProperties[field.name] = { type: "double" };
+            }
+            else if (field.type === "boolean") {
+                metadataProperties[field.name] = { type: "boolean" };
+            }
+            else {
+                metadataProperties[field.name] = { type: "keyword" };
+            }
+        }
+        await this.client.indices.create({
+            index: this.indexName,
+            body: {
+                settings: {
+                    index: {
+                        knn: true,
+                        "knn.algo_param.ef_search": this.efSearch,
+                    },
+                },
+                mappings: {
+                    properties: {
+                        id: { type: "keyword" },
+                        content: { type: "text" },
+                        embedding: {
+                            type: "knn_vector",
+                            dimension: this.dimensions,
+                            method: {
+                                name: "hnsw",
+                                space_type: this.spaceType,
+                                engine: this.engine,
+                                parameters: {
+                                    ef_construction: this.efConstruction,
+                                    m: this.m,
+                                },
+                            },
+                        },
+                        namespace: { type: "keyword" },
+                        metadata: {
+                            type: "object",
+                            dynamic: true, // undeclared fields still work via dynamic mapping
+                            properties: metadataProperties,
+                        },
+                    },
+                },
+            },
+        });
+    }
+    // -------------------------------------------------------------------------
+    // VectorStore abstract method implementations
+    // -------------------------------------------------------------------------
+    /**
+     * Add documents to the store.
+     * Embeddings are generated automatically using the configured provider.
+     */
+    async addDocuments(documents, options) {
+        if (!this.embeddings) {
+            throw new Error("No embeddings provider configured. " +
+                "Use addEmbeddedDocuments() with pre-computed embeddings, " +
+                "or pass an embeddings provider in the config.");
+        }
+        const texts = documents.map((d) => d.content);
+        const vectors = await this.embeddings.embed(texts);
+        const embedded = documents.map((doc, i) => ({
+            ...doc,
+            embedding: vectors[i],
+        }));
+        return this.addEmbeddedDocuments(embedded, options);
+    }
+    /**
+     * Add documents with pre-computed embeddings.
+     * Uses OpenSearch bulk API for efficiency.
+     */
+    async addEmbeddedDocuments(documents, options) {
+        if (documents.length === 0)
+            return [];
+        const namespace = options?.namespace;
+        // Flatten documents into bulk request body
+        const body = [];
+        for (const doc of documents) {
+            body.push({ index: { _index: this.indexName, _id: doc.id } });
+            const osDoc = {
+                id: doc.id,
+                content: doc.content,
+                embedding: doc.embedding,
+                metadata: doc.metadata,
+            };
+            if (namespace)
+                osDoc.namespace = namespace;
+            body.push(osDoc);
+        }
+        await this.client.bulk({ body, refresh: true });
+        return documents.map((d) => d.id);
+    }
+    /**
+     * Search for documents similar to the query text.
+     * The query is embedded automatically using the configured embeddings provider.
+     */
+    async search(query, options) {
+        if (!this.embeddings) {
+            throw new Error("No embeddings provider configured. " +
+                "Use searchByVector() with a pre-computed query embedding, " +
+                "or pass an embeddings provider in the config.");
+        }
+        const queryVector = await this.embeddings.embedQuery(query);
+        return this.searchByVector(queryVector, options);
+    }
+    /**
+     * Search using a pre-computed embedding vector.
+     * Executes a k-NN query against the OpenSearch index.
+     */
+    async searchByVector(embedding, options) {
+        const limit = options?.limit ?? 10;
+        const scoreThreshold = options?.scoreThreshold;
+        const namespace = options?.namespace;
+        const filter = options?.filter;
+        // Build bool filters for namespace and metadata
+        const filters = [];
+        if (namespace) {
+            filters.push({ term: { namespace } });
+        }
+        if (filter) {
+            for (const [key, value] of Object.entries(filter)) {
+                // Declared keyword fields can be queried directly.
+                // Undeclared string values are dynamically mapped as text+keyword;
+                // append ".keyword" to target the exact-match sub-field.
+                const isUndeclaredString = typeof value === "string" && !this.keywordFields.has(key);
+                const fieldPath = isUndeclaredString
+                    ? `metadata.${key}.keyword`
+                    : `metadata.${key}`;
+                filters.push({ term: { [fieldPath]: value } });
+            }
+        }
+        const knnClause = {
+            embedding: { vector: embedding, k: limit },
+        };
+        const queryBody = filters.length > 0
+            ? {
+                bool: {
+                    must: [{ knn: knnClause }],
+                    filter: filters,
+                },
+            }
+            : { knn: knnClause };
+        const response = await this.client.search({
+            index: this.indexName,
+            body: { size: limit, query: queryBody },
+        });
+        const hits = response.body.hits?.hits ?? [];
+        const results = [];
+        for (const hit of hits) {
+            const score = normalizeScore(hit._score, this.spaceType);
+            if (scoreThreshold !== undefined && score < scoreThreshold)
+                continue;
+            results.push({
+                document: {
+                    id: hit._source.id,
+                    content: hit._source.content,
+                    metadata: hit._source.metadata,
+                },
+                score,
+            });
+        }
+        return results;
+    }
+    /**
+     * Delete documents by their IDs.
+     * @returns Number of documents actually deleted.
+     */
+    async delete(ids, _options) {
+        if (ids.length === 0)
+            return 0;
+        const body = ids.map((id) => ({
+            delete: { _index: this.indexName, _id: id },
+        }));
+        const response = await this.client.bulk({ body, refresh: true });
+        return (response.body.items ?? []).filter((item) => item.delete?.result === "deleted").length;
+    }
+    /**
+     * Delete all documents, optionally scoped to a namespace.
+     */
+    async clear(options) {
+        const namespace = options?.namespace;
+        const queryBody = namespace
+            ? { query: { term: { namespace } } }
+            : { query: { match_all: {} } };
+        await this.client.deleteByQuery({
+            index: this.indexName,
+            body: queryBody,
+            refresh: true,
+        });
+    }
+    /**
+     * Retrieve a document by its ID.
+     * @returns The document, or `null` if not found.
+     */
+    async getById(id, _options) {
+        try {
+            const response = await this.client.get({
+                index: this.indexName,
+                id,
+            });
+            if (!response.body.found)
+                return null;
+            const src = response.body._source;
+            return {
+                id: src.id,
+                content: src.content,
+                metadata: src.metadata,
+            };
+        }
+        catch (err) {
+            if (err.statusCode === 404)
+                return null;
+            throw err;
+        }
+    }
+    /**
+     * Get existing documents by their content hashes.
+     * Used by the ingestion pipeline for deduplication.
+     *
+     * Requires that documents were stored with chunk metadata containing a
+     * `hash` field (automatically set by chunkers in this library).
+     *
+     * @returns Map of hash → document ID for hashes that already exist.
+     */
+    async getByHashes(hashes, _options) {
+        const hashMap = new Map();
+        if (hashes.length === 0)
+            return hashMap;
+        const response = await this.client.search({
+            index: this.indexName,
+            body: {
+                size: hashes.length,
+                query: { terms: { "metadata.hash": hashes } },
+                _source: ["id", "metadata.hash"],
+            },
+        });
+        for (const hit of response.body.hits?.hits ?? []) {
+            const hash = hit._source.metadata
+                ?.hash;
+            if (hash) {
+                hashMap.set(hash, hit._source.id);
+            }
+        }
+        return hashMap;
+    }
+    // -------------------------------------------------------------------------
+    // OpenSearch-specific accessors
+    // -------------------------------------------------------------------------
+    /**
+     * Delete the entire OpenSearch index.
+     * WARNING: This permanently removes all indexed documents and the mapping.
+     */
+    async deleteIndex() {
+        await this.client.indices.delete({ index: this.indexName });
+    }
+    /** The OpenSearch index name used by this store. */
+    getIndexName() {
+        return this.indexName;
+    }
+    /** The configured vector dimensions. */
+    getDimensions() {
+        return this.dimensions;
+    }
+    /** The configured embeddings provider, if any. */
+    getEmbeddings() {
+        return this.embeddings;
+    }
+    /** The underlying OpenSearch client instance. */
+    getClient() {
+        return this.client;
+    }
+}
+exports.OpenSearchVectorStore = OpenSearchVectorStore;
+//# sourceMappingURL=OpenSearchVectorStore.js.map

package/dist/vectorstore/VectorStore.d.ts

@@ -5,6 +5,31 @@
  * enabling document storage with embeddings and semantic search capabilities.
  */
 import { Tool } from "../tools/Tool";
+/**
+ * Supported types for metadata field definitions.
+ */
+export type MetadataFieldType = "string" | "number" | "boolean";
+/**
+ * Definition for a typed metadata field.
+ * Used by vector store implementations to declare explicit field types
+ * for metadata properties, enabling correct indexing and filtering.
+ *
+ * @example
+ * ```typescript
+ * const fields: MetadataFieldDefinition[] = [
+ *   { name: "source", type: "string" },
+ *   { name: "page", type: "number" },
+ * ];
+ * ```
+ */
+export interface MetadataFieldDefinition {
+    /** Name of the metadata field. Use snake_case (e.g. `tenant_id`). */
+    name: string;
+    /** Data type for the field */
+    type: MetadataFieldType;
+    /** Whether the field can be null (default: true) */
+    nullable?: boolean;
+}
 /**
  * Represents a document with its content and optional metadata.
  */

package/dist/vectorstore/index.d.ts

@@ -22,8 +22,9 @@
  * const addTool = store.toAddDocumentsTool("Save new documents");
  * ```
  */
-export { VectorStore, Document, EmbeddedDocument, SearchResult, AddDocumentsOptions, SearchOptions, DeleteOptions, RetrievalToolOptions, AddDocumentsToolOptions, } from "./VectorStore";
-export { LanceDBVectorStore, LanceDBVectorStoreConfig, MetadataFieldType, MetadataFieldDefinition, } from "./LanceDBVectorStore";
+export { VectorStore, Document, EmbeddedDocument, SearchResult, AddDocumentsOptions, SearchOptions, DeleteOptions, RetrievalToolOptions, AddDocumentsToolOptions, MetadataFieldType, MetadataFieldDefinition, } from "./VectorStore";
+export { LanceDBVectorStore, LanceDBVectorStoreConfig, } from "./LanceDBVectorStore";
+export { OpenSearchVectorStore, OpenSearchVectorStoreConfig, OpenSearchSpaceType, OpenSearchKnnEngine, } from "./OpenSearchVectorStore";
 export { Embeddings, EmbeddingOptions } from "../embeddings/Embeddings";
 export { OpenAIEmbeddings, OpenAIEmbeddingsConfig, OpenAIEmbeddingModel, } from "../embeddings/OpenAIEmbeddings";
 export { VoyageAIEmbeddings, VoyageAIEmbeddingsConfig, VoyageAIEmbeddingModel, VoyageAIMultimodalModel, } from "../embeddings/VoyageAIEmbeddings";

package/dist/vectorstore/index.js

@@ -24,11 +24,13 @@
  * ```
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.VoyageAIEmbeddings = exports.OpenAIEmbeddings = exports.Embeddings = exports.LanceDBVectorStore = exports.VectorStore = void 0;
+exports.VoyageAIEmbeddings = exports.OpenAIEmbeddings = exports.Embeddings = exports.OpenSearchVectorStore = exports.LanceDBVectorStore = exports.VectorStore = void 0;
 var VectorStore_1 = require("./VectorStore");
 Object.defineProperty(exports, "VectorStore", { enumerable: true, get: function () { return VectorStore_1.VectorStore; } });
 var LanceDBVectorStore_1 = require("./LanceDBVectorStore");
 Object.defineProperty(exports, "LanceDBVectorStore", { enumerable: true, get: function () { return LanceDBVectorStore_1.LanceDBVectorStore; } });
+var OpenSearchVectorStore_1 = require("./OpenSearchVectorStore");
+Object.defineProperty(exports, "OpenSearchVectorStore", { enumerable: true, get: function () { return OpenSearchVectorStore_1.OpenSearchVectorStore; } });
 // Re-export embeddings for backward compatibility
 var Embeddings_1 = require("../embeddings/Embeddings");
 Object.defineProperty(exports, "Embeddings", { enumerable: true, get: function () { return Embeddings_1.Embeddings; } });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@agentionai/agents",
   "author": "Laurent Zuijdwijk",
-  "version": "0.10.2",
+  "version": "0.12.0-beta",
   "description": "Agent Library",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -30,6 +30,10 @@
       "types": "./dist/gemini.d.ts",
       "default": "./dist/gemini.js"
     },
+    "./ollama": {
+      "types": "./dist/agents/ollama/OllamaAgent.d.ts",
+      "default": "./dist/agents/ollama/OllamaAgent.js"
+    },
     "./embeddings": {
       "types": "./dist/embeddings/index.d.ts",
       "default": "./dist/embeddings/index.js"
@@ -61,6 +65,26 @@
     "./history/plugins": {
       "types": "./dist/history/plugins/index.d.ts",
       "default": "./dist/history/plugins/index.js"
+    },
+    "./parsers": {
+      "types": "./dist/parsers/index.d.ts",
+      "default": "./dist/parsers/index.js"
+    },
+    "./parsers/unstructured-local": {
+      "types": "./dist/parsers/UnstructuredLocalParser.d.ts",
+      "default": "./dist/parsers/UnstructuredLocalParser.js"
+    },
+    "./parsers/unstructured-api": {
+      "types": "./dist/parsers/UnstructuredAPIParser.d.ts",
+      "default": "./dist/parsers/UnstructuredAPIParser.js"
+    },
+    "./parsers/llamaindex": {
+      "types": "./dist/parsers/LlamaIndexParser.d.ts",
+      "default": "./dist/parsers/LlamaIndexParser.js"
+    },
+    "./parsers/ollama-ocr": {
+      "types": "./dist/parsers/OllamaOCRParser.d.ts",
+      "default": "./dist/parsers/OllamaOCRParser.js"
     }
   },
   "files": [
@@ -141,7 +165,12 @@
     "@modelcontextprotocol/sdk": "^1.26.0",
     "apache-arrow": "^18.0.0",
     "openai": "^6.16.0",
-    "voyageai": "^0.0.3"
+    "voyageai": "^0.0.3",
+    "@epilogo/unstructured-io-node": "*",
+    "unstructured-client": "*",
+    "llamaindex": "*",
+    "@llamaindex/readers": "*",
+    "pdf-to-img": "*"
   },
   "peerDependenciesMeta": {
     "@lancedb/lancedb": {
@@ -167,9 +196,28 @@
     },
     "voyageai": {
       "optional": true
+    },
+    "@opensearch-project/opensearch": {
+      "optional": true
+    },
+    "@epilogo/unstructured-io-node": {
+      "optional": true
+    },
+    "unstructured-client": {
+      "optional": true
+    },
+    "llamaindex": {
+      "optional": true
+    },
+    "@llamaindex/readers": {
+      "optional": true
+    },
+    "pdf-to-img": {
+      "optional": true
     }
   },
   "dependencies": {
+    "@opensearch-project/opensearch": "^3.5.1",
     "tokenx": "^1.2.1"
   }
 }