@agentionai/agents 0.10.2 → 0.12.0-beta

package/dist/parsers/UnstructuredLocalParser.js

@@ -0,0 +1,118 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UnstructuredLocalParser = void 0;
+const DocumentParser_1 = require("./DocumentParser");
+/**
+ * Document parser that uses the **local** (open-source Python) version of
+ * Unstructured via the `@epilogo/unstructured-io-node` npm bridge.
+ *
+ * The bridge spawns a Python virtual environment and calls the Python
+ * `unstructured` library directly — no API key required, but Python 3.8+
+ * and system dependencies (poppler, tesseract, etc.) must be available.
+ *
+ * **Peer dependency:** `@epilogo/unstructured-io-node`
+ *
+ * @example
+ * ```typescript
+ * import { UnstructuredLocalParser } from "@agentionai/agents/parsers";
+ *
+ * const parser = new UnstructuredLocalParser();
+ * const doc = await parser.parse("/path/to/report.pdf", {
+ *   strategy: "hi_res",
+ *   languages: ["eng"],
+ * });
+ * console.log(doc.elements?.length, "elements");
+ *
+ * // Use with IngestionPipeline
+ * await pipeline.ingestFile("/path/to/report.pdf", parser);
+ * ```
+ */
+class UnstructuredLocalParser extends DocumentParser_1.DocumentParser {
+    constructor() {
+        super(...arguments);
+        this.name = "unstructured-local";
+    }
+    /**
+     * Parse a file using the local Python Unstructured library.
+     *
+     * On first call, `ensureEnvironmentSetup()` is invoked to download the
+     * Python venv if it does not already exist (one-time, slow operation).
+     *
+     * @param filePath - Path to the document to parse
+     * @param options  - Strategy, languages, and any other unstructured kwargs
+     */
+    async parse(filePath, options) {
+        const pkg = "@epilogo/unstructured-io-node";
+        let UnstructuredIO;
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ({ UnstructuredIO } = await Promise.resolve(`${pkg}`).then(s => __importStar(require(s))));
+        }
+        catch {
+            throw new Error("UnstructuredLocalParser requires '@epilogo/unstructured-io-node'. " +
+                "Install it with: npm install @epilogo/unstructured-io-node");
+        }
+        await UnstructuredIO.ensureEnvironmentSetup();
+        const { strategy, languages, ...rest } = options ?? {};
+        const rawElements = await UnstructuredIO.partition({
+            filename: filePath,
+            strategy: strategy ?? "auto",
+            ...(languages ? { languages } : {}),
+            ...rest,
+        });
+        const elements = this.mapRawElements(rawElements);
+        return {
+            text: this.elementsToText(elements),
+            elements,
+        };
+    }
+    mapRawElements(raw) {
+        return raw.map((el) => {
+            const e = el;
+            return {
+                type: typeof e["type"] === "string" ? e["type"] : "unknown",
+                text: typeof e["text"] === "string" ? e["text"] : "",
+                metadata: e["metadata"] != null &&
+                    typeof e["metadata"] === "object" &&
+                    !Array.isArray(e["metadata"])
+                    ? e["metadata"]
+                    : undefined,
+            };
+        });
+    }
+}
+exports.UnstructuredLocalParser = UnstructuredLocalParser;
+//# sourceMappingURL=UnstructuredLocalParser.js.map

package/dist/parsers/index.d.ts

@@ -0,0 +1,3 @@
+export type { ParsedElement, ParsedDocument, ParseOptions } from "./types";
+export { DocumentParser } from "./DocumentParser";
+//# sourceMappingURL=index.d.ts.map

package/dist/parsers/index.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DocumentParser = void 0;
+var DocumentParser_1 = require("./DocumentParser");
+Object.defineProperty(exports, "DocumentParser", { enumerable: true, get: function () { return DocumentParser_1.DocumentParser; } });
+//# sourceMappingURL=index.js.map

package/dist/parsers/types.d.ts

@@ -0,0 +1,50 @@
+/**
+ * A single structured element extracted from a document.
+ * Matches the element format returned by Unstructured and similar parsers.
+ */
+export interface ParsedElement {
+    /**
+     * Element type — e.g. "Title", "NarrativeText", "Table", "Image",
+     * "ListItem", "Header", "Footer", "Document", etc.
+     */
+    type: string;
+    /** Text content of this element */
+    text: string;
+    /**
+     * Parser-provided metadata — e.g. page_number, coordinates, languages,
+     * file_directory, filename, filetype, etc.
+     */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * The result of parsing a document file.
+ */
+export interface ParsedDocument {
+    /** Full plain-text content (elements joined by double newlines) */
+    text: string;
+    /**
+     * Structured elements if the parser supports them.
+     * Absent when the parser only returns plain text.
+     */
+    elements?: ParsedElement[];
+    /** File-level metadata from the parser, when available */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Options shared across all document parsers.
+ */
+export interface ParseOptions {
+    /**
+     * Parsing strategy.
+     * - `"auto"`: Let the parser decide (default)
+     * - `"fast"`: Text extraction only, no OCR
+     * - `"hi_res"`: High-resolution layout analysis with OCR
+     * - `"ocr_only"`: Force OCR on every page
+     */
+    strategy?: "auto" | "fast" | "hi_res" | "ocr_only";
+    /** Languages to use for OCR (ISO 639-1 codes, e.g. `["eng", "fra"]`) */
+    languages?: string[];
+    /** Pass-through options specific to the underlying parser */
+    [key: string]: unknown;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/parsers/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/vectorstore/LanceDBVectorStore.d.ts

@@ -8,23 +8,8 @@
  * @requires apache-arrow - Install with: npm install apache-arrow
  */
 import type { Connection, Table, ConnectionOptions } from "@lancedb/lancedb";
-import { VectorStore, Document, EmbeddedDocument, SearchResult, AddDocumentsOptions, SearchOptions, DeleteOptions } from "./VectorStore";
+import { VectorStore, Document, EmbeddedDocument, SearchResult, AddDocumentsOptions, SearchOptions, DeleteOptions, MetadataFieldDefinition } from "./VectorStore";
 import { Embeddings } from "../embeddings/Embeddings";
-/**
- * Supported types for metadata fields.
- */
-export type MetadataFieldType = "string" | "number" | "boolean";
-/**
- * Definition for a metadata field that will be stored as a separate column.
- */
-export interface MetadataFieldDefinition {
-    /** Name of the metadata field. Use snake_case (e.g. `tenant_id`) to avoid SQL filter issues. */
-    name: string;
-    /** Data type for the field */
-    type: MetadataFieldType;
-    /** Whether the field can be null (default: true) */
-    nullable?: boolean;
-}
 /**
  * Configuration for LanceDBVectorStore.
  */

package/dist/vectorstore/OpenSearchVectorStore.d.ts

@@ -0,0 +1,259 @@
+/**
+ * OpenSearch implementation of the VectorStore interface.
+ *
+ * Uses the OpenSearch k-NN plugin for approximate nearest-neighbour search
+ * via HNSW indexing. Supports cosine similarity, L2, and inner product
+ * space types.
+ *
+ * @requires @opensearch-project/opensearch - Install with: npm install @opensearch-project/opensearch
+ */
+import { VectorStore, Document, EmbeddedDocument, SearchResult, AddDocumentsOptions, SearchOptions, DeleteOptions, MetadataFieldDefinition } from "./VectorStore";
+import { Embeddings } from "../embeddings/Embeddings";
+interface OpenSearchHit<T> {
+    _id: string;
+    _score: number;
+    _source: T;
+}
+interface OpenSearchClient {
+    indices: {
+        exists(params: {
+            index: string;
+        }): Promise<{
+            body: boolean;
+        }>;
+        create(params: {
+            index: string;
+            body: unknown;
+        }): Promise<unknown>;
+        delete(params: {
+            index: string;
+        }): Promise<unknown>;
+    };
+    bulk(params: {
+        body: unknown[];
+        refresh?: boolean | string;
+    }): Promise<{
+        body: {
+            items?: Array<{
+                delete?: {
+                    result?: string;
+                };
+                index?: {
+                    result?: string;
+                };
+            }>;
+            errors?: boolean;
+        };
+    }>;
+    search<T>(params: {
+        index: string;
+        body: unknown;
+    }): Promise<{
+        body: {
+            hits?: {
+                hits?: OpenSearchHit<T>[];
+            };
+        };
+    }>;
+    get<T>(params: {
+        index: string;
+        id: string;
+    }): Promise<{
+        body: {
+            found: boolean;
+            _source: T;
+        };
+    }>;
+    deleteByQuery(params: {
+        index: string;
+        body: unknown;
+        refresh?: boolean | string;
+    }): Promise<unknown>;
+}
+/**
+ * k-NN vector space type used by the OpenSearch k-NN plugin.
+ * - `cosinesimil` — cosine similarity (default, normalised vectors recommended)
+ * - `l2`          — Euclidean L2 distance
+ * - `innerproduct` — inner / dot product
+ */
+export type OpenSearchSpaceType = "cosinesimil" | "l2" | "innerproduct";
+/**
+ * k-NN engine used by the OpenSearch k-NN plugin.
+ * - `lucene` — native Lucene ANN (cosinesimil and l2 only); default since OpenSearch 3.x
+ * - `faiss`  — high-throughput GPU-accelerated (l2 and innerproduct only)
+ * - `nmslib` — deprecated and removed in OpenSearch 3.0; do not use
+ */
+export type OpenSearchKnnEngine = "lucene" | "faiss" | "nmslib";
+/**
+ * Configuration for OpenSearchVectorStore.
+ */
+export interface OpenSearchVectorStoreConfig {
+    /** Name identifier for this store instance */
+    name: string;
+    /** OpenSearch node URL (e.g. `https://localhost:9200`) */
+    node: string;
+    /** Basic-auth credentials */
+    auth?: {
+        username: string;
+        password: string;
+    };
+    /** SSL options — set `rejectUnauthorized: false` for self-signed certs */
+    ssl?: {
+        rejectUnauthorized?: boolean;
+    };
+    /** OpenSearch index name to use for document storage */
+    indexName: string;
+    /** Embeddings provider for automatic embedding generation */
+    embeddings?: Embeddings;
+    /**
+     * Vector dimensions.
+     * Defaults to `embeddings.dimensions` when an embeddings provider is given,
+     * otherwise falls back to `1536`.
+     */
+    dimensions?: number;
+    /**
+     * k-NN vector space type (default: `"cosinesimil"`).
+     * Must match the space type the embeddings model was trained for.
+     */
+    spaceType?: OpenSearchSpaceType;
+    /**
+     * k-NN engine (default: `"lucene"`).
+     * `nmslib` was removed in OpenSearch 3.0 and cannot be used for new indices.
+     */
+    engine?: OpenSearchKnnEngine;
+    /**
+     * HNSW `ef_search` parameter — controls recall vs. latency at query time.
+     * Higher values improve recall at the cost of latency. Default: `512`.
+     */
+    efSearch?: number;
+    /**
+     * HNSW `ef_construction` parameter — controls graph quality at index time.
+     * Higher values improve recall at the cost of indexing speed. Default: `512`.
+     */
+    efConstruction?: number;
+    /**
+     * HNSW `M` parameter — number of bidirectional links per node.
+     * Higher values improve recall but increase memory usage. Default: `16`.
+     */
+    m?: number;
+    /**
+     * Optional user-defined metadata field definitions.
+     *
+     * When provided, these fields are declared in the index mapping with proper
+     * types (`keyword` for strings, `double` for numbers, `boolean` for booleans),
+     * which enables reliable exact-match filtering via `SearchOptions.filter`.
+     *
+     * Without this option, OpenSearch uses dynamic mapping for the `metadata`
+     * object. String fields are mapped as `text` with a `.keyword` sub-field —
+     * the store handles this automatically by appending `.keyword` to undeclared
+     * string filter values at query time.
+     *
+     * Chunk metadata fields produced by the library's chunkers (`hash`,
+     * `prev_id`, `next_id`, etc.) are always declared explicitly — you do not
+     * need to list them here.
+     *
+     * @example
+     * ```typescript
+     * metadataFields: [
+     *   { name: "source", type: "string" },
+     *   { name: "page",   type: "number" },
+     * ]
+     * ```
+     */
+    metadataFields?: MetadataFieldDefinition[];
+}
+export declare class OpenSearchVectorStore extends VectorStore {
+    readonly name: string;
+    private client;
+    private indexName;
+    private embeddings?;
+    private dimensions;
+    private spaceType;
+    private engine;
+    private efSearch;
+    private efConstruction;
+    private m;
+    private metadataFields?;
+    /** Set of metadata field names declared as keyword (string) type. */
+    private keywordFields;
+    private constructor();
+    /**
+     * Create a new OpenSearchVectorStore instance.
+     *
+     * Connects to the given OpenSearch node and creates the index (with k-NN
+     * mapping) if it does not already exist.
+     *
+     * @param config - Store configuration
+     * @returns A ready-to-use OpenSearchVectorStore instance
+     * @throws Error if `@opensearch-project/opensearch` is not installed
+     */
+    static create(config: OpenSearchVectorStoreConfig): Promise<OpenSearchVectorStore>;
+    /**
+     * Create the k-NN index if it does not already exist.
+     *
+     * The `metadata` object always includes explicit mappings for chunk metadata
+     * fields (hash, prev_id, etc.) so they work correctly in term queries.
+     * Any user-declared `metadataFields` are also mapped with proper types.
+     * All other metadata fields fall back to dynamic mapping.
+     */
+    private ensureIndex;
+    /**
+     * Add documents to the store.
+     * Embeddings are generated automatically using the configured provider.
+     */
+    addDocuments(documents: Document[], options?: AddDocumentsOptions): Promise<string[]>;
+    /**
+     * Add documents with pre-computed embeddings.
+     * Uses OpenSearch bulk API for efficiency.
+     */
+    addEmbeddedDocuments(documents: EmbeddedDocument[], options?: AddDocumentsOptions): Promise<string[]>;
+    /**
+     * Search for documents similar to the query text.
+     * The query is embedded automatically using the configured embeddings provider.
+     */
+    search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
+    /**
+     * Search using a pre-computed embedding vector.
+     * Executes a k-NN query against the OpenSearch index.
+     */
+    searchByVector(embedding: number[], options?: SearchOptions): Promise<SearchResult[]>;
+    /**
+     * Delete documents by their IDs.
+     * @returns Number of documents actually deleted.
+     */
+    delete(ids: string[], _options?: DeleteOptions): Promise<number>;
+    /**
+     * Delete all documents, optionally scoped to a namespace.
+     */
+    clear(options?: DeleteOptions): Promise<void>;
+    /**
+     * Retrieve a document by its ID.
+     * @returns The document, or `null` if not found.
+     */
+    getById(id: string, _options?: DeleteOptions): Promise<Document | null>;
+    /**
+     * Get existing documents by their content hashes.
+     * Used by the ingestion pipeline for deduplication.
+     *
+     * Requires that documents were stored with chunk metadata containing a
+     * `hash` field (automatically set by chunkers in this library).
+     *
+     * @returns Map of hash → document ID for hashes that already exist.
+     */
+    getByHashes(hashes: string[], _options?: DeleteOptions): Promise<Map<string, string>>;
+    /**
+     * Delete the entire OpenSearch index.
+     * WARNING: This permanently removes all indexed documents and the mapping.
+     */
+    deleteIndex(): Promise<void>;
+    /** The OpenSearch index name used by this store. */
+    getIndexName(): string;
+    /** The configured vector dimensions. */
+    getDimensions(): number;
+    /** The configured embeddings provider, if any. */
+    getEmbeddings(): Embeddings | undefined;
+    /** The underlying OpenSearch client instance. */
+    getClient(): OpenSearchClient;
+}
+export {};
+//# sourceMappingURL=OpenSearchVectorStore.d.ts.map