@agentionai/agents 0.7.0 → 0.8.1-beta

package/dist/chunkers/Chunker.d.ts

@@ -29,7 +29,7 @@ export declare abstract class Chunker {
      */
     protected computeHash(content: string): string;
     /**
-     * Link chunks with previousChunkId and nextChunkId.
+     * Link chunks with prev_id and next_id.
      */
     protected linkChunks(chunks: Chunk[]): void;
     /**

package/dist/chunkers/Chunker.js

@@ -53,17 +53,17 @@ class Chunker {
             }
             const id = this.generateId(content, i, options?.sourceId);
             const metadata = {
-                chunkIndex: i,
-                totalChunks: splits.length,
-                previousChunkId: null, // Will be linked after
-                nextChunkId: null, // Will be linked after
-                startOffset,
-                endOffset,
-                sourceId: options?.sourceId,
-                sourcePath: options?.sourcePath,
-                charCount: content.length,
+                index: i,
+                total: splits.length,
+                prev_id: null, // Will be linked after
+                next_id: null, // Will be linked after
+                start: startOffset,
+                end: endOffset,
+                source_id: options?.sourceId,
+                source_path: options?.sourcePath,
+                char_count: content.length,
                 hash: this.computeHash(content),
-                sectionTitle: currentSection,
+                section: currentSection,
                 ...options?.metadata,
             };
             chunks.push({ id, content, metadata });
@@ -71,9 +71,9 @@ class Chunker {
         }
         // Link chunks together
         this.linkChunks(chunks);
-        // Update totalChunks now that we know the final count
+        // Update total now that we know the final count
         for (const chunk of chunks) {
-            chunk.metadata.totalChunks = chunks.length;
+            chunk.metadata.total = chunks.length;
         }
         // Apply processor if provided
         if (this.config.chunkProcessor) {
@@ -100,15 +100,15 @@ class Chunker {
         return (0, crypto_1.createHash)("sha256").update(content).digest("hex");
     }
     /**
-     * Link chunks with previousChunkId and nextChunkId.
+     * Link chunks with prev_id and next_id.
      */
     linkChunks(chunks) {
         for (let i = 0; i < chunks.length; i++) {
             if (i > 0) {
-                chunks[i].metadata.previousChunkId = chunks[i - 1].id;
+                chunks[i].metadata.prev_id = chunks[i - 1].id;
             }
             if (i < chunks.length - 1) {
-                chunks[i].metadata.nextChunkId = chunks[i + 1].id;
+                chunks[i].metadata.next_id = chunks[i + 1].id;
             }
         }
     }
@@ -128,11 +128,10 @@ class Chunker {
         }
         // Re-link after filtering and update indices
         for (let i = 0; i < processed.length; i++) {
-            processed[i].metadata.chunkIndex = i;
-            processed[i].metadata.totalChunks = processed.length;
-            processed[i].metadata.previousChunkId =
-                i > 0 ? processed[i - 1].id : null;
-            processed[i].metadata.nextChunkId =
+            processed[i].metadata.index = i;
+            processed[i].metadata.total = processed.length;
+            processed[i].metadata.prev_id = i > 0 ? processed[i - 1].id : null;
+            processed[i].metadata.next_id =
                 i < processed.length - 1 ? processed[i + 1].id : null;
         }
         return processed;

package/dist/chunkers/TokenChunker.d.ts

@@ -26,7 +26,7 @@ export declare function resetTokenxCache(): void;
  * });
  *
  * const chunks = await chunker.chunk(longDocument);
- * // Each chunk.metadata.tokenCount contains estimated tokens
+ * // Each chunk.metadata.token_count contains estimated tokens
  * ```
  */
 export declare class TokenChunker extends Chunker {

package/dist/chunkers/TokenChunker.js

@@ -46,7 +46,6 @@ let tokenxModule = null;
  */
 async function loadTokenx() {
     if (!tokenxModule) {
-        // Use dynamic import for ESM module
         tokenxModule = await Promise.resolve().then(() => __importStar(require("tokenx")));
     }
     return tokenxModule;
@@ -73,7 +72,7 @@ function resetTokenxCache() {
  * });
  *
  * const chunks = await chunker.chunk(longDocument);
- * // Each chunk.metadata.tokenCount contains estimated tokens
+ * // Each chunk.metadata.token_count contains estimated tokens
  * ```
  */
 class TokenChunker extends Chunker_1.Chunker {
@@ -160,7 +159,7 @@ class TokenChunker extends Chunker_1.Chunker {
         const { estimateTokenCount } = tokenx;
         // Add token count to each chunk's metadata
         for (const chunk of chunks) {
-            chunk.metadata.tokenCount = estimateTokenCount(chunk.content);
+            chunk.metadata.token_count = estimateTokenCount(chunk.content);
         }
         return chunks;
     }

package/dist/chunkers/types.d.ts

@@ -11,32 +11,38 @@ export interface Chunk {
 }
 /**
  * Metadata associated with each chunk.
+ *
+ * When stored in LanceDB via `LanceDBVectorStore`, these fields are
+ * automatically packed into a `chunk_metadata` struct column — they do
+ * not need to be declared in `metadataFields`.
  */
 export interface ChunkMetadata {
     /** Zero-based index of this chunk in the sequence */
-    chunkIndex: number;
+    index: number;
     /** Total number of chunks in the sequence */
-    totalChunks: number;
+    total: number;
     /** ID of the previous chunk, or null if first */
-    previousChunkId: string | null;
+    prev_id: string | null;
     /** ID of the next chunk, or null if last */
-    nextChunkId: string | null;
+    next_id: string | null;
     /** Character offset where this chunk starts in the source text */
-    startOffset: number;
+    start: number;
     /** Character offset where this chunk ends in the source text */
-    endOffset: number;
+    end: number;
     /** Optional identifier for the source document */
-    sourceId?: string;
+    source_id?: string;
     /** Optional path to the source file */
-    sourcePath?: string;
+    source_path?: string;
     /** Number of characters in the chunk content */
-    charCount: number;
+    char_count: number;
     /** Estimated number of tokens (when available) */
-    tokenCount?: number;
+    token_count?: number;
     /** SHA-256 hash of the content for deduplication */
     hash: string;
     /** Section title if detected (e.g., markdown headers) */
-    sectionTitle?: string;
+    section?: string;
+    /** Page number in the source document (e.g., PDF page) */
+    page?: number;
     [key: string]: unknown;
 }
 /**

package/dist/claude.d.ts

@@ -1,9 +1,4 @@
-export * from "./agents/BaseAgent";
+export * from "./core";
 export * from "./agents/anthropic/ClaudeAgent";
-export * from "./agents/model-types";
 export { anthropicTransformer } from "./history/transformers";
-export * from "./history/History";
-export * from "./history/types";
-export * from "./tools/Tool";
-export * from "./graph/AgentGraph";
 //# sourceMappingURL=claude.d.ts.map

package/dist/claude.js

@@ -16,14 +16,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.anthropicTransformer = void 0;
 // Claude Agent Entry Point
-__exportStar(require("./agents/BaseAgent"), exports);
+__exportStar(require("./core"), exports);
 __exportStar(require("./agents/anthropic/ClaudeAgent"), exports);
-__exportStar(require("./agents/model-types"), exports);
 var transformers_1 = require("./history/transformers");
 Object.defineProperty(exports, "anthropicTransformer", { enumerable: true, get: function () { return transformers_1.anthropicTransformer; } });
-// Re-export core functionality
-__exportStar(require("./history/History"), exports);
-__exportStar(require("./history/types"), exports);
-__exportStar(require("./tools/Tool"), exports);
-__exportStar(require("./graph/AgentGraph"), exports);
 //# sourceMappingURL=claude.js.map

package/dist/core.d.ts

@@ -9,6 +9,7 @@ export * from "./graph/AgentGraph";
 export * from "./tools/Tool";
 export * from "./mcp";
 export * from "./viz";
+export * from "./embeddings";
 export * from "./vectorstore";
 export * from "./chunkers";
 export * from "./ingestion";

package/dist/core.js

@@ -33,6 +33,8 @@ __exportStar(require("./tools/Tool"), exports);
 __exportStar(require("./mcp"), exports);
 // Visualization
 __exportStar(require("./viz"), exports);
+// Embeddings
+__exportStar(require("./embeddings"), exports);
 // Vector Store
 __exportStar(require("./vectorstore"), exports);
 // Chunkers

package/dist/gemini.d.ts

@@ -1,9 +1,4 @@
-export * from "./agents/BaseAgent";
+export * from "./core";
 export { GeminiAgent } from "./agents/google/GeminiAgent";
-export * from "./agents/model-types";
 export { geminiTransformer } from "./history/transformers";
-export * from "./history/History";
-export * from "./history/types";
-export * from "./tools/Tool";
-export * from "./graph/AgentGraph";
 //# sourceMappingURL=gemini.d.ts.map

package/dist/gemini.js

@@ -16,15 +16,9 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.geminiTransformer = exports.GeminiAgent = void 0;
 // Gemini Agent Entry Point
-__exportStar(require("./agents/BaseAgent"), exports);
+__exportStar(require("./core"), exports);
 var GeminiAgent_1 = require("./agents/google/GeminiAgent");
 Object.defineProperty(exports, "GeminiAgent", { enumerable: true, get: function () { return GeminiAgent_1.GeminiAgent; } });
-__exportStar(require("./agents/model-types"), exports);
 var transformers_1 = require("./history/transformers");
 Object.defineProperty(exports, "geminiTransformer", { enumerable: true, get: function () { return transformers_1.geminiTransformer; } });
-// Re-export core functionality
-__exportStar(require("./history/History"), exports);
-__exportStar(require("./history/types"), exports);
-__exportStar(require("./tools/Tool"), exports);
-__exportStar(require("./graph/AgentGraph"), exports);
 //# sourceMappingURL=gemini.js.map

package/dist/index.d.ts

@@ -4,6 +4,9 @@ export { OpenAiAgent } from "./agents/openai/OpenAiAgent";
 export { MistralAgent } from "./agents/mistral/MistralAgent";
 export { GeminiAgent } from "./agents/google/GeminiAgent";
 export * from "./agents/model-types";
+export * from "./agents/AgentConfig";
+export * from "./agents/AgentEvent";
+export * from "./agents/errors/AgentError";
 export * from "./history/History";
 export * from "./history/types";
 export { anthropicTransformer, openAiTransformer, mistralTransformer, geminiTransformer, } from "./history/transformers";

package/dist/index.js

@@ -33,6 +33,9 @@ Object.defineProperty(exports, "MistralAgent", { enumerable: true, get: function
 var GeminiAgent_1 = require("./agents/google/GeminiAgent");
 Object.defineProperty(exports, "GeminiAgent", { enumerable: true, get: function () { return GeminiAgent_1.GeminiAgent; } });
 __exportStar(require("./agents/model-types"), exports);
+__exportStar(require("./agents/AgentConfig"), exports);
+__exportStar(require("./agents/AgentEvent"), exports);
+__exportStar(require("./agents/errors/AgentError"), exports);
 // History
 __exportStar(require("./history/History"), exports);
 __exportStar(require("./history/types"), exports);

package/dist/mistral.d.ts

@@ -1,9 +1,4 @@
-export * from "./agents/BaseAgent";
+export * from "./core";
 export { MistralAgent } from "./agents/mistral/MistralAgent";
-export * from "./agents/model-types";
 export { mistralTransformer } from "./history/transformers";
-export * from "./history/History";
-export * from "./history/types";
-export * from "./tools/Tool";
-export * from "./graph/AgentGraph";
 //# sourceMappingURL=mistral.d.ts.map

package/dist/mistral.js

@@ -16,15 +16,9 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.mistralTransformer = exports.MistralAgent = void 0;
 // Mistral Agent Entry Point
-__exportStar(require("./agents/BaseAgent"), exports);
+__exportStar(require("./core"), exports);
 var MistralAgent_1 = require("./agents/mistral/MistralAgent");
 Object.defineProperty(exports, "MistralAgent", { enumerable: true, get: function () { return MistralAgent_1.MistralAgent; } });
-__exportStar(require("./agents/model-types"), exports);
 var transformers_1 = require("./history/transformers");
 Object.defineProperty(exports, "mistralTransformer", { enumerable: true, get: function () { return transformers_1.mistralTransformer; } });
-// Re-export core functionality
-__exportStar(require("./history/History"), exports);
-__exportStar(require("./history/types"), exports);
-__exportStar(require("./tools/Tool"), exports);
-__exportStar(require("./graph/AgentGraph"), exports);
 //# sourceMappingURL=mistral.js.map

package/dist/openai.d.ts

@@ -1,9 +1,4 @@
-export * from "./agents/BaseAgent";
+export * from "./core";
 export { OpenAiAgent } from "./agents/openai/OpenAiAgent";
-export * from "./agents/model-types";
 export { openAiTransformer } from "./history/transformers";
-export * from "./history/History";
-export * from "./history/types";
-export * from "./tools/Tool";
-export * from "./graph/AgentGraph";
 //# sourceMappingURL=openai.d.ts.map

package/dist/openai.js

@@ -16,15 +16,9 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.openAiTransformer = exports.OpenAiAgent = void 0;
 // OpenAI Agent Entry Point
-__exportStar(require("./agents/BaseAgent"), exports);
+__exportStar(require("./core"), exports);
 var OpenAiAgent_1 = require("./agents/openai/OpenAiAgent");
 Object.defineProperty(exports, "OpenAiAgent", { enumerable: true, get: function () { return OpenAiAgent_1.OpenAiAgent; } });
-__exportStar(require("./agents/model-types"), exports);
 var transformers_1 = require("./history/transformers");
 Object.defineProperty(exports, "openAiTransformer", { enumerable: true, get: function () { return transformers_1.openAiTransformer; } });
-// Re-export core functionality
-__exportStar(require("./history/History"), exports);
-__exportStar(require("./history/types"), exports);
-__exportStar(require("./tools/Tool"), exports);
-__exportStar(require("./graph/AgentGraph"), exports);
 //# sourceMappingURL=openai.js.map

package/dist/vectorstore/LanceDBVectorStore.d.ts

@@ -18,7 +18,7 @@ export type MetadataFieldType = "string" | "number" | "boolean";
  * Definition for a metadata field that will be stored as a separate column.
  */
 export interface MetadataFieldDefinition {
-    /** Name of the metadata field */
+    /** Name of the metadata field. Use snake_case (e.g. `tenant_id`) to avoid SQL filter issues. */
     name: string;
     /** Data type for the field */
     type: MetadataFieldType;
@@ -42,73 +42,84 @@ export interface LanceDBVectorStoreConfig {
     /** Additional connection options */
     connectionOptions?: Partial<ConnectionOptions>;
     /**
-     * Metadata field definitions for filterable columns.
-     * When specified, metadata fields are stored as separate columns enabling efficient filtering.
-     * If not specified, metadata is stored as a JSON string (legacy behavior).
+     * User-defined metadata field definitions.
+     *
+     * When provided, these fields are stored as typed Arrow columns and are
+     * filterable via SQL predicates in `search()`. The table is created on
+     * first insert using an explicit Arrow schema built from these definitions.
+     *
+     * **Important:** Use `snake_case` for field names (e.g. `tenant_id`, not
+     * `tenantId`). LanceDB uses DataFusion for SQL filtering, which normalizes
+     * unquoted identifiers to lowercase. Mixed-case names like `tenantId` will
+     * fail to match the column `tenantId` because the filter resolves to
+     * `tenantid`.
+     *
+     * Chunk metadata fields (index, hash, prev_id, etc.) are handled
+     * automatically via a `chunk_metadata` struct column — they do not need
+     * to be listed here.
+     *
+     * When omitted, the store connects to a **pre-existing** table (created
+     * independently, e.g. via the LanceDB CLI or another tool). In that case
+     * the schema is not managed by this class and all non-system columns are
+     * returned as metadata on read.
      */
     metadataFields?: MetadataFieldDefinition[];
 }
 /**
  * LanceDB implementation of the VectorStore interface.
  *
- * @example Basic usage with JSON metadata (legacy)
- * ```typescript
- * import { LanceDBVectorStore, OpenAIEmbeddings } from "@agentionai/agents";
+ * Supports two modes of operation:
  *
- * // Create with OpenAI embeddings
- * const embeddings = new OpenAIEmbeddings({
- *   model: "text-embedding-3-small",
- * });
- *
- * const store = await LanceDBVectorStore.create({
- *   name: "knowledge_base",
- *   uri: "./my-database",
- *   tableName: "documents",
- *   embeddings,
- * });
+ * **Managed mode** (`metadataFields` provided): The store creates the LanceDB
+ * table on first insert using an explicit Arrow schema derived from
+ * `metadataFields`. User-defined fields are stored as typed top-level columns.
+ * Chunk metadata (from chunkers) is automatically packed into a `chunk_metadata`
+ * struct column.
  *
- * // Add documents (embeddings generated automatically)
- * await store.addDocuments([
- *   { id: "1", content: "LanceDB is a vector database" },
- *   { id: "2", content: "Vector search enables semantic queries" },
- * ]);
+ * **Pre-existing table mode** (`metadataFields` omitted): The store connects
+ * to a table that was created independently (e.g. via LanceDB CLI or another
+ * tool). No schema management is performed; all non-system columns are returned
+ * as metadata on read.
  *
- * // Search
- * const results = await store.search("What is LanceDB?", { limit: 5 });
+ * @example Managed mode — user-defined metadata fields
+ * ```typescript
+ * import { LanceDBVectorStore } from "@agentionai/agents";
+ * import { OpenAIEmbeddings } from "@agentionai/agents/embeddings";
  *
- * // Create a tool for agents
- * const searchTool = store.toRetrievalTool("Search the knowledge base");
- * ```
+ * const embeddings = new OpenAIEmbeddings({ model: "text-embedding-3-small" });
  *
- * @example With filterable metadata fields
- * ```typescript
  * const store = await LanceDBVectorStore.create({
  *   name: "knowledge_base",
  *   uri: "./my-database",
- *   tableName: "documents",
+ *   tableName: "chunks",
  *   embeddings,
  *   metadataFields: [
- *     { name: "category", type: "string" },
- *     { name: "source", type: "string" },
- *     { name: "year", type: "number" },
- *     { name: "verified", type: "boolean" },
- *     { name: "hash", type: "string" }, // Enables efficient deduplication
+ *     { name: "author", type: "string", nullable: true },
+ *     { name: "category", type: "string", nullable: true },
  *   ],
  * });
  *
- * // Add documents with metadata
+ * // Chunk metadata (index, hash, prev_id, etc.) is stored automatically
+ * // in a chunk_metadata struct column — no need to declare it.
  * await store.addDocuments([
- *   {
- *     id: "1",
- *     content: "LanceDB is a vector database",
- *     metadata: { category: "database", source: "docs", year: 2024, verified: true },
- *   },
+ *   { id: "1", content: "LanceDB is a vector database", metadata: { category: "db" } },
  * ]);
  *
- * // Search with filters on metadata columns
+ * // Search with filters on user metadata columns
  * const results = await store.search("vector database", {
  *   limit: 5,
- *   filter: { category: "database", year: 2024 },
+ *   filter: { category: "db" },
+ * });
+ * ```
+ *
+ * @example Pre-existing table mode — connect to externally managed table
+ * ```typescript
+ * const store = await LanceDBVectorStore.create({
+ *   name: "my_store",
+ *   uri: "./my-database",
+ *   tableName: "existing_table", // table already exists with its own schema
+ *   embeddings,
+ *   // metadataFields omitted — schema is not managed by this class
  * });
  * ```
  */
@@ -124,14 +135,25 @@ export declare class LanceDBVectorStore extends VectorStore {
     /**
      * Create a new LanceDBVectorStore instance.
      *
-     * This is an async factory method since LanceDB connection is asynchronous.
+     * - If the table already exists it is opened immediately.
+     * - If `metadataFields` is provided and the table does not exist yet, it
+     *   will be created on the first insert with an explicit Arrow schema.
+     * - If `metadataFields` is **not** provided and the table does not exist,
+     *   an error is thrown — the store cannot manage an unknown schema.
      *
      * @param config - Configuration for the store
      * @returns A configured LanceDBVectorStore instance
      *
      * @throws Error if @lancedb/lancedb is not installed
+     * @throws Error if the table does not exist and no metadataFields are provided
      */
     static create(config: LanceDBVectorStoreConfig): Promise<LanceDBVectorStore>;
+    /**
+     * Create the table with an explicit Arrow schema derived from `metadataFields`
+     * plus a `chunk_metadata` struct column.
+     * Called on the first insert when operating in managed mode.
+     */
+    private createManagedTable;
     /**
      * Add documents to the vector store.
      * If an embeddings provider is configured, embeddings are generated automatically.
@@ -139,8 +161,29 @@ export declare class LanceDBVectorStore extends VectorStore {
     addDocuments(documents: Document[], _options?: AddDocumentsOptions): Promise<string[]>;
     /**
      * Add documents with pre-computed embeddings.
+     *
+     * In managed mode, chunk metadata fields are packed into a `chunk_metadata`
+     * struct and user-defined fields are projected to their declared columns.
+     * The table is created on the first call; subsequent calls append directly.
+     *
+     * In pre-existing table mode all metadata is spread flat as-is.
      */
     addEmbeddedDocuments(documents: EmbeddedDocument[], _options?: AddDocumentsOptions): Promise<string[]>;
+    /**
+     * Pack chunk metadata fields from flat metadata into a struct object.
+     * Returns a plain object for the `chunk_metadata` column, or null if
+     * no chunk metadata fields are present.
+     */
+    private packChunkMetadata;
+    /**
+     * Unpack a chunk_metadata struct value back to flat metadata keys.
+     */
+    private unpackChunkMetadata;
+    /**
+     * Project a record to only the columns declared in the schema
+     * (id, text, vector, chunk_metadata, plus all metadataFields).
+     */
+    private projectToSchema;
     /**
      * Search for documents similar to the query.
      */
@@ -165,8 +208,9 @@ export declare class LanceDBVectorStore extends VectorStore {
      * Get existing documents by their content hashes.
      * Used for deduplication during ingestion.
      *
-     * Note: If using metadataFields, include a "hash" field of type "string"
-     * for efficient hash lookups. Otherwise, falls back to LIKE queries on JSON metadata.
+     * Requires that documents were stored with chunk metadata containing
+     * a `hash` field (automatically present when using chunkers from this library).
+     * Queries the `chunk_metadata.hash` struct sub-field.
      */
     getByHashes(hashes: string[], _options?: DeleteOptions): Promise<Map<string, string>>;
     /**
@@ -174,9 +218,9 @@ export declare class LanceDBVectorStore extends VectorStore {
      */
     getConnection(): Connection;
     /**
-     * Get the underlying LanceDB table.
+     * Get the underlying LanceDB table, or null if no data has been inserted yet.
      */
-    getTable(): Table;
+    getTable(): Table | null;
     /**
      * Get the configured embeddings provider.
      */
@@ -194,6 +238,10 @@ export declare class LanceDBVectorStore extends VectorStore {
      * Optimize the table for better performance.
      */
     optimize(): Promise<void>;
+    /**
+     * Get the configured metadata fields.
+     */
+    getMetadataFields(): MetadataFieldDefinition[] | undefined;
     /**
      * Build a SQL filter string from a filter object.
      */
@@ -203,12 +251,12 @@ export declare class LanceDBVectorStore extends VectorStore {
      */
     private processResults;
     /**
-     * Extract metadata from a row based on metadataFields configuration.
+     * Extract metadata from a row.
+     *
+     * In managed mode: returns user-defined fields plus unpacked chunk_metadata.
+     * In pre-existing table mode: returns all non-system columns, with
+     * chunk_metadata unpacked if present.
      */
     private extractMetadata;
-    /**
-     * Get the configured metadata fields.
-     */
-    getMetadataFields(): MetadataFieldDefinition[] | undefined;
 }
 //# sourceMappingURL=LanceDBVectorStore.d.ts.map

package/dist/vectorstore/LanceDBVectorStore.js

@@ -44,67 +44,71 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LanceDBVectorStore = void 0;
 const VectorStore_1 = require("./VectorStore");
+/**
+ * All known ChunkMetadata field names.
+ * Used to separate chunk metadata from user metadata when packing/unpacking.
+ */
+const CHUNK_METADATA_KEYS = [
+    "index", "total", "prev_id", "next_id",
+    "start", "end", "source_id", "source_path",
+    "char_count", "token_count", "hash", "section", "page",
+];
+const CHUNK_METADATA_KEY_SET = new Set(CHUNK_METADATA_KEYS);
 /**
  * LanceDB implementation of the VectorStore interface.
  *
- * @example Basic usage with JSON metadata (legacy)
- * ```typescript
- * import { LanceDBVectorStore, OpenAIEmbeddings } from "@agentionai/agents";
+ * Supports two modes of operation:
  *
- * // Create with OpenAI embeddings
- * const embeddings = new OpenAIEmbeddings({
- *   model: "text-embedding-3-small",
- * });
+ * **Managed mode** (`metadataFields` provided): The store creates the LanceDB
+ * table on first insert using an explicit Arrow schema derived from
+ * `metadataFields`. User-defined fields are stored as typed top-level columns.
+ * Chunk metadata (from chunkers) is automatically packed into a `chunk_metadata`
+ * struct column.
  *
- * const store = await LanceDBVectorStore.create({
- *   name: "knowledge_base",
- *   uri: "./my-database",
- *   tableName: "documents",
- *   embeddings,
- * });
+ * **Pre-existing table mode** (`metadataFields` omitted): The store connects
+ * to a table that was created independently (e.g. via LanceDB CLI or another
+ * tool). No schema management is performed; all non-system columns are returned
+ * as metadata on read.
  *
- * // Add documents (embeddings generated automatically)
- * await store.addDocuments([
- *   { id: "1", content: "LanceDB is a vector database" },
- *   { id: "2", content: "Vector search enables semantic queries" },
- * ]);
- *
- * // Search
- * const results = await store.search("What is LanceDB?", { limit: 5 });
+ * @example Managed mode — user-defined metadata fields
+ * ```typescript
+ * import { LanceDBVectorStore } from "@agentionai/agents";
+ * import { OpenAIEmbeddings } from "@agentionai/agents/embeddings";
  *
- * // Create a tool for agents
- * const searchTool = store.toRetrievalTool("Search the knowledge base");
- * ```
+ * const embeddings = new OpenAIEmbeddings({ model: "text-embedding-3-small" });
  *
- * @example With filterable metadata fields
- * ```typescript
  * const store = await LanceDBVectorStore.create({
  *   name: "knowledge_base",
  *   uri: "./my-database",
- *   tableName: "documents",
+ *   tableName: "chunks",
  *   embeddings,
  *   metadataFields: [
- *     { name: "category", type: "string" },
- *     { name: "source", type: "string" },
- *     { name: "year", type: "number" },
- *     { name: "verified", type: "boolean" },
- *     { name: "hash", type: "string" }, // Enables efficient deduplication
+ *     { name: "author", type: "string", nullable: true },
+ *     { name: "category", type: "string", nullable: true },
  *   ],
  * });
  *
- * // Add documents with metadata
+ * // Chunk metadata (index, hash, prev_id, etc.) is stored automatically
+ * // in a chunk_metadata struct column — no need to declare it.
  * await store.addDocuments([
- *   {
- *     id: "1",
- *     content: "LanceDB is a vector database",
- *     metadata: { category: "database", source: "docs", year: 2024, verified: true },
- *   },
+ *   { id: "1", content: "LanceDB is a vector database", metadata: { category: "db" } },
  * ]);
  *
- * // Search with filters on metadata columns
+ * // Search with filters on user metadata columns
  * const results = await store.search("vector database", {
  *   limit: 5,
- *   filter: { category: "database", year: 2024 },
+ *   filter: { category: "db" },
+ * });
+ * ```
+ *
+ * @example Pre-existing table mode — connect to externally managed table
+ * ```typescript
+ * const store = await LanceDBVectorStore.create({
+ *   name: "my_store",
+ *   uri: "./my-database",
+ *   tableName: "existing_table", // table already exists with its own schema
+ *   embeddings,
+ *   // metadataFields omitted — schema is not managed by this class
  * });
  * ```
  */
@@ -123,15 +127,19 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
     /**
      * Create a new LanceDBVectorStore instance.
      *
-     * This is an async factory method since LanceDB connection is asynchronous.
+     * - If the table already exists it is opened immediately.
+     * - If `metadataFields` is provided and the table does not exist yet, it
+     *   will be created on the first insert with an explicit Arrow schema.
+     * - If `metadataFields` is **not** provided and the table does not exist,
+     *   an error is thrown — the store cannot manage an unknown schema.
      *
      * @param config - Configuration for the store
      * @returns A configured LanceDBVectorStore instance
      *
      * @throws Error if @lancedb/lancedb is not installed
+     * @throws Error if the table does not exist and no metadataFields are provided
      */
     static async create(config) {
-        // Dynamic import to make lancedb an optional dependency
         let lancedb;
         try {
             lancedb = await Promise.resolve().then(() => __importStar(require("@lancedb/lancedb")));
@@ -141,55 +149,80 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
         }
         const connection = await lancedb.connect(config.uri, config.connectionOptions);
         const tableNames = await connection.tableNames();
-        let table;
-        const dimensions = config.dimensions ?? config.embeddings?.dimensions ?? 1536;
+        let table = null;
         if (tableNames.includes(config.tableName)) {
             table = await connection.openTable(config.tableName);
         }
-        else {
-            // Create table with schema
-            let arrow;
-            try {
-                arrow = await Promise.resolve().then(() => __importStar(require("apache-arrow")));
+        else if (!config.metadataFields) {
+            throw new Error(`Table "${config.tableName}" does not exist and no metadataFields were provided. ` +
+                `Either create the table independently or provide metadataFields so the store can create it on first insert.`);
+        }
+        // Table doesn't exist but metadataFields provided → will be created on first insert.
+        return new LanceDBVectorStore(config, connection, table);
+    }
+    /**
+     * Create the table with an explicit Arrow schema derived from `metadataFields`
+     * plus a `chunk_metadata` struct column.
+     * Called on the first insert when operating in managed mode.
+     */
+    async createManagedTable(records) {
+        let arrow;
+        try {
+            arrow = await Promise.resolve().then(() => __importStar(require("apache-arrow")));
+        }
+        catch {
+            throw new Error("apache-arrow is not installed. Install it with: npm install apache-arrow");
+        }
+        const schemaFields = [
+            new arrow.Field("id", new arrow.Utf8(), false),
+            new arrow.Field("text", new arrow.Utf8(), false),
+            new arrow.Field("vector", new arrow.FixedSizeList(this.dimensions, new arrow.Field("item", new arrow.Float32(), true)), false),
+        ];
+        // Warn about non-snake_case field names (DataFusion normalizes SQL identifiers to lowercase)
+        for (const fieldDef of this.metadataFields) {
+            if (fieldDef.name !== fieldDef.name.toLowerCase()) {
+                console.warn(`[LanceDBVectorStore] Warning: metadata field "${fieldDef.name}" contains uppercase characters. ` +
+                    `LanceDB uses DataFusion for SQL filtering, which normalizes unquoted identifiers to lowercase. ` +
+                    `Use snake_case names (e.g. "${fieldDef.name.replace(/[A-Z]/g, (c) => "_" + c.toLowerCase()).replace(/^_/, "")}") to avoid filter issues.`);
             }
-            catch {
-                throw new Error("apache-arrow is not installed. Install it with: npm install apache-arrow");
+        }
+        // User-defined metadata columns
+        for (const fieldDef of this.metadataFields) {
+            const nullable = fieldDef.nullable !== false; // default true
+            let arrowType;
+            if (fieldDef.type === "number") {
+                arrowType = new arrow.Float64();
             }
-            // Build schema fields - use explicit type to allow different Field types
-            const schemaFields = [
-                new arrow.Field("id", new arrow.Utf8(), false),
-                new arrow.Field("text", new arrow.Utf8(), false),
-                new arrow.Field("vector", new arrow.FixedSizeList(dimensions, new arrow.Field("item", new arrow.Float32(), true)), false),
-            ];
-            // Add metadata fields - either as separate columns or as a JSON string
-            if (config.metadataFields && config.metadataFields.length > 0) {
-                for (const field of config.metadataFields) {
-                    const nullable = field.nullable !== false;
-                    let arrowType;
-                    switch (field.type) {
-                        case "string":
-                            arrowType = new arrow.Utf8();
-                            break;
-                        case "number":
-                            arrowType = new arrow.Float64();
-                            break;
-                        case "boolean":
-                            arrowType = new arrow.Bool();
-                            break;
-                        default:
-                            throw new Error(`Unsupported metadata field type: ${field.type}`);
-                    }
-                    schemaFields.push(new arrow.Field(field.name, arrowType, nullable));
-                }
+            else if (fieldDef.type === "boolean") {
+                arrowType = new arrow.Bool();
             }
             else {
-                // Legacy: store metadata as JSON string
-                schemaFields.push(new arrow.Field("metadata", new arrow.Utf8(), true));
+                arrowType = new arrow.Utf8();
             }
-            const schema = new arrow.Schema(schemaFields);
-            table = await connection.createEmptyTable(config.tableName, schema);
+            schemaFields.push(new arrow.Field(fieldDef.name, arrowType, nullable));
         }
-        return new LanceDBVectorStore(config, connection, table);
+        // Chunk metadata struct column (always included, nullable for non-chunk docs)
+        schemaFields.push(new arrow.Field("chunk_metadata", new arrow.Struct([
+            new arrow.Field("index", new arrow.Float64(), true),
+            new arrow.Field("total", new arrow.Float64(), true),
+            new arrow.Field("prev_id", new arrow.Utf8(), true),
+            new arrow.Field("next_id", new arrow.Utf8(), true),
+            new arrow.Field("start", new arrow.Float64(), true),
+            new arrow.Field("end", new arrow.Float64(), true),
+            new arrow.Field("source_id", new arrow.Utf8(), true),
+            new arrow.Field("source_path", new arrow.Utf8(), true),
+            new arrow.Field("char_count", new arrow.Float64(), true),
+            new arrow.Field("token_count", new arrow.Float64(), true),
+            new arrow.Field("hash", new arrow.Utf8(), true),
+            new arrow.Field("section", new arrow.Utf8(), true),
+            new arrow.Field("page", new arrow.Float64(), true),
+        ]), true // nullable — non-chunk documents get null
+        ));
+        const schema = new arrow.Schema(schemaFields);
+        this.table = await this.connection.createTable(this.tableName, records, {
+            schema,
+        });
+        return this.table;
     }
     /**
      * Add documents to the vector store.
@@ -199,10 +232,8 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
         if (!this.embeddings) {
             throw new Error("No embeddings provider configured. Use addEmbeddedDocuments() with pre-computed embeddings, or configure an embeddings provider.");
         }
-        // Generate embeddings for all documents
         const texts = documents.map((doc) => doc.content);
         const vectors = await this.embeddings.embed(texts);
-        // Convert to embedded documents
         const embeddedDocs = documents.map((doc, i) => ({
             ...doc,
             embedding: vectors[i],
@@ -211,6 +242,12 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
     }
     /**
      * Add documents with pre-computed embeddings.
+     *
+     * In managed mode, chunk metadata fields are packed into a `chunk_metadata`
+     * struct and user-defined fields are projected to their declared columns.
+     * The table is created on the first call; subsequent calls append directly.
+     *
+     * In pre-existing table mode all metadata is spread flat as-is.
      */
     async addEmbeddedDocuments(documents, _options) {
         const records = documents.map((doc) => {
@@ -218,23 +255,69 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
                 id: doc.id,
                 text: doc.content,
                 vector: doc.embedding,
+                ...doc.metadata,
             };
-            if (this.metadataFields && this.metadataFields.length > 0) {
-                // Store each metadata field as a separate column
-                for (const field of this.metadataFields) {
-                    const value = doc.metadata?.[field.name];
-                    record[field.name] = value !== undefined ? value : null;
-                }
-            }
-            else {
-                // Legacy: store metadata as JSON string
-                record.metadata = doc.metadata ? JSON.stringify(doc.metadata) : undefined;
+            // In managed mode, pack chunk metadata into struct and project to schema
+            if (this.metadataFields) {
+                const packed = this.packChunkMetadata(doc.metadata ?? {});
+                record.chunk_metadata = packed;
+                return this.projectToSchema(record);
             }
             return record;
         });
-        await this.table.add(records);
+        if (this.table) {
+            await this.table.add(records);
+        }
+        else {
+            // Managed mode: metadataFields must be present (enforced in create())
+            await this.createManagedTable(records);
+        }
         return documents.map((d) => d.id);
     }
+    /**
+     * Pack chunk metadata fields from flat metadata into a struct object.
+     * Returns a plain object for the `chunk_metadata` column, or null if
+     * no chunk metadata fields are present.
+     */
+    packChunkMetadata(metadata) {
+        const struct = {};
+        let found = false;
+        for (const key of CHUNK_METADATA_KEYS) {
+            if (key in metadata) {
+                struct[key] = metadata[key] ?? null;
+                found = true;
+            }
+        }
+        return found ? struct : null;
+    }
+    /**
+     * Unpack a chunk_metadata struct value back to flat metadata keys.
+     */
+    unpackChunkMetadata(struct, target) {
+        for (const key of CHUNK_METADATA_KEYS) {
+            const value = struct[key];
+            if (value !== null && value !== undefined) {
+                target[key] = value;
+            }
+        }
+    }
+    /**
+     * Project a record to only the columns declared in the schema
+     * (id, text, vector, chunk_metadata, plus all metadataFields).
+     */
+    projectToSchema(record) {
+        const projected = { id: record.id, text: record.text };
+        if (record.vector !== undefined) {
+            projected.vector = record.vector;
+        }
+        // User-defined metadata fields
+        for (const f of this.metadataFields) {
+            projected[f.name] = record[f.name] ?? null;
+        }
+        // Chunk metadata struct
+        projected.chunk_metadata = record.chunk_metadata ?? null;
+        return projected;
+    }
     /**
      * Search for documents similar to the query.
      */
@@ -249,6 +332,9 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
      * Search using a pre-computed embedding vector.
      */
     async searchByVector(embedding, options) {
+        if (!this.table) {
+            return [];
+        }
         const limit = options?.limit ?? 10;
         const scoreThreshold = options?.scoreThreshold;
         let queryBuilder = this.table.vectorSearch(embedding).limit(limit);
@@ -265,6 +351,8 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
      * Delete documents by their IDs.
      */
     async delete(ids, _options) {
+        if (!this.table)
+            return 0;
         const idList = ids.map((id) => `'${id}'`).join(", ");
         const filter = `id IN (${idList})`;
         const countBefore = await this.table.countRows();
@@ -276,12 +364,16 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
      * Delete all documents.
      */
     async clear(_options) {
+        if (!this.table)
+            return;
         await this.table.delete("id IS NOT NULL");
     }
     /**
      * Get a document by its ID.
      */
     async getById(id, _options) {
+        if (!this.table)
+            return null;
         const results = await this.table
             .query()
             .where(`id = '${id}'`)
@@ -301,34 +393,21 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
      * Get existing documents by their content hashes.
      * Used for deduplication during ingestion.
      *
-     * Note: If using metadataFields, include a "hash" field of type "string"
-     * for efficient hash lookups. Otherwise, falls back to LIKE queries on JSON metadata.
+     * Requires that documents were stored with chunk metadata containing
+     * a `hash` field (automatically present when using chunkers from this library).
+     * Queries the `chunk_metadata.hash` struct sub-field.
      */
     async getByHashes(hashes, _options) {
         const hashMap = new Map();
-        if (hashes.length === 0) {
+        if (hashes.length === 0 || !this.table) {
             return hashMap;
         }
-        // Check if hash is a defined metadata field for efficient queries
-        const hasHashField = this.metadataFields?.some((field) => field.name === "hash");
         for (const hash of hashes) {
-            let results;
-            if (hasHashField) {
-                // Efficient direct column query
-                results = await this.table
-                    .query()
-                    .where(`hash = '${hash}'`)
-                    .limit(1)
-                    .toArray();
-            }
-            else {
-                // Legacy: search for hash string in JSON metadata
-                results = await this.table
-                    .query()
-                    .where(`metadata LIKE '%${hash}%'`)
-                    .limit(1)
-                    .toArray();
-            }
+            const results = await this.table
+                .query()
+                .where(`chunk_metadata.hash = '${hash}'`)
+                .limit(1)
+                .toArray();
             if (results.length > 0) {
                 const record = results[0];
                 hashMap.set(hash, record.id);
@@ -343,7 +422,7 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
         return this.connection;
     }
     /**
-     * Get the underlying LanceDB table.
+     * Get the underlying LanceDB table, or null if no data has been inserted yet.
      */
     getTable() {
         return this.table;
@@ -365,14 +444,24 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
      * Recommended for tables with more than 10,000 rows.
      */
     async createIndex() {
+        if (!this.table)
+            throw new Error("Table not yet created — insert data first.");
         await this.table.createIndex("vector");
     }
     /**
      * Optimize the table for better performance.
      */
     async optimize() {
+        if (!this.table)
+            return;
         await this.table.optimize();
     }
+    /**
+     * Get the configured metadata fields.
+     */
+    getMetadataFields() {
+        return this.metadataFields;
+    }
     /**
      * Build a SQL filter string from a filter object.
      */
@@ -397,19 +486,16 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
     processResults(results, scoreThreshold) {
         const searchResults = [];
         for (const row of results) {
-            // LanceDB returns _distance for vector search
             const distance = row._distance ?? 0;
-            // Convert distance to similarity score (lower distance = higher similarity)
             const score = 1 / (1 + distance);
             if (scoreThreshold !== undefined && score < scoreThreshold) {
                 continue;
             }
-            const metadata = this.extractMetadata(row);
             searchResults.push({
                 document: {
                     id: row.id,
                     content: row.text,
-                    metadata,
+                    metadata: this.extractMetadata(row),
                 },
                 score,
             });
@@ -417,13 +503,18 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
         return searchResults;
     }
     /**
-     * Extract metadata from a row based on metadataFields configuration.
+     * Extract metadata from a row.
+     *
+     * In managed mode: returns user-defined fields plus unpacked chunk_metadata.
+     * In pre-existing table mode: returns all non-system columns, with
+     * chunk_metadata unpacked if present.
      */
     extractMetadata(row) {
+        const SYSTEM_COLS = new Set(["id", "text", "vector", "_distance", "chunk_metadata"]);
+        const metadata = {};
+        let hasValue = false;
         if (this.metadataFields && this.metadataFields.length > 0) {
-            // Reconstruct metadata from separate columns
-            const metadata = {};
-            let hasValue = false;
+            // Managed mode: collect declared user fields
             for (const field of this.metadataFields) {
                 const value = row[field.name];
                 if (value !== null && value !== undefined) {
@@ -431,18 +522,23 @@ class LanceDBVectorStore extends VectorStore_1.VectorStore {
                     hasValue = true;
                 }
             }
-            return hasValue ? metadata : undefined;
         }
         else {
-            // Legacy: parse metadata from JSON string
-            return row.metadata ? JSON.parse(row.metadata) : undefined;
+            // Pre-existing table mode: return all non-system columns
+            for (const [key, value] of Object.entries(row)) {
+                if (!SYSTEM_COLS.has(key) && value !== null && value !== undefined) {
+                    metadata[key] = value;
+                    hasValue = true;
+                }
+            }
         }
-    }
-    /**
-     * Get the configured metadata fields.
-     */
-    getMetadataFields() {
-        return this.metadataFields;
+        // Unpack chunk_metadata struct if present
+        const chunkStruct = row.chunk_metadata;
+        if (chunkStruct && typeof chunkStruct === "object") {
+            this.unpackChunkMetadata(chunkStruct, metadata);
+            hasValue = true;
+        }
+        return hasValue ? metadata : undefined;
     }
 }
 exports.LanceDBVectorStore = LanceDBVectorStore;

package/dist/vectorstore/VectorStore.d.ts

@@ -235,7 +235,7 @@ export declare abstract class VectorStore {
     }>;
     /**
      * Create a tool that agents can use to retrieve a chunk by its ID.
-     * Useful for navigating chunk chains using previousChunkId/nextChunkId metadata.
+     * Useful for navigating chunk chains using prev_id/next_id metadata.
      *
      * @param description - Description of what the tool does (e.g., "Get a specific chunk by ID to read adjacent context")
      * @param options - Configuration options for the tool
@@ -245,7 +245,7 @@ export declare abstract class VectorStore {
      * ```typescript
      * const store = new LanceDBVectorStore({ ... });
      * const tool = store.toGetChunkByIdTool(
-     *   "Retrieve a specific chunk by ID. Use previousChunkId or nextChunkId from search results to get surrounding context."
+     *   "Retrieve a specific chunk by ID. Use prev_id or next_id from search results to get surrounding context."
      * );
      * agent.addTools([tool]);
      * ```

package/dist/vectorstore/VectorStore.js

@@ -165,7 +165,7 @@ class VectorStore {
     }
     /**
      * Create a tool that agents can use to retrieve a chunk by its ID.
-     * Useful for navigating chunk chains using previousChunkId/nextChunkId metadata.
+     * Useful for navigating chunk chains using prev_id/next_id metadata.
      *
      * @param description - Description of what the tool does (e.g., "Get a specific chunk by ID to read adjacent context")
      * @param options - Configuration options for the tool
@@ -175,7 +175,7 @@ class VectorStore {
      * ```typescript
      * const store = new LanceDBVectorStore({ ... });
      * const tool = store.toGetChunkByIdTool(
-     *   "Retrieve a specific chunk by ID. Use previousChunkId or nextChunkId from search results to get surrounding context."
+     *   "Retrieve a specific chunk by ID. Use prev_id or next_id from search results to get surrounding context."
      * );
      * agent.addTools([tool]);
      * ```
@@ -187,7 +187,7 @@ class VectorStore {
             properties: {
                 id: {
                     type: "string",
-                    description: "The chunk ID to retrieve (e.g., from previousChunkId or nextChunkId metadata)",
+                    description: "The chunk ID to retrieve (e.g., from prev_id or next_id metadata)",
                 },
             },
             required: ["id"],

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@agentionai/agents",
   "author": "Laurent Zuijdwijk",
-  "version": "0.7.0",
+  "version": "0.8.1-beta",
   "description": "Agent Library",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -33,6 +33,26 @@
     "./embeddings": {
       "types": "./dist/embeddings/index.d.ts",
       "default": "./dist/embeddings/index.js"
+    },
+    "./vectorstore": {
+      "types": "./dist/vectorstore/index.d.ts",
+      "default": "./dist/vectorstore/index.js"
+    },
+    "./mcp": {
+      "types": "./dist/mcp/index.d.ts",
+      "default": "./dist/mcp/index.js"
+    },
+    "./viz": {
+      "types": "./dist/viz/index.d.ts",
+      "default": "./dist/viz/index.js"
+    },
+    "./chunkers": {
+      "types": "./dist/chunkers/index.d.ts",
+      "default": "./dist/chunkers/index.js"
+    },
+    "./ingestion": {
+      "types": "./dist/ingestion/index.d.ts",
+      "default": "./dist/ingestion/index.js"
     }
   },
   "files": [
@@ -54,8 +74,8 @@
     "lint:fix": "eslint 'src/**/*.{js,ts}' --fix",
     "format": "prettier --write 'src/**/*.{js,ts,json,md}'",
     "prepare": "npm run build",
-    "example": "ts-node examples/index.ts",
-    "example:watch": "nodemon --watch examples --watch src --ext ts --exec 'ts-node' examples/index.ts",
+    "example": "tsx",
+    "example:watch": "nodemon --watch examples --watch src --ext ts --exec 'ts-node --project tsconfig.esm.json' examples/index.ts",
     "docs": "npm run docs:api && npm run docs:site",
     "docs:api": "typedoc",
     "docs:site": "vitepress build docs",