@agentionai/agents 0.3.0-beta

package/dist/chunkers/RecursiveChunker.js

@@ -0,0 +1,166 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecursiveChunker = void 0;
+const Chunker_1 = require("./Chunker");
+/**
+ * Recursive text chunker that tries to split on semantic boundaries.
+ * It attempts to split by larger separators first (paragraphs), then
+ * falls back to smaller ones (sentences, words) to keep semantic units together.
+ *
+ * @example
+ * ```typescript
+ * const chunker = new RecursiveChunker({
+ *   chunkSize: 1000,
+ *   chunkOverlap: 100,
+ *   separators: ["\n\n", "\n", ". ", " "],
+ * });
+ *
+ * const chunks = await chunker.chunk(document);
+ * ```
+ */
+class RecursiveChunker extends Chunker_1.Chunker {
+    constructor(config) {
+        super(config);
+        this.name = "RecursiveChunker";
+        this.separators = config.separators ?? ["\n\n", "\n", ". ", " "];
+    }
+    /**
+     * Split text recursively using the separator hierarchy.
+     */
+    splitText(text) {
+        return this.recursiveSplit(text, 0);
+    }
+    /**
+     * Recursively split text using separators at the given index.
+     */
+    recursiveSplit(text, separatorIndex) {
+        const { chunkSize, chunkOverlap = 0 } = this.config;
+        // Base case: text fits in one chunk
+        if (text.length <= chunkSize) {
+            return text.trim() ? [text] : [];
+        }
+        // No more separators: force split by character
+        if (separatorIndex >= this.separators.length) {
+            return this.forceSplit(text);
+        }
+        const separator = this.separators[separatorIndex];
+        const parts = this.splitBySeparator(text, separator);
+        // If separator didn't help, try the next one
+        if (parts.length <= 1) {
+            return this.recursiveSplit(text, separatorIndex + 1);
+        }
+        // Merge parts into chunks respecting size limit
+        const chunks = [];
+        let currentChunk = "";
+        for (const part of parts) {
+            const partWithSep = currentChunk ? separator + part : part;
+            const wouldBeLength = currentChunk.length + partWithSep.length;
+            if (wouldBeLength <= chunkSize) {
+                // Part fits in current chunk
+                currentChunk = currentChunk ? currentChunk + separator + part : part;
+            }
+            else {
+                // Save current chunk if it has content
+                if (currentChunk.trim()) {
+                    chunks.push(currentChunk);
+                }
+                // Check if part itself is too big
+                if (part.length > chunkSize) {
+                    // Recursively split the oversized part
+                    const subChunks = this.recursiveSplit(part, separatorIndex + 1);
+                    chunks.push(...subChunks);
+                    currentChunk = "";
+                }
+                else {
+                    currentChunk = part;
+                }
+            }
+        }
+        // Don't forget the last chunk
+        if (currentChunk.trim()) {
+            chunks.push(currentChunk);
+        }
+        // Apply overlap if configured
+        if (chunkOverlap > 0 && chunks.length > 1) {
+            return this.applyOverlap(chunks, separator);
+        }
+        return chunks;
+    }
+    /**
+     * Split text by separator, keeping the parts clean.
+     */
+    splitBySeparator(text, separator) {
+        if (separator === ". ") {
+            // Special handling for sentence boundaries - keep the period
+            return text.split(/(?<=\.)\s+/).filter((p) => p.trim());
+        }
+        return text.split(separator).filter((p) => p.trim());
+    }
+    /**
+     * Force split text by character count when no separator works.
+     */
+    forceSplit(text) {
+        const { chunkSize, chunkOverlap = 0 } = this.config;
+        const chunks = [];
+        const step = chunkSize - chunkOverlap;
+        let start = 0;
+        while (start < text.length) {
+            const end = Math.min(start + chunkSize, text.length);
+            const chunk = text.slice(start, end);
+            if (chunk.trim()) {
+                chunks.push(chunk);
+            }
+            if (end >= text.length)
+                break;
+            start += step;
+        }
+        return chunks;
+    }
+    /**
+     * Apply overlap between chunks by prepending context from previous chunk.
+     */
+    applyOverlap(chunks, separator) {
+        const { chunkOverlap = 0 } = this.config;
+        if (chunkOverlap === 0 || chunks.length <= 1) {
+            return chunks;
+        }
+        const result = [chunks[0]];
+        for (let i = 1; i < chunks.length; i++) {
+            const prevChunk = chunks[i - 1];
+            const currentChunk = chunks[i];
+            // Get overlap from end of previous chunk
+            const overlapText = this.getOverlapText(prevChunk, chunkOverlap, separator);
+            if (overlapText) {
+                result.push(overlapText + separator + currentChunk);
+            }
+            else {
+                result.push(currentChunk);
+            }
+        }
+        return result;
+    }
+    /**
+     * Extract overlap text from the end of a chunk, trying to break at separator.
+     */
+    getOverlapText(text, overlapSize, separator) {
+        if (text.length <= overlapSize) {
+            return text;
+        }
+        // Try to find a clean break point near the overlap size
+        const overlapStart = text.length - overlapSize;
+        const sepIndex = text.indexOf(separator, overlapStart);
+        if (sepIndex !== -1 && sepIndex < text.length - 1) {
+            return text.slice(sepIndex + separator.length);
+        }
+        // Fall back to exact character overlap
+        return text.slice(overlapStart);
+    }
+    /**
+     * Get the configured separators.
+     */
+    getSeparators() {
+        return [...this.separators];
+    }
+}
+exports.RecursiveChunker = RecursiveChunker;
+//# sourceMappingURL=RecursiveChunker.js.map

package/dist/chunkers/TextChunker.d.ts

@@ -0,0 +1,27 @@
+import { Chunker } from "./Chunker";
+import { ChunkerConfig } from "./types";
+/**
+ * Simple text chunker that splits by character count with optional overlap.
+ *
+ * @example
+ * ```typescript
+ * const chunker = new TextChunker({
+ *   chunkSize: 1000,
+ *   chunkOverlap: 200,
+ * });
+ *
+ * const chunks = await chunker.chunk(longText, {
+ *   sourceId: 'doc-123',
+ *   sourcePath: '/docs/readme.md',
+ * });
+ * ```
+ */
+export declare class TextChunker extends Chunker {
+    readonly name = "TextChunker";
+    constructor(config: ChunkerConfig);
+    /**
+     * Split text by character count with overlap.
+     */
+    protected splitText(text: string): string[];
+}
+//# sourceMappingURL=TextChunker.d.ts.map

package/dist/chunkers/TextChunker.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TextChunker = void 0;
+const Chunker_1 = require("./Chunker");
+/**
+ * Simple text chunker that splits by character count with optional overlap.
+ *
+ * @example
+ * ```typescript
+ * const chunker = new TextChunker({
+ *   chunkSize: 1000,
+ *   chunkOverlap: 200,
+ * });
+ *
+ * const chunks = await chunker.chunk(longText, {
+ *   sourceId: 'doc-123',
+ *   sourcePath: '/docs/readme.md',
+ * });
+ * ```
+ */
+class TextChunker extends Chunker_1.Chunker {
+    constructor(config) {
+        super(config);
+        this.name = "TextChunker";
+    }
+    /**
+     * Split text by character count with overlap.
+     */
+    splitText(text) {
+        const { chunkSize, chunkOverlap = 0 } = this.config;
+        const chunks = [];
+        if (text.length <= chunkSize) {
+            return [text];
+        }
+        const step = chunkSize - chunkOverlap;
+        let start = 0;
+        while (start < text.length) {
+            const end = Math.min(start + chunkSize, text.length);
+            chunks.push(text.slice(start, end));
+            // If we've reached the end, stop
+            if (end >= text.length) {
+                break;
+            }
+            start += step;
+        }
+        return chunks;
+    }
+}
+exports.TextChunker = TextChunker;
+//# sourceMappingURL=TextChunker.js.map

package/dist/chunkers/TokenChunker.d.ts

@@ -0,0 +1,60 @@
+import { Chunker } from "./Chunker";
+import { Chunk, TokenChunkerConfig } from "./types";
+/**
+ * Load tokenx module using dynamic import.
+ * This function can be mocked in tests.
+ * @internal
+ */
+export declare function loadTokenx(): Promise<typeof import("tokenx")>;
+/**
+ * Reset the tokenx module cache. Used in tests.
+ * @internal
+ */
+export declare function resetTokenxCache(): void;
+/**
+ * Token-aware text chunker using the tokenx library.
+ * Splits text based on token count rather than character count,
+ * ensuring chunks fit within LLM token limits.
+ *
+ * Uses tokenx for fast token estimation (~96% accuracy, ~2kB).
+ *
+ * @example
+ * ```typescript
+ * const chunker = new TokenChunker({
+ *   chunkSize: 500,       // 500 tokens per chunk
+ *   chunkOverlap: 50,     // 50 token overlap
+ * });
+ *
+ * const chunks = await chunker.chunk(longDocument);
+ * // Each chunk.metadata.tokenCount contains estimated tokens
+ * ```
+ */
+export declare class TokenChunker extends Chunker {
+    readonly name = "TokenChunker";
+    constructor(config: TokenChunkerConfig);
+    /**
+     * Protected method to get tokenx - can be overridden in tests
+     */
+    protected getTokenx(): Promise<typeof import("tokenx")>;
+    /**
+     * Split text by token count using tokenx.
+     */
+    protected splitText(text: string): Promise<string[]>;
+    /**
+     * Apply token-based overlap between chunks.
+     */
+    private applyTokenOverlap;
+    /**
+     * Get approximately chunkOverlap tokens from the end of text.
+     */
+    private getTokenOverlap;
+    /**
+     * Override chunk to add token count to metadata.
+     */
+    chunk(text: string, options?: import("./types").ChunkOptions): Promise<Chunk[]>;
+    /**
+     * Estimate token count for a given text.
+     */
+    static estimateTokens(text: string): Promise<number>;
+}
+//# sourceMappingURL=TokenChunker.d.ts.map

package/dist/chunkers/TokenChunker.js

@@ -0,0 +1,176 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TokenChunker = void 0;
+exports.loadTokenx = loadTokenx;
+exports.resetTokenxCache = resetTokenxCache;
+const Chunker_1 = require("./Chunker");
+// Cache the tokenx module after first import
+let tokenxModule = null;
+/**
+ * Load tokenx module using dynamic import.
+ * This function can be mocked in tests.
+ * @internal
+ */
+async function loadTokenx() {
+    if (!tokenxModule) {
+        // Use dynamic import for ESM module
+        tokenxModule = await Promise.resolve().then(() => __importStar(require("tokenx")));
+    }
+    return tokenxModule;
+}
+/**
+ * Reset the tokenx module cache. Used in tests.
+ * @internal
+ */
+function resetTokenxCache() {
+    tokenxModule = null;
+}
+/**
+ * Token-aware text chunker using the tokenx library.
+ * Splits text based on token count rather than character count,
+ * ensuring chunks fit within LLM token limits.
+ *
+ * Uses tokenx for fast token estimation (~96% accuracy, ~2kB).
+ *
+ * @example
+ * ```typescript
+ * const chunker = new TokenChunker({
+ *   chunkSize: 500,       // 500 tokens per chunk
+ *   chunkOverlap: 50,     // 50 token overlap
+ * });
+ *
+ * const chunks = await chunker.chunk(longDocument);
+ * // Each chunk.metadata.tokenCount contains estimated tokens
+ * ```
+ */
+class TokenChunker extends Chunker_1.Chunker {
+    constructor(config) {
+        super(config);
+        this.name = "TokenChunker";
+    }
+    /**
+     * Protected method to get tokenx - can be overridden in tests
+     */
+    async getTokenx() {
+        return loadTokenx();
+    }
+    /**
+     * Split text by token count using tokenx.
+     */
+    async splitText(text) {
+        const { chunkSize, chunkOverlap = 0 } = this.config;
+        const tokenx = await this.getTokenx();
+        const { splitByTokens } = tokenx;
+        // Use tokenx's splitByTokens for token-aware splitting
+        const chunks = splitByTokens(text, chunkSize);
+        // Apply overlap if configured
+        if (chunkOverlap > 0 && chunks.length > 1) {
+            return this.applyTokenOverlap(chunks, text);
+        }
+        return chunks;
+    }
+    /**
+     * Apply token-based overlap between chunks.
+     */
+    async applyTokenOverlap(chunks, _originalText) {
+        const { chunkOverlap = 0 } = this.config;
+        const result = [chunks[0]];
+        for (let i = 1; i < chunks.length; i++) {
+            const prevChunk = chunks[i - 1];
+            const currentChunk = chunks[i];
+            // Get overlap from end of previous chunk
+            const overlapText = await this.getTokenOverlap(prevChunk, chunkOverlap);
+            if (overlapText && overlapText.trim()) {
+                result.push(overlapText + " " + currentChunk);
+            }
+            else {
+                result.push(currentChunk);
+            }
+        }
+        return result;
+    }
+    /**
+     * Get approximately chunkOverlap tokens from the end of text.
+     */
+    async getTokenOverlap(text, overlapTokens) {
+        const tokenx = await this.getTokenx();
+        const { estimateTokenCount } = tokenx;
+        // Estimate characters per token (roughly 4 chars per token for English)
+        const estimatedChars = overlapTokens * 4;
+        if (text.length <= estimatedChars) {
+            return text;
+        }
+        // Start from estimated position and find a word boundary
+        let start = text.length - estimatedChars;
+        // Find the next space to start at a word boundary
+        const spaceIndex = text.indexOf(" ", start);
+        if (spaceIndex !== -1 && spaceIndex < text.length - 1) {
+            start = spaceIndex + 1;
+        }
+        const overlap = text.slice(start);
+        // Verify we're close to the target token count
+        const actualTokens = estimateTokenCount(overlap);
+        if (actualTokens > overlapTokens * 1.5) {
+            // Too many tokens, trim more aggressively
+            const words = overlap.split(/\s+/);
+            const targetWords = Math.ceil(words.length * (overlapTokens / actualTokens));
+            return words.slice(-targetWords).join(" ");
+        }
+        return overlap;
+    }
+    /**
+     * Override chunk to add token count to metadata.
+     */
+    async chunk(text, options) {
+        const chunks = await super.chunk(text, options);
+        const tokenx = await this.getTokenx();
+        const { estimateTokenCount } = tokenx;
+        // Add token count to each chunk's metadata
+        for (const chunk of chunks) {
+            chunk.metadata.tokenCount = estimateTokenCount(chunk.content);
+        }
+        return chunks;
+    }
+    /**
+     * Estimate token count for a given text.
+     */
+    static async estimateTokens(text) {
+        const tokenx = await loadTokenx();
+        return tokenx.estimateTokenCount(text);
+    }
+}
+exports.TokenChunker = TokenChunker;
+//# sourceMappingURL=TokenChunker.js.map

package/dist/chunkers/index.d.ts

@@ -0,0 +1,6 @@
+export { Chunk, ChunkMetadata, ChunkerConfig, ChunkOptions, RecursiveChunkerConfig, TokenChunkerConfig, } from "./types";
+export { Chunker } from "./Chunker";
+export { TextChunker } from "./TextChunker";
+export { RecursiveChunker } from "./RecursiveChunker";
+export { TokenChunker } from "./TokenChunker";
+//# sourceMappingURL=index.d.ts.map

package/dist/chunkers/index.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TokenChunker = exports.RecursiveChunker = exports.TextChunker = exports.Chunker = void 0;
+// Base class
+var Chunker_1 = require("./Chunker");
+Object.defineProperty(exports, "Chunker", { enumerable: true, get: function () { return Chunker_1.Chunker; } });
+// Implementations
+var TextChunker_1 = require("./TextChunker");
+Object.defineProperty(exports, "TextChunker", { enumerable: true, get: function () { return TextChunker_1.TextChunker; } });
+var RecursiveChunker_1 = require("./RecursiveChunker");
+Object.defineProperty(exports, "RecursiveChunker", { enumerable: true, get: function () { return RecursiveChunker_1.RecursiveChunker; } });
+var TokenChunker_1 = require("./TokenChunker");
+Object.defineProperty(exports, "TokenChunker", { enumerable: true, get: function () { return TokenChunker_1.TokenChunker; } });
+//# sourceMappingURL=index.js.map

package/dist/chunkers/types.d.ts

@@ -0,0 +1,95 @@
+/**
+ * Represents a chunk of text with metadata for tracking and linking.
+ */
+export interface Chunk {
+    /** Unique identifier for this chunk */
+    id: string;
+    /** The text content of the chunk */
+    content: string;
+    /** Metadata about the chunk */
+    metadata: ChunkMetadata;
+}
+/**
+ * Metadata associated with each chunk.
+ */
+export interface ChunkMetadata {
+    /** Zero-based index of this chunk in the sequence */
+    chunkIndex: number;
+    /** Total number of chunks in the sequence */
+    totalChunks: number;
+    /** ID of the previous chunk, or null if first */
+    previousChunkId: string | null;
+    /** ID of the next chunk, or null if last */
+    nextChunkId: string | null;
+    /** Character offset where this chunk starts in the source text */
+    startOffset: number;
+    /** Character offset where this chunk ends in the source text */
+    endOffset: number;
+    /** Optional identifier for the source document */
+    sourceId?: string;
+    /** Optional path to the source file */
+    sourcePath?: string;
+    /** Number of characters in the chunk content */
+    charCount: number;
+    /** Estimated number of tokens (when available) */
+    tokenCount?: number;
+    /** SHA-256 hash of the content for deduplication */
+    hash: string;
+    /** Section title if detected (e.g., markdown headers) */
+    sectionTitle?: string;
+    [key: string]: unknown;
+}
+/**
+ * Configuration for creating a chunker.
+ */
+export interface ChunkerConfig {
+    /** Target size for each chunk (in characters or tokens depending on chunker) */
+    chunkSize: number;
+    /** Number of characters/tokens to overlap between chunks (default: 0) */
+    chunkOverlap?: number;
+    /**
+     * Optional processor function applied to each chunk.
+     * Can modify the chunk or return null to filter it out.
+     */
+    chunkProcessor?: (chunk: Chunk, index: number, all: Chunk[]) => Chunk | null | Promise<Chunk | null>;
+    /**
+     * Custom ID generator function.
+     * @param content - The chunk content
+     * @param index - The chunk index
+     * @param sourceId - Optional source document ID
+     * @returns A unique ID for the chunk
+     */
+    idGenerator?: (content: string, index: number, sourceId?: string) => string;
+}
+/**
+ * Options passed when chunking text.
+ */
+export interface ChunkOptions {
+    /** Identifier for the source document */
+    sourceId?: string;
+    /** Path to the source file */
+    sourcePath?: string;
+    /** Additional metadata to merge into each chunk */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Configuration specific to RecursiveChunker.
+ */
+export interface RecursiveChunkerConfig extends ChunkerConfig {
+    /**
+     * Separators to try in order, from largest to smallest semantic unit.
+     * Default: ["\n\n", "\n", ". ", " "]
+     */
+    separators?: string[];
+}
+/**
+ * Configuration specific to TokenChunker.
+ */
+export interface TokenChunkerConfig extends ChunkerConfig {
+    /**
+     * Chunk size is in tokens, not characters.
+     * Uses tokenx for estimation (~96% accuracy).
+     */
+    chunkSize: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/chunkers/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map