@toolpack-sdk/knowledge 1.3.0 → 2.0.0-alpha.1

package/dist/index.d.cts

@@ -14,6 +14,8 @@ interface QueryOptions {
     filter?: MetadataFilter;
     includeMetadata?: boolean;
     includeVectors?: boolean;
+    searchType?: 'semantic' | 'keyword' | 'hybrid';
+    semanticWeight?: number;
 }
 interface MetadataFilter {
     [key: string]: string | number | boolean | {
@@ -32,9 +34,11 @@ interface QueryResult {
 interface KnowledgeProvider {
     add(chunks: Chunk[]): Promise<void>;
     query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    keywordQuery?(query: string, options?: QueryOptions): Promise<QueryResult[]>;
     delete(ids: string[]): Promise<void>;
     clear(): Promise<void>;
     validateDimensions(dimensions: number): Promise<void>;
+    getAllChunks?(): Promise<Chunk[]>;
     close?(): void;
 }
 interface KnowledgeSource {
@@ -81,6 +85,7 @@ interface KnowledgeOptions {
     onError?: ErrorHandler;
     onSync?: SyncEventHandler;
     onEmbeddingProgress?: EmbeddingProgressHandler;
+    streamingBatchSize?: number;
 }
 type ErrorHandler = (error: Error, context: {
     file?: string;
@@ -109,8 +114,20 @@ declare class Knowledge {
     private constructor();
     static create(options: KnowledgeOptions): Promise<Knowledge>;
     query(text: string, options?: QueryOptions): Promise<QueryResult[]>;
+    private semanticQuery;
+    private keywordQuery;
+    private combineHybridResults;
+    private getAllChunks;
     sync(): Promise<void>;
     private embedChunks;
+    /**
+     * Add a single content item to the knowledge base without triggering a full re-sync.
+     * This is useful for runtime additions like conversation history or agent state.
+     * @param content The text content to add
+     * @param metadata Optional metadata to attach to the chunk
+     * @returns The ID of the added chunk
+     */
+    add(content: string, metadata?: Record<string, unknown>): Promise<string>;
     stop(): Promise<void>;
     toTool(): KnowledgeTool;
 }
@@ -156,8 +173,10 @@ declare class MemoryProvider implements KnowledgeProvider {
     validateDimensions(dimensions: number): Promise<void>;
     add(chunks: Chunk[]): Promise<void>;
     query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    keywordQuery(query: string, options?: QueryOptions): Promise<QueryResult[]>;
     delete(ids: string[]): Promise<void>;
     clear(): Promise<void>;
+    getAllChunks(): Promise<Chunk[]>;
 }
 
 interface PersistentKnowledgeProviderOptions {
@@ -176,8 +195,10 @@ declare class PersistentKnowledgeProvider implements KnowledgeProvider {
     validateDimensions(dimensions: number): Promise<void>;
     add(chunks: Chunk[]): Promise<void>;
     query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    keywordQuery(query: string, options?: QueryOptions): Promise<QueryResult[]>;
     delete(ids: string[]): Promise<void>;
     clear(): Promise<void>;
+    getAllChunks(): Promise<Chunk[]>;
     shouldReSync(): boolean;
     close(): void;
 }
@@ -201,6 +222,140 @@ declare class MarkdownSource implements KnowledgeSource {
     private generateChunkId;
 }
 
+interface WebUrlSourceOptions {
+    maxChunkSize?: number;
+    chunkOverlap?: number;
+    minChunkSize?: number;
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+    maxDepth?: number;
+    userAgent?: string;
+    delayMs?: number;
+    timeoutMs?: number;
+    sameDomainOnly?: boolean;
+    maxPagesPerDomain?: number;
+}
+declare class WebUrlSource implements KnowledgeSource {
+    private urls;
+    private options;
+    private crawledUrls;
+    private domainPageCount;
+    private lastRequestTime;
+    constructor(urls: string[], options?: WebUrlSourceOptions);
+    load(): AsyncIterable<Chunk>;
+    private crawlUrls;
+    private fetchPage;
+    private chunkPage;
+    private generateChunkId;
+}
+
+interface ApiDataSourceOptions {
+    maxChunkSize?: number;
+    chunkOverlap?: number;
+    minChunkSize?: number;
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+    headers?: Record<string, string>;
+    method?: 'GET' | 'POST';
+    body?: unknown;
+    timeoutMs?: number;
+    pagination?: {
+        param: string;
+        start: number;
+        step: number;
+        maxPages?: number;
+    } | null;
+    dataPath?: string;
+    contentExtractor?: (item: unknown) => string;
+    metadataExtractor?: (item: unknown) => Record<string, unknown>;
+}
+declare class ApiDataSource implements KnowledgeSource {
+    private url;
+    private options;
+    constructor(url: string, options?: ApiDataSourceOptions);
+    load(): AsyncIterable<Chunk>;
+    private fetchData;
+    private buildUrl;
+    private fetchPage;
+    private extractItems;
+    private chunkItem;
+    private defaultContentExtractor;
+    private defaultMetadataExtractor;
+    private generateChunkId;
+}
+
+interface JSONSourceOptions {
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+    filter?: (item: unknown) => boolean;
+    chunkSize?: number;
+    /** Required. Transform each JSON item into a human-readable string for AI embedding. */
+    toContent: (item: unknown) => string;
+}
+/**
+ * Knowledge source for JSON files.
+ * Supports jq-like filtering and chunking of large arrays.
+ */
+declare class JSONSource implements KnowledgeSource {
+    private filePath;
+    private options;
+    constructor(filePath: string, options: JSONSourceOptions);
+    load(): AsyncIterable<Chunk>;
+}
+
+interface SQLiteSourceOptions {
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+    query?: string;
+    chunkSize?: number;
+    /** Required. Transform each database row into a human-readable string for AI embedding. */
+    toContent: (row: Record<string, unknown>) => string;
+    preLoadCSV?: {
+        tableName: string;
+        csvPath: string;
+        delimiter?: string;
+        headers?: boolean;
+    };
+}
+/**
+ * Knowledge source for SQLite databases.
+ * Supports SQL queries and optional CSV/TSV pre-loading.
+ * Note: This requires the 'better-sqlite3' package to be installed.
+ */
+declare class SQLiteSource implements KnowledgeSource {
+    private dbPath;
+    private options;
+    constructor(dbPath: string, options: SQLiteSourceOptions);
+    load(): AsyncIterable<Chunk>;
+    private loadCSV;
+}
+
+interface PostgresSourceOptions {
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+    query: string;
+    chunkSize?: number;
+    /** Required. Transform each database row into a human-readable string for AI embedding. */
+    toContent: (row: Record<string, unknown>) => string;
+    connectionString?: string;
+    host?: string;
+    port?: number;
+    database?: string;
+    user?: string;
+    password?: string;
+    ssl?: boolean;
+}
+/**
+ * Knowledge source for PostgreSQL databases.
+ * Supports SQL queries with optional chunking.
+ * Note: This requires the 'pg' package to be installed.
+ */
+declare class PostgresSource implements KnowledgeSource {
+    private options;
+    constructor(options: PostgresSourceOptions);
+    load(): AsyncIterable<Chunk>;
+}
+
 interface OllamaEmbedderOptions {
     model: string;
     baseUrl?: string;
@@ -236,4 +391,26 @@ declare class OpenAIEmbedder implements Embedder {
     embedBatch(texts: string[]): Promise<number[][]>;
 }
 
-export { type Chunk, ChunkTooLargeError, type ChunkUpdate, DimensionMismatchError, type Embedder, EmbeddingError, type EmbeddingProgressEvent, type EmbeddingProgressHandler, type ErrorHandler, IngestionError, Knowledge, KnowledgeError, type KnowledgeOptions, type KnowledgeProvider, KnowledgeProviderError, type KnowledgeSource, type KnowledgeTool, type KnowledgeToolParams, type KnowledgeToolResult, MarkdownSource, type MarkdownSourceOptions, MemoryProvider, type MemoryProviderOptions, type MetadataFilter, OllamaEmbedder, type OllamaEmbedderOptions, OpenAIEmbedder, type OpenAIEmbedderOptions, PersistentKnowledgeProvider, type PersistentKnowledgeProviderOptions, type QueryOptions, type QueryResult, type SyncEvent, type SyncEventHandler };
+interface OpenRouterEmbedderOptions {
+    model: string;
+    apiKey: string;
+    /** Override output dimensions for models not in the built-in map */
+    dimensions?: number;
+    retries?: number;
+    retryDelay?: number;
+    timeout?: number;
+}
+declare class OpenRouterEmbedder implements Embedder {
+    private options;
+    readonly dimensions: number;
+    private client;
+    constructor(options: OpenRouterEmbedderOptions);
+    private getModelDimensions;
+    embed(text: string): Promise<number[]>;
+    embedBatch(texts: string[]): Promise<number[][]>;
+}
+
+declare function keywordSearch(text: string, query: string): number;
+declare function combineScores(semanticScore: number, keywordScore: number, semanticWeight?: number): number;
+
+export { ApiDataSource, type ApiDataSourceOptions, type Chunk, ChunkTooLargeError, type ChunkUpdate, DimensionMismatchError, type Embedder, EmbeddingError, type EmbeddingProgressEvent, type EmbeddingProgressHandler, type ErrorHandler, IngestionError, JSONSource, type JSONSourceOptions, Knowledge, KnowledgeError, type KnowledgeOptions, type KnowledgeProvider, KnowledgeProviderError, type KnowledgeSource, type KnowledgeTool, type KnowledgeToolParams, type KnowledgeToolResult, MarkdownSource, type MarkdownSourceOptions, MemoryProvider, type MemoryProviderOptions, type MetadataFilter, OllamaEmbedder, type OllamaEmbedderOptions, OpenAIEmbedder, type OpenAIEmbedderOptions, OpenRouterEmbedder, type OpenRouterEmbedderOptions, PersistentKnowledgeProvider, type PersistentKnowledgeProviderOptions, PostgresSource, type PostgresSourceOptions, type QueryOptions, type QueryResult, SQLiteSource, type SQLiteSourceOptions, type SyncEvent, type SyncEventHandler, WebUrlSource, type WebUrlSourceOptions, combineScores, keywordSearch };

package/dist/index.d.ts

@@ -14,6 +14,8 @@ interface QueryOptions {
     filter?: MetadataFilter;
     includeMetadata?: boolean;
     includeVectors?: boolean;
+    searchType?: 'semantic' | 'keyword' | 'hybrid';
+    semanticWeight?: number;
 }
 interface MetadataFilter {
     [key: string]: string | number | boolean | {
@@ -32,9 +34,11 @@ interface QueryResult {
 interface KnowledgeProvider {
     add(chunks: Chunk[]): Promise<void>;
     query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    keywordQuery?(query: string, options?: QueryOptions): Promise<QueryResult[]>;
     delete(ids: string[]): Promise<void>;
     clear(): Promise<void>;
     validateDimensions(dimensions: number): Promise<void>;
+    getAllChunks?(): Promise<Chunk[]>;
     close?(): void;
 }
 interface KnowledgeSource {
@@ -81,6 +85,7 @@ interface KnowledgeOptions {
     onError?: ErrorHandler;
     onSync?: SyncEventHandler;
     onEmbeddingProgress?: EmbeddingProgressHandler;
+    streamingBatchSize?: number;
 }
 type ErrorHandler = (error: Error, context: {
     file?: string;
@@ -109,8 +114,20 @@ declare class Knowledge {
     private constructor();
     static create(options: KnowledgeOptions): Promise<Knowledge>;
     query(text: string, options?: QueryOptions): Promise<QueryResult[]>;
+    private semanticQuery;
+    private keywordQuery;
+    private combineHybridResults;
+    private getAllChunks;
     sync(): Promise<void>;
     private embedChunks;
+    /**
+     * Add a single content item to the knowledge base without triggering a full re-sync.
+     * This is useful for runtime additions like conversation history or agent state.
+     * @param content The text content to add
+     * @param metadata Optional metadata to attach to the chunk
+     * @returns The ID of the added chunk
+     */
+    add(content: string, metadata?: Record<string, unknown>): Promise<string>;
     stop(): Promise<void>;
     toTool(): KnowledgeTool;
 }
@@ -156,8 +173,10 @@ declare class MemoryProvider implements KnowledgeProvider {
     validateDimensions(dimensions: number): Promise<void>;
     add(chunks: Chunk[]): Promise<void>;
     query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    keywordQuery(query: string, options?: QueryOptions): Promise<QueryResult[]>;
     delete(ids: string[]): Promise<void>;
     clear(): Promise<void>;
+    getAllChunks(): Promise<Chunk[]>;
 }
 
 interface PersistentKnowledgeProviderOptions {
@@ -176,8 +195,10 @@ declare class PersistentKnowledgeProvider implements KnowledgeProvider {
     validateDimensions(dimensions: number): Promise<void>;
     add(chunks: Chunk[]): Promise<void>;
     query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    keywordQuery(query: string, options?: QueryOptions): Promise<QueryResult[]>;
     delete(ids: string[]): Promise<void>;
     clear(): Promise<void>;
+    getAllChunks(): Promise<Chunk[]>;
     shouldReSync(): boolean;
     close(): void;
 }
@@ -201,6 +222,140 @@ declare class MarkdownSource implements KnowledgeSource {
     private generateChunkId;
 }
 
+interface WebUrlSourceOptions {
+    maxChunkSize?: number;
+    chunkOverlap?: number;
+    minChunkSize?: number;
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+    maxDepth?: number;
+    userAgent?: string;
+    delayMs?: number;
+    timeoutMs?: number;
+    sameDomainOnly?: boolean;
+    maxPagesPerDomain?: number;
+}
+declare class WebUrlSource implements KnowledgeSource {
+    private urls;
+    private options;
+    private crawledUrls;
+    private domainPageCount;
+    private lastRequestTime;
+    constructor(urls: string[], options?: WebUrlSourceOptions);
+    load(): AsyncIterable<Chunk>;
+    private crawlUrls;
+    private fetchPage;
+    private chunkPage;
+    private generateChunkId;
+}
+
+interface ApiDataSourceOptions {
+    maxChunkSize?: number;
+    chunkOverlap?: number;
+    minChunkSize?: number;
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+    headers?: Record<string, string>;
+    method?: 'GET' | 'POST';
+    body?: unknown;
+    timeoutMs?: number;
+    pagination?: {
+        param: string;
+        start: number;
+        step: number;
+        maxPages?: number;
+    } | null;
+    dataPath?: string;
+    contentExtractor?: (item: unknown) => string;
+    metadataExtractor?: (item: unknown) => Record<string, unknown>;
+}
+declare class ApiDataSource implements KnowledgeSource {
+    private url;
+    private options;
+    constructor(url: string, options?: ApiDataSourceOptions);
+    load(): AsyncIterable<Chunk>;
+    private fetchData;
+    private buildUrl;
+    private fetchPage;
+    private extractItems;
+    private chunkItem;
+    private defaultContentExtractor;
+    private defaultMetadataExtractor;
+    private generateChunkId;
+}
+
+interface JSONSourceOptions {
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+    filter?: (item: unknown) => boolean;
+    chunkSize?: number;
+    /** Required. Transform each JSON item into a human-readable string for AI embedding. */
+    toContent: (item: unknown) => string;
+}
+/**
+ * Knowledge source for JSON files.
+ * Supports jq-like filtering and chunking of large arrays.
+ */
+declare class JSONSource implements KnowledgeSource {
+    private filePath;
+    private options;
+    constructor(filePath: string, options: JSONSourceOptions);
+    load(): AsyncIterable<Chunk>;
+}
+
+interface SQLiteSourceOptions {
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+    query?: string;
+    chunkSize?: number;
+    /** Required. Transform each database row into a human-readable string for AI embedding. */
+    toContent: (row: Record<string, unknown>) => string;
+    preLoadCSV?: {
+        tableName: string;
+        csvPath: string;
+        delimiter?: string;
+        headers?: boolean;
+    };
+}
+/**
+ * Knowledge source for SQLite databases.
+ * Supports SQL queries and optional CSV/TSV pre-loading.
+ * Note: This requires the 'better-sqlite3' package to be installed.
+ */
+declare class SQLiteSource implements KnowledgeSource {
+    private dbPath;
+    private options;
+    constructor(dbPath: string, options: SQLiteSourceOptions);
+    load(): AsyncIterable<Chunk>;
+    private loadCSV;
+}
+
+interface PostgresSourceOptions {
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+    query: string;
+    chunkSize?: number;
+    /** Required. Transform each database row into a human-readable string for AI embedding. */
+    toContent: (row: Record<string, unknown>) => string;
+    connectionString?: string;
+    host?: string;
+    port?: number;
+    database?: string;
+    user?: string;
+    password?: string;
+    ssl?: boolean;
+}
+/**
+ * Knowledge source for PostgreSQL databases.
+ * Supports SQL queries with optional chunking.
+ * Note: This requires the 'pg' package to be installed.
+ */
+declare class PostgresSource implements KnowledgeSource {
+    private options;
+    constructor(options: PostgresSourceOptions);
+    load(): AsyncIterable<Chunk>;
+}
+
 interface OllamaEmbedderOptions {
     model: string;
     baseUrl?: string;
@@ -236,4 +391,26 @@ declare class OpenAIEmbedder implements Embedder {
     embedBatch(texts: string[]): Promise<number[][]>;
 }
 
-export { type Chunk, ChunkTooLargeError, type ChunkUpdate, DimensionMismatchError, type Embedder, EmbeddingError, type EmbeddingProgressEvent, type EmbeddingProgressHandler, type ErrorHandler, IngestionError, Knowledge, KnowledgeError, type KnowledgeOptions, type KnowledgeProvider, KnowledgeProviderError, type KnowledgeSource, type KnowledgeTool, type KnowledgeToolParams, type KnowledgeToolResult, MarkdownSource, type MarkdownSourceOptions, MemoryProvider, type MemoryProviderOptions, type MetadataFilter, OllamaEmbedder, type OllamaEmbedderOptions, OpenAIEmbedder, type OpenAIEmbedderOptions, PersistentKnowledgeProvider, type PersistentKnowledgeProviderOptions, type QueryOptions, type QueryResult, type SyncEvent, type SyncEventHandler };
+interface OpenRouterEmbedderOptions {
+    model: string;
+    apiKey: string;
+    /** Override output dimensions for models not in the built-in map */
+    dimensions?: number;
+    retries?: number;
+    retryDelay?: number;
+    timeout?: number;
+}
+declare class OpenRouterEmbedder implements Embedder {
+    private options;
+    readonly dimensions: number;
+    private client;
+    constructor(options: OpenRouterEmbedderOptions);
+    private getModelDimensions;
+    embed(text: string): Promise<number[]>;
+    embedBatch(texts: string[]): Promise<number[][]>;
+}
+
+declare function keywordSearch(text: string, query: string): number;
+declare function combineScores(semanticScore: number, keywordScore: number, semanticWeight?: number): number;
+
+export { ApiDataSource, type ApiDataSourceOptions, type Chunk, ChunkTooLargeError, type ChunkUpdate, DimensionMismatchError, type Embedder, EmbeddingError, type EmbeddingProgressEvent, type EmbeddingProgressHandler, type ErrorHandler, IngestionError, JSONSource, type JSONSourceOptions, Knowledge, KnowledgeError, type KnowledgeOptions, type KnowledgeProvider, KnowledgeProviderError, type KnowledgeSource, type KnowledgeTool, type KnowledgeToolParams, type KnowledgeToolResult, MarkdownSource, type MarkdownSourceOptions, MemoryProvider, type MemoryProviderOptions, type MetadataFilter, OllamaEmbedder, type OllamaEmbedderOptions, OpenAIEmbedder, type OpenAIEmbedderOptions, OpenRouterEmbedder, type OpenRouterEmbedderOptions, PersistentKnowledgeProvider, type PersistentKnowledgeProviderOptions, PostgresSource, type PostgresSourceOptions, type QueryOptions, type QueryResult, SQLiteSource, type SQLiteSourceOptions, type SyncEvent, type SyncEventHandler, WebUrlSource, type WebUrlSourceOptions, combineScores, keywordSearch };