@retrivora-ai/rag-engine 1.2.1 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DocumentChunker-BXOUMKoP.d.ts +93 -0
- package/dist/DocumentChunker-D1dg5iCi.d.mts +93 -0
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/{index-DbtE8wLM.d.ts → index-B67KQ9NN.d.ts} +1 -1
- package/dist/{RagConfig-BOLOz0_O.d.mts → index-Cti1u0y1.d.mts} +86 -94
- package/dist/{RagConfig-BOLOz0_O.d.ts → index-Cti1u0y1.d.ts} +86 -94
- package/dist/{index-64BDupW3.d.mts → index-kUXnRvuI.d.mts} +1 -1
- package/dist/index.d.mts +52 -78
- package/dist/index.d.ts +52 -78
- package/dist/index.js +93 -27
- package/dist/index.mjs +97 -27
- package/dist/server.d.mts +5 -4
- package/dist/server.d.ts +5 -4
- package/package.json +1 -1
- package/src/components/ChatWidget.tsx +1 -8
- package/src/components/ChatWindow.tsx +119 -29
- package/src/components/ConfigProvider.tsx +7 -33
- package/src/components/DocumentUpload.tsx +1 -8
- package/src/components/MessageBubble.tsx +1 -12
- package/src/components/ProductCard.tsx +1 -7
- package/src/components/ProductCarousel.tsx +1 -7
- package/src/components/SourceCard.tsx +1 -6
- package/src/config/RagConfig.ts +2 -0
- package/src/config/uiConstants.ts +23 -0
- package/src/core/Pipeline.ts +2 -1
- package/src/core/VectorPlugin.ts +1 -1
- package/src/handlers/index.ts +1 -1
- package/src/hooks/useRagChat.ts +1 -45
- package/src/hooks/useStoredMessages.ts +1 -1
- package/src/index.ts +20 -5
- package/src/llm/ILLMProvider.ts +1 -13
- package/src/llm/providers/AnthropicProvider.ts +2 -1
- package/src/llm/providers/GeminiProvider.ts +2 -1
- package/src/llm/providers/OllamaProvider.ts +2 -1
- package/src/llm/providers/OpenAIProvider.ts +2 -1
- package/src/llm/providers/UniversalLLMAdapter.ts +2 -1
- package/src/server.ts +2 -2
- package/src/types/chat.ts +53 -0
- package/src/types/index.ts +3 -0
- package/src/types/props.ts +79 -0
- package/dist/DocumentChunker-Dh9TvmGG.d.mts +0 -45
- package/dist/DocumentChunker-Dh9TvmGG.d.ts +0 -45
package/src/handlers/index.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { NextRequest, NextResponse } from 'next/server';
|
|
2
2
|
import { VectorPlugin } from '../core/VectorPlugin';
|
|
3
3
|
import { RagConfig } from '../config/RagConfig';
|
|
4
|
-
import { ChatMessage } from '../
|
|
4
|
+
import { ChatMessage } from '../types';
|
|
5
5
|
import { DocumentParser } from '../utils/DocumentParser';
|
|
6
6
|
|
|
7
7
|
// ─── SSE helpers ──────────────────────────────────────────────────────────────
|
package/src/hooks/useRagChat.ts
CHANGED
|
@@ -19,53 +19,9 @@
|
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
21
|
import { useState, useCallback, useRef, useEffect } from 'react';
|
|
22
|
-
import { VectorMatch } from '../types';
|
|
22
|
+
import { VectorMatch, RagMessage, UseRagChatOptions, UseRagChatReturn } from '../types';
|
|
23
23
|
import { useStoredMessages } from './useStoredMessages';
|
|
24
24
|
|
|
25
|
-
// ─── Types ────────────────────────────────────────────────────────────────────
|
|
26
|
-
|
|
27
|
-
export type MessageRole = 'user' | 'assistant';
|
|
28
|
-
|
|
29
|
-
export interface RagMessage {
|
|
30
|
-
id: string;
|
|
31
|
-
role: MessageRole;
|
|
32
|
-
content: string;
|
|
33
|
-
/** Retrieved source chunks (assistant messages only) */
|
|
34
|
-
sources?: VectorMatch[];
|
|
35
|
-
/** ISO timestamp */
|
|
36
|
-
createdAt: string;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
export interface UseRagChatOptions {
|
|
40
|
-
/** Override the chat API endpoint (default: /api/chat) */
|
|
41
|
-
apiUrl?: string;
|
|
42
|
-
/** Override project namespace */
|
|
43
|
-
namespace?: string;
|
|
44
|
-
/** Persist chat history to localStorage (default: true) */
|
|
45
|
-
persist?: boolean;
|
|
46
|
-
/** Called after each successful assistant reply */
|
|
47
|
-
onReply?: (message: RagMessage) => void;
|
|
48
|
-
/** Called on error */
|
|
49
|
-
onError?: (error: string) => void;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export interface UseRagChatReturn {
|
|
53
|
-
/** All messages in the current conversation */
|
|
54
|
-
messages: RagMessage[];
|
|
55
|
-
/** Whether a response is in flight */
|
|
56
|
-
isLoading: boolean;
|
|
57
|
-
/** Current error message, or null */
|
|
58
|
-
error: string | null;
|
|
59
|
-
/** Send a user message */
|
|
60
|
-
send: (text: string) => Promise<void>;
|
|
61
|
-
/** Clear the conversation (and localStorage if persist=true) */
|
|
62
|
-
clear: () => void;
|
|
63
|
-
/** Retry the last failed send */
|
|
64
|
-
retry: () => Promise<void>;
|
|
65
|
-
/** Programmatically set the conversation (e.g. to restore from a DB) */
|
|
66
|
-
setMessages: React.Dispatch<React.SetStateAction<RagMessage[]>>;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
25
|
// ─── SSE Frame Parser ──────────────────────────────────────────────────────────
|
|
70
26
|
|
|
71
27
|
interface SseTextFrame { type: 'text'; text: string }
|
package/src/index.ts
CHANGED
|
@@ -16,9 +16,24 @@ export { useRagChat } from './hooks/useRagChat';
|
|
|
16
16
|
|
|
17
17
|
// ── Types (Interfaces/Types only, no Node.js deps) ─────────────
|
|
18
18
|
export type { RagConfig, VectorDBConfig, VectorDBProvider, LLMConfig, LLMProvider, EmbeddingConfig, EmbeddingProvider, UIConfig, RAGConfig } from './config/RagConfig';
|
|
19
|
-
export type {
|
|
20
|
-
export type { ILLMProvider, ChatMessage, ChatOptions, EmbedOptions } from './llm/ILLMProvider';
|
|
21
|
-
export type { IngestDocument, ChatResponse } from './types';
|
|
19
|
+
export type { ILLMProvider, EmbedOptions } from './llm/ILLMProvider';
|
|
22
20
|
export type { Chunk, ChunkOptions } from './rag/DocumentChunker';
|
|
23
|
-
export type {
|
|
24
|
-
|
|
21
|
+
export type {
|
|
22
|
+
VectorMatch,
|
|
23
|
+
UpsertDocument,
|
|
24
|
+
IngestDocument,
|
|
25
|
+
ChatResponse,
|
|
26
|
+
ChatMessage,
|
|
27
|
+
ChatOptions,
|
|
28
|
+
RagMessage,
|
|
29
|
+
ClientConfig,
|
|
30
|
+
UseRagChatOptions,
|
|
31
|
+
UseRagChatReturn,
|
|
32
|
+
ChatWindowProps,
|
|
33
|
+
ChatWidgetProps,
|
|
34
|
+
MessageBubbleProps,
|
|
35
|
+
ProductCardProps,
|
|
36
|
+
ProductCarouselProps,
|
|
37
|
+
DocumentUploadProps,
|
|
38
|
+
SourceCardProps
|
|
39
|
+
} from './types';
|
package/src/llm/ILLMProvider.ts
CHANGED
|
@@ -4,19 +4,7 @@
|
|
|
4
4
|
* provider can handle both responsibilities when appropriate.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
role: 'user' | 'assistant' | 'system';
|
|
9
|
-
content: string;
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
export interface ChatOptions {
|
|
13
|
-
/** Override max tokens for this specific call */
|
|
14
|
-
maxTokens?: number;
|
|
15
|
-
/** Override temperature for this specific call */
|
|
16
|
-
temperature?: number;
|
|
17
|
-
/** Stop sequences */
|
|
18
|
-
stop?: string[];
|
|
19
|
-
}
|
|
7
|
+
import { ChatMessage, ChatOptions } from '../types';
|
|
20
8
|
|
|
21
9
|
export interface EmbedOptions {
|
|
22
10
|
/** Override model for this specific embed call */
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import Anthropic from '@anthropic-ai/sdk';
|
|
6
|
-
import { ILLMProvider,
|
|
6
|
+
import { ILLMProvider, EmbedOptions } from '../ILLMProvider';
|
|
7
|
+
import { ChatMessage, ChatOptions } from '../../types';
|
|
7
8
|
import { LLMConfig, EmbeddingConfig } from '../../config/RagConfig';
|
|
8
9
|
import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
|
|
9
10
|
import { ValidationError } from '../../core/ConfigValidator';
|
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
17
|
import { GoogleGenAI } from '@google/genai';
|
|
18
|
-
import { ILLMProvider,
|
|
18
|
+
import { ILLMProvider, EmbedOptions } from '../ILLMProvider';
|
|
19
|
+
import { ChatMessage, ChatOptions } from '../../types';
|
|
19
20
|
import { LLMConfig, EmbeddingConfig } from '../../config/RagConfig';
|
|
20
21
|
import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
|
|
21
22
|
import { ValidationError } from '../../core/ConfigValidator';
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import axios, { AxiosInstance } from 'axios';
|
|
6
|
-
import { ILLMProvider,
|
|
6
|
+
import { ILLMProvider, EmbedOptions } from '../ILLMProvider';
|
|
7
|
+
import { ChatMessage, ChatOptions } from '../../types';
|
|
7
8
|
import { LLMConfig, EmbeddingConfig } from '../../config/RagConfig';
|
|
8
9
|
import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
|
|
9
10
|
import { ValidationError } from '../../core/ConfigValidator';
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import OpenAI from 'openai';
|
|
6
|
-
import { ILLMProvider,
|
|
6
|
+
import { ILLMProvider, EmbedOptions } from '../ILLMProvider';
|
|
7
|
+
import { ChatMessage, ChatOptions } from '../../types';
|
|
7
8
|
import { LLMConfig, EmbeddingConfig } from '../../config/RagConfig';
|
|
8
9
|
import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
|
|
9
10
|
import { ValidationError } from '../../core/ConfigValidator';
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import axios, { AxiosInstance } from 'axios';
|
|
2
|
-
import { ILLMProvider
|
|
2
|
+
import { ILLMProvider } from '../ILLMProvider';
|
|
3
|
+
import { ChatMessage } from '../../types';
|
|
3
4
|
import { LLMConfig, EmbeddingConfig } from '../../config/RagConfig';
|
|
4
5
|
import { resolvePath, buildPayload } from '../../utils/templateUtils';
|
|
5
6
|
import { LLM_PROFILES } from '../../config/UniversalProfiles';
|
package/src/server.ts
CHANGED
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
|
|
8
8
|
// ── Shared Types ──────────────────────────────────────────────
|
|
9
9
|
export type { RagConfig, VectorDBConfig, LLMConfig, EmbeddingConfig, UIConfig, RAGConfig, VectorDBProvider, LLMProvider, EmbeddingProvider } from './config/RagConfig';
|
|
10
|
-
export type { VectorMatch, UpsertDocument, IngestDocument, ChatResponse } from './types';
|
|
11
|
-
export type { ILLMProvider,
|
|
10
|
+
export type { VectorMatch, UpsertDocument, IngestDocument, ChatResponse, ChatMessage, ChatOptions } from './types';
|
|
11
|
+
export type { ILLMProvider, EmbedOptions } from './llm/ILLMProvider';
|
|
12
12
|
export type { Chunk, ChunkOptions } from './rag/DocumentChunker';
|
|
13
13
|
export type { HealthCheckResult, IProviderValidator, IProviderHealthChecker } from './core/ProviderInterfaces';
|
|
14
14
|
export type { ValidationError } from './core/ConfigValidator';
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { VectorMatch } from './index';
|
|
2
|
+
|
|
3
|
+
export type MessageRole = 'user' | 'assistant' | 'system';
|
|
4
|
+
|
|
5
|
+
export interface ChatMessage {
|
|
6
|
+
role: MessageRole;
|
|
7
|
+
content: string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface RagMessage extends ChatMessage {
|
|
11
|
+
id: string;
|
|
12
|
+
sources?: VectorMatch[];
|
|
13
|
+
createdAt: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface ChatOptions {
|
|
17
|
+
/** Override max tokens for this specific call */
|
|
18
|
+
maxTokens?: number;
|
|
19
|
+
/** Override temperature for this specific call */
|
|
20
|
+
temperature?: number;
|
|
21
|
+
/** Stop sequences */
|
|
22
|
+
stop?: string[];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface UseRagChatOptions {
|
|
26
|
+
/** Override the chat API endpoint (default: /api/chat) */
|
|
27
|
+
apiUrl?: string;
|
|
28
|
+
/** Override project namespace */
|
|
29
|
+
namespace?: string;
|
|
30
|
+
/** Persist chat history to localStorage (default: true) */
|
|
31
|
+
persist?: boolean;
|
|
32
|
+
/** Called after each successful assistant reply */
|
|
33
|
+
onReply?: (message: RagMessage) => void;
|
|
34
|
+
/** Called on error */
|
|
35
|
+
onError?: (error: string) => void;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface UseRagChatReturn {
|
|
39
|
+
/** All messages in the current conversation */
|
|
40
|
+
messages: RagMessage[];
|
|
41
|
+
/** Whether a response is in flight */
|
|
42
|
+
isLoading: boolean;
|
|
43
|
+
/** Current error message, or null */
|
|
44
|
+
error: string | null;
|
|
45
|
+
/** Send a user message */
|
|
46
|
+
send: (text: string) => Promise<void>;
|
|
47
|
+
/** Clear the conversation (and localStorage if persist=true) */
|
|
48
|
+
clear: () => void;
|
|
49
|
+
/** Retry the last failed send */
|
|
50
|
+
retry: () => Promise<void>;
|
|
51
|
+
/** Programmatically set the conversation (e.g. to restore from a DB) */
|
|
52
|
+
setMessages: React.Dispatch<React.SetStateAction<RagMessage[]>>;
|
|
53
|
+
}
|
package/src/types/index.ts
CHANGED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { ReactNode, CSSProperties, MouseEvent } from 'react';
|
|
2
|
+
import { Product, VectorMatch } from './index';
|
|
3
|
+
import { RagMessage } from './chat';
|
|
4
|
+
import { UIConfig } from '../config/RagConfig';
|
|
5
|
+
|
|
6
|
+
export interface ChatWindowProps {
|
|
7
|
+
/** Additional className for the wrapper div */
|
|
8
|
+
className?: string;
|
|
9
|
+
/** Inline styles for the wrapper div */
|
|
10
|
+
style?: CSSProperties;
|
|
11
|
+
/** Called when the close button is clicked (for widget mode) */
|
|
12
|
+
onClose?: () => void;
|
|
13
|
+
/** Whether to show a close (X) button in the header */
|
|
14
|
+
showClose?: boolean;
|
|
15
|
+
/** Called when the user starts dragging the resize handle */
|
|
16
|
+
onResizeStart?: (e: MouseEvent) => void;
|
|
17
|
+
/** Called when the user clicks the reset size button */
|
|
18
|
+
onResetResize?: () => void;
|
|
19
|
+
/** Whether the window has been resized from its default */
|
|
20
|
+
isResized?: boolean;
|
|
21
|
+
/** Called when the user clicks the maximize/minimize button */
|
|
22
|
+
onMaximize?: () => void;
|
|
23
|
+
/** Whether the window is currently maximized */
|
|
24
|
+
isMaximized?: boolean;
|
|
25
|
+
/** Called when the user clicks 'Add to Cart' on a product */
|
|
26
|
+
onAddToCart?: (product: Product) => void;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface ChatWidgetProps {
|
|
30
|
+
/** Position of the floating button. Defaults to bottom-right. */
|
|
31
|
+
position?: 'bottom-right' | 'bottom-left';
|
|
32
|
+
/** Called when the user clicks 'Add to Cart' on a product */
|
|
33
|
+
onAddToCart?: (product: Product) => void;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface MessageBubbleProps {
|
|
37
|
+
message: RagMessage;
|
|
38
|
+
sources?: VectorMatch[];
|
|
39
|
+
isStreaming?: boolean;
|
|
40
|
+
primaryColor: string;
|
|
41
|
+
accentColor: string;
|
|
42
|
+
onAddToCart?: (product: Product) => void;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface ProductCardProps {
|
|
46
|
+
product: Product;
|
|
47
|
+
primaryColor?: string;
|
|
48
|
+
onAddToCart?: (product: Product) => void;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface ProductCarouselProps {
|
|
52
|
+
products: Product[];
|
|
53
|
+
primaryColor?: string;
|
|
54
|
+
onAddToCart?: (product: Product) => void;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface DocumentUploadProps {
|
|
58
|
+
/** Optional namespace for the upload */
|
|
59
|
+
namespace?: string;
|
|
60
|
+
/** Callback when upload completes */
|
|
61
|
+
onUploadComplete?: (results: unknown) => void;
|
|
62
|
+
/** Additional className */
|
|
63
|
+
className?: string;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface SourceCardProps {
|
|
67
|
+
source: VectorMatch;
|
|
68
|
+
index: number;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface ClientConfig {
|
|
72
|
+
projectId: string;
|
|
73
|
+
ui: Required<UIConfig>;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export interface ConfigProviderProps {
|
|
77
|
+
config?: { projectId?: string; ui?: Partial<UIConfig> };
|
|
78
|
+
children: ReactNode;
|
|
79
|
+
}
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* DocumentChunker — splits long text into overlapping chunks
|
|
3
|
-
* suitable for vector embedding and retrieval.
|
|
4
|
-
*
|
|
5
|
-
* Strategy: sentence-aware sliding window
|
|
6
|
-
* 1. Split on sentence boundaries
|
|
7
|
-
* 2. Accumulate sentences into chunks up to `chunkSize` characters
|
|
8
|
-
* 3. Carry over `chunkOverlap` characters from the previous chunk
|
|
9
|
-
*/
|
|
10
|
-
interface Chunk {
|
|
11
|
-
id: string;
|
|
12
|
-
content: string;
|
|
13
|
-
metadata?: Record<string, unknown>;
|
|
14
|
-
}
|
|
15
|
-
interface ChunkOptions {
|
|
16
|
-
/** Target chunk size in characters (default 1000) */
|
|
17
|
-
chunkSize?: number;
|
|
18
|
-
/** Overlap between consecutive chunks in characters (default 200) */
|
|
19
|
-
chunkOverlap?: number;
|
|
20
|
-
/** Source document identifier used as ID prefix */
|
|
21
|
-
docId?: string | number;
|
|
22
|
-
/** Extra metadata to attach to every chunk */
|
|
23
|
-
metadata?: Record<string, unknown>;
|
|
24
|
-
/** Characters used to split text, in order of priority */
|
|
25
|
-
separators?: string[];
|
|
26
|
-
}
|
|
27
|
-
declare class DocumentChunker {
|
|
28
|
-
private readonly chunkSize;
|
|
29
|
-
private readonly chunkOverlap;
|
|
30
|
-
private readonly separators;
|
|
31
|
-
constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
|
|
32
|
-
/**
|
|
33
|
-
* Split a single text string into overlapping chunks using a recursive strategy.
|
|
34
|
-
* Preserves structural boundaries (Markdown headers) where possible.
|
|
35
|
-
*/
|
|
36
|
-
chunk(text: string, options?: ChunkOptions): Chunk[];
|
|
37
|
-
private recursiveSplit;
|
|
38
|
-
chunkMany(documents: Array<{
|
|
39
|
-
content: string;
|
|
40
|
-
docId?: string | number;
|
|
41
|
-
metadata?: Record<string, unknown>;
|
|
42
|
-
}>): Chunk[];
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
export { type Chunk as C, DocumentChunker as D, type ChunkOptions as a };
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* DocumentChunker — splits long text into overlapping chunks
|
|
3
|
-
* suitable for vector embedding and retrieval.
|
|
4
|
-
*
|
|
5
|
-
* Strategy: sentence-aware sliding window
|
|
6
|
-
* 1. Split on sentence boundaries
|
|
7
|
-
* 2. Accumulate sentences into chunks up to `chunkSize` characters
|
|
8
|
-
* 3. Carry over `chunkOverlap` characters from the previous chunk
|
|
9
|
-
*/
|
|
10
|
-
interface Chunk {
|
|
11
|
-
id: string;
|
|
12
|
-
content: string;
|
|
13
|
-
metadata?: Record<string, unknown>;
|
|
14
|
-
}
|
|
15
|
-
interface ChunkOptions {
|
|
16
|
-
/** Target chunk size in characters (default 1000) */
|
|
17
|
-
chunkSize?: number;
|
|
18
|
-
/** Overlap between consecutive chunks in characters (default 200) */
|
|
19
|
-
chunkOverlap?: number;
|
|
20
|
-
/** Source document identifier used as ID prefix */
|
|
21
|
-
docId?: string | number;
|
|
22
|
-
/** Extra metadata to attach to every chunk */
|
|
23
|
-
metadata?: Record<string, unknown>;
|
|
24
|
-
/** Characters used to split text, in order of priority */
|
|
25
|
-
separators?: string[];
|
|
26
|
-
}
|
|
27
|
-
declare class DocumentChunker {
|
|
28
|
-
private readonly chunkSize;
|
|
29
|
-
private readonly chunkOverlap;
|
|
30
|
-
private readonly separators;
|
|
31
|
-
constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
|
|
32
|
-
/**
|
|
33
|
-
* Split a single text string into overlapping chunks using a recursive strategy.
|
|
34
|
-
* Preserves structural boundaries (Markdown headers) where possible.
|
|
35
|
-
*/
|
|
36
|
-
chunk(text: string, options?: ChunkOptions): Chunk[];
|
|
37
|
-
private recursiveSplit;
|
|
38
|
-
chunkMany(documents: Array<{
|
|
39
|
-
content: string;
|
|
40
|
-
docId?: string | number;
|
|
41
|
-
metadata?: Record<string, unknown>;
|
|
42
|
-
}>): Chunk[];
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
export { type Chunk as C, DocumentChunker as D, type ChunkOptions as a };
|