@vibe-agent-toolkit/rag 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunking/chunk-by-tokens.d.ts +8 -1
- package/dist/chunking/chunk-by-tokens.d.ts.map +1 -1
- package/dist/chunking/chunk-by-tokens.js +96 -11
- package/dist/chunking/chunk-by-tokens.js.map +1 -1
- package/dist/chunking/chunk-resource.d.ts.map +1 -1
- package/dist/chunking/chunk-resource.js +49 -11
- package/dist/chunking/chunk-resource.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/interfaces/index.d.ts +1 -1
- package/dist/interfaces/index.d.ts.map +1 -1
- package/dist/interfaces/provider.d.ts +36 -1
- package/dist/interfaces/provider.d.ts.map +1 -1
- package/dist/schemas/chunk.d.ts +6 -0
- package/dist/schemas/chunk.d.ts.map +1 -1
- package/dist/schemas/core-chunk.d.ts +6 -0
- package/dist/schemas/core-chunk.d.ts.map +1 -1
- package/dist/schemas/core-chunk.js +3 -0
- package/dist/schemas/core-chunk.js.map +1 -1
- package/dist/schemas/query.d.ts +10 -0
- package/dist/schemas/query.d.ts.map +1 -1
- package/package.json +3 -3
|
@@ -8,7 +8,12 @@ import type { ChunkingConfig, RawChunk } from './types.js';
|
|
|
8
8
|
* Chunk text by token count
|
|
9
9
|
*
|
|
10
10
|
* Splits text when it exceeds the effective target size (targetChunkSize * paddingFactor).
|
|
11
|
-
* Respects paragraph boundaries when possible, falls back to
|
|
11
|
+
* Respects paragraph boundaries when possible, falls back to line splitting for large paragraphs.
|
|
12
|
+
*
|
|
13
|
+
* Complexity note: This function handles multiple chunking strategies (early return, paragraph-level,
|
|
14
|
+
* line-level fallback) with position tracking for accurate line numbers. The core loop requires
|
|
15
|
+
* coordinated state management that doesn't simplify well into smaller functions without losing
|
|
16
|
+
* clarity. Helper functions already extracted for line-level chunking and chunk creation.
|
|
12
17
|
*
|
|
13
18
|
* @param text - Text to chunk
|
|
14
19
|
* @param config - Chunking configuration
|
|
@@ -18,5 +23,7 @@ import type { ChunkingConfig, RawChunk } from './types.js';
|
|
|
18
23
|
export declare function chunkByTokens(text: string, config: ChunkingConfig, metadata?: {
|
|
19
24
|
headingPath?: string;
|
|
20
25
|
headingLevel?: number;
|
|
26
|
+
startLine?: number;
|
|
27
|
+
endLine?: number;
|
|
21
28
|
}): RawChunk[];
|
|
22
29
|
//# sourceMappingURL=chunk-by-tokens.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk-by-tokens.d.ts","sourceRoot":"","sources":["../../src/chunking/chunk-by-tokens.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"chunk-by-tokens.d.ts","sourceRoot":"","sources":["../../src/chunking/chunk-by-tokens.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AA0F3D;;;;;;;;;;;;;;;GAeG;AAEH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,cAAc,EACtB,QAAQ,CAAC,EAAE;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/F,QAAQ,EAAE,CAwFZ"}
|
|
@@ -4,17 +4,86 @@
|
|
|
4
4
|
* Splits text by token count, respecting paragraph boundaries.
|
|
5
5
|
*/
|
|
6
6
|
import { calculateEffectiveTarget, splitByParagraphs } from './utils.js';
|
|
7
|
+
/**
|
|
8
|
+
* Chunk a large paragraph by lines
|
|
9
|
+
*
|
|
10
|
+
* When a single paragraph exceeds the effective target, split it into smaller chunks by lines.
|
|
11
|
+
*
|
|
12
|
+
* @param paragraph - The large paragraph to split
|
|
13
|
+
* @param text - Original text for line number tracking
|
|
14
|
+
* @param textPosition - Current position in original text
|
|
15
|
+
* @param baseStartLine - Base line number for this section
|
|
16
|
+
* @param config - Chunking configuration
|
|
17
|
+
* @param metadata - Metadata to attach to chunks
|
|
18
|
+
* @returns Array of raw chunks from this paragraph
|
|
19
|
+
*/
|
|
20
|
+
function chunkLargeParagraphByLines(paragraph, text, textPosition, baseStartLine, config, metadata) {
|
|
21
|
+
const { tokenCounter } = config;
|
|
22
|
+
const effectiveTarget = calculateEffectiveTarget(config.targetChunkSize, config.paddingFactor);
|
|
23
|
+
const chunks = [];
|
|
24
|
+
const lines = paragraph.split('\n');
|
|
25
|
+
let lineChunk = '';
|
|
26
|
+
let lineChunkTokens = 0;
|
|
27
|
+
let lineChunkStartPos = textPosition;
|
|
28
|
+
for (const line of lines) {
|
|
29
|
+
const lineTokens = tokenCounter.count(line);
|
|
30
|
+
// If adding this line would exceed target, save current chunk
|
|
31
|
+
if (lineChunkTokens > 0 && lineChunkTokens + lineTokens > effectiveTarget) {
|
|
32
|
+
chunks.push(createChunkWithLineNumbers(lineChunk, text, lineChunkStartPos, baseStartLine, metadata));
|
|
33
|
+
lineChunk = '';
|
|
34
|
+
lineChunkTokens = 0;
|
|
35
|
+
lineChunkStartPos = text.indexOf(line, lineChunkStartPos + lineChunk.length);
|
|
36
|
+
}
|
|
37
|
+
// Add line to current chunk
|
|
38
|
+
lineChunk += (lineChunk.length > 0 ? '\n' : '') + line;
|
|
39
|
+
lineChunkTokens += lineTokens;
|
|
40
|
+
}
|
|
41
|
+
// Add final line chunk
|
|
42
|
+
if (lineChunk.trim().length > 0) {
|
|
43
|
+
chunks.push(createChunkWithLineNumbers(lineChunk, text, lineChunkStartPos, baseStartLine, metadata));
|
|
44
|
+
}
|
|
45
|
+
return chunks;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Create a chunk with line number tracking
|
|
49
|
+
*
|
|
50
|
+
* @param content - Chunk content
|
|
51
|
+
* @param text - Original text for line tracking
|
|
52
|
+
* @param chunkStartPosition - Character position where chunk starts
|
|
53
|
+
* @param baseStartLine - Base line number for section
|
|
54
|
+
* @param metadata - Metadata to attach
|
|
55
|
+
* @returns Raw chunk with line numbers
|
|
56
|
+
*/
|
|
57
|
+
function createChunkWithLineNumbers(content, text, chunkStartPosition, baseStartLine, metadata) {
|
|
58
|
+
const textUpToChunkStart = text.substring(0, chunkStartPosition);
|
|
59
|
+
const linesBeforeChunk = (textUpToChunkStart.match(/\n/g) ?? []).length;
|
|
60
|
+
const chunkStartLine = baseStartLine + linesBeforeChunk;
|
|
61
|
+
const linesInChunk = content.trim().split('\n').length;
|
|
62
|
+
const chunkEndLine = chunkStartLine + linesInChunk - 1;
|
|
63
|
+
return {
|
|
64
|
+
content: content.trim(),
|
|
65
|
+
...metadata,
|
|
66
|
+
startLine: chunkStartLine,
|
|
67
|
+
endLine: chunkEndLine,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
7
70
|
/**
|
|
8
71
|
* Chunk text by token count
|
|
9
72
|
*
|
|
10
73
|
* Splits text when it exceeds the effective target size (targetChunkSize * paddingFactor).
|
|
11
|
-
* Respects paragraph boundaries when possible, falls back to
|
|
74
|
+
* Respects paragraph boundaries when possible, falls back to line splitting for large paragraphs.
|
|
75
|
+
*
|
|
76
|
+
* Complexity note: This function handles multiple chunking strategies (early return, paragraph-level,
|
|
77
|
+
* line-level fallback) with position tracking for accurate line numbers. The core loop requires
|
|
78
|
+
* coordinated state management that doesn't simplify well into smaller functions without losing
|
|
79
|
+
* clarity. Helper functions already extracted for line-level chunking and chunk creation.
|
|
12
80
|
*
|
|
13
81
|
* @param text - Text to chunk
|
|
14
82
|
* @param config - Chunking configuration
|
|
15
83
|
* @param metadata - Optional metadata to attach to chunks
|
|
16
84
|
* @returns Array of raw chunks
|
|
17
85
|
*/
|
|
86
|
+
// eslint-disable-next-line sonarjs/cognitive-complexity
|
|
18
87
|
export function chunkByTokens(text, config, metadata) {
|
|
19
88
|
if (text.trim().length === 0) {
|
|
20
89
|
return [];
|
|
@@ -36,6 +105,11 @@ export function chunkByTokens(text, config, metadata) {
|
|
|
36
105
|
const chunks = [];
|
|
37
106
|
let currentChunk = '';
|
|
38
107
|
let currentTokens = 0;
|
|
108
|
+
// Track line numbers for chunks
|
|
109
|
+
const baseStartLine = metadata?.startLine ?? 1;
|
|
110
|
+
// Track position in original text for accurate line number calculation
|
|
111
|
+
let textPosition = 0;
|
|
112
|
+
let chunkStartPosition = 0; // Character position where current chunk starts
|
|
39
113
|
for (const paragraph of paragraphs) {
|
|
40
114
|
const paragraphTokens = tokenCounter.count(paragraph);
|
|
41
115
|
// Check if paragraph itself exceeds model limit
|
|
@@ -43,25 +117,36 @@ export function chunkByTokens(text, config, metadata) {
|
|
|
43
117
|
throw new Error(`Paragraph exceeds model token limit (${paragraphTokens} > ${modelTokenLimit}). ` +
|
|
44
118
|
'Consider splitting by sentences or reducing content.');
|
|
45
119
|
}
|
|
46
|
-
//
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
120
|
+
// Find this paragraph in the original text
|
|
121
|
+
const paragraphIndex = text.indexOf(paragraph.trim(), textPosition);
|
|
122
|
+
const actualParagraphIndex = paragraphIndex === -1 ? textPosition : paragraphIndex;
|
|
123
|
+
// If paragraph itself exceeds effective target, split it by lines
|
|
124
|
+
if (paragraphTokens > effectiveTarget) {
|
|
125
|
+
const paragraphChunks = chunkLargeParagraphByLines(paragraph, text, actualParagraphIndex, baseStartLine, config, metadata);
|
|
126
|
+
chunks.push(...paragraphChunks);
|
|
127
|
+
textPosition = actualParagraphIndex + paragraph.length;
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
// If adding this paragraph would exceed target, save current chunk and start new one
|
|
131
|
+
const wouldExceedTarget = currentTokens > 0 && currentTokens + paragraphTokens > effectiveTarget;
|
|
132
|
+
if (wouldExceedTarget) {
|
|
133
|
+
chunks.push(createChunkWithLineNumbers(currentChunk, text, chunkStartPosition, baseStartLine, metadata));
|
|
52
134
|
currentChunk = '';
|
|
53
135
|
currentTokens = 0;
|
|
54
136
|
}
|
|
137
|
+
// Mark chunk start position when starting new chunk
|
|
138
|
+
const isNewChunk = currentChunk.length === 0;
|
|
139
|
+
if (isNewChunk) {
|
|
140
|
+
chunkStartPosition = actualParagraphIndex;
|
|
141
|
+
}
|
|
55
142
|
// Add paragraph to current chunk
|
|
56
143
|
currentChunk += (currentChunk.length > 0 ? '\n\n' : '') + paragraph;
|
|
57
144
|
currentTokens += paragraphTokens;
|
|
145
|
+
textPosition = actualParagraphIndex + paragraph.length;
|
|
58
146
|
}
|
|
59
147
|
// Add final chunk
|
|
60
148
|
if (currentChunk.trim().length > 0) {
|
|
61
|
-
chunks.push(
|
|
62
|
-
content: currentChunk.trim(),
|
|
63
|
-
...metadata,
|
|
64
|
-
});
|
|
149
|
+
chunks.push(createChunkWithLineNumbers(currentChunk, text, chunkStartPosition, baseStartLine, metadata));
|
|
65
150
|
}
|
|
66
151
|
return chunks;
|
|
67
152
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk-by-tokens.js","sourceRoot":"","sources":["../../src/chunking/chunk-by-tokens.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEzE
|
|
1
|
+
{"version":3,"file":"chunk-by-tokens.js","sourceRoot":"","sources":["../../src/chunking/chunk-by-tokens.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEzE;;;;;;;;;;;;GAYG;AACH,SAAS,0BAA0B,CACjC,SAAiB,EACjB,IAAY,EACZ,YAAoB,EACpB,aAAqB,EACrB,MAAsB,EACtB,QAAgG;IAEhG,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC;IAChC,MAAM,eAAe,GAAG,wBAAwB,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;IAC/F,MAAM,MAAM,GAAe,EAAE,CAAC;IAE9B,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,SAAS,GAAG,EAAE,CAAC;IACnB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,iBAAiB,GAAG,YAAY,CAAC;IAErC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE5C,8DAA8D;QAC9D,IAAI,eAAe,GAAG,CAAC,IAAI,eAAe,GAAG,UAAU,GAAG,eAAe,EAAE,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,SAAS,EAAE,IAAI,EAAE,iBAAiB,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC;YAErG,SAAS,GAAG,EAAE,CAAC;YACf,eAAe,GAAG,CAAC,CAAC;YACpB,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QAC/E,CAAC;QAED,4BAA4B;QAC5B,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;QACvD,eAAe,IAAI,UAAU,CAAC;IAChC,CAAC;IAED,uBAAuB;IACvB,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,SAAS,EAAE,IAAI,EAAE,iBAAiB,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC;IACvG,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,0BAA0B,CACjC,OAAe,EACf,IAAY,EACZ,kBAA0B,EAC1B,aAAqB,EACrB,QAAgG;IAEhG,MAAM,kBAAkB,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;IACjE,MAAM,gBAAgB,GAAG,CAAC,kBAAkB,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,cAAc,GAAG,aAAa,GAAG,gBAAgB,CAAC;IAExD,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACvD,MAAM,YAAY,GAAG,cAAc,GAAG,YAAY,GAAG,CAAC,CAAC;IAEvD,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;QACvB,GAAG,QAAQ;QACX,SAAS,EAAE,cAAc;QACzB,OAAO,EAAE,YAAY;KACtB,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,wDAAwD;AACxD,MAAM,UAAU,aAAa,CAC3B,IAAY,EACZ,MAAsB,EACtB,QAAgG;IAEhG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC;IACjF,MAAM,eAAe,GAAG,wBAAwB,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;IAEjF,4BAA4B;IAC5B,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,WAAW,IAAI,eAAe,EAAE,CAAC;QACnC,OAAO;YACL;gBACE,OAAO,EAAE,IAAI;gBACb,GAAG,QAAQ;aACZ;SACF,CAAC;IACJ,CAAC;IAED,4BAA4B;IAC5B,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,gCAAgC;IAChC,MAAM,aAAa,GAAG,QAAQ,EAAE,SAAS,IAAI,CAAC,CAAC;IAE/C,uEAAuE;IACvE,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,kBAAkB,GAAG,CAAC,CAAC,CAAC,gDAAgD;IAE5E,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,eAAe,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEtD,gDAAgD;QAChD,IAAI,eAAe,GAAG,eAAe,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,wCAAwC,eAAe,MAAM,eAAe,KAAK;gBAC/E,sDAAsD,CACzD,CAAC;QACJ,CAAC;QAED,2CAA2C;QAC3C,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,YAAY,CAAC,CAAC;QACpE,MAAM,oBAAoB,GAAG,cAAc,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC;QAEnF,kEAAkE;QAClE,IAAI,eAAe,GAAG,eAAe,EAAE,CAAC;YACtC,MAAM,eAAe,GAAG,0BAA0B,CAChD,SAAS,EACT,IAAI,EACJ,oBAAoB,EACpB,aAAa,EACb,MAAM,EACN,QAAQ,CACT,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAChC,YAAY,GAAG,oBAAoB,GAAG,SAAS,CAAC,MAAM,CAAC;YACvD,SAAS;QACX,CAAC;QAED,qFAAqF;QACrF,MAAM,iBAAiB,GAAG,aAAa,GAAG,CAAC,IAAI,aAAa,GAAG,eAAe,GAAG,eAAe,CAAC;QACjG,IAAI,iBAAiB,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,YAAY,EAAE,IAAI,EAAE,kBAAkB,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC;YACzG,YAAY,GAAG,EAAE,CAAC;YAClB,aAAa,GAAG,CAAC,CAAC;QACpB,CAAC;QAED,oDAAoD;QACpD,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC;QAC7C,IAAI,UAAU,EAAE,CAAC;YACf,kBAAkB,GAAG,oBAAoB,CAAC;QAC5C,CAAC;QAED,iCAAiC;QACjC,YAAY,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC;QACpE,aAAa,IAAI,eAAe,CAAC;QACjC,YAAY,GAAG,oBAAoB,GAAG,SAAS,CAAC,MAAM,CAAC;IACzD,CAAC;IAED,kBAAkB;IAClB,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,YAAY,EAAE,IAAI,EAAE,kBAAkB,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC3G,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk-resource.d.ts","sourceRoot":"","sources":["../../src/chunking/chunk-resource.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"chunk-resource.d.ts","sourceRoot":"","sources":["../../src/chunking/chunk-resource.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAe,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAEnF,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAG3E;;;;;GAKG;AACH,MAAM,WAAW,iBAAkB,SAAQ,gBAAgB;IACzD,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,uCAAuC;IACvC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,EAAE,cAAc,GACrB,cAAc,CAqEhB;AA+DD;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAC1B,SAAS,EAAE,QAAQ,EAAE,EACrB,QAAQ,EAAE,iBAAiB,EAC3B,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,cAAc,EAAE,MAAM,GACrB,QAAQ,EAAE,CA6BZ"}
|
|
@@ -19,37 +19,48 @@ import { generateChunkId, generateContentHash } from './utils.js';
|
|
|
19
19
|
*/
|
|
20
20
|
export function chunkResource(resource, config) {
|
|
21
21
|
const rawChunks = [];
|
|
22
|
-
|
|
22
|
+
// Flatten nested heading tree into a sorted list
|
|
23
|
+
const flatHeadings = flattenHeadings(resource.headings);
|
|
24
|
+
if (flatHeadings.length === 0) {
|
|
23
25
|
// No headings - chunk entire content by tokens
|
|
24
|
-
const
|
|
26
|
+
const lines = resource.content.split('\n');
|
|
27
|
+
const chunks = chunkByTokens(resource.content, config, {
|
|
28
|
+
startLine: 1,
|
|
29
|
+
endLine: lines.length,
|
|
30
|
+
});
|
|
25
31
|
rawChunks.push(...chunks);
|
|
26
32
|
}
|
|
27
33
|
else {
|
|
28
34
|
// Extract content between headings
|
|
29
35
|
const lines = resource.content.split('\n');
|
|
30
|
-
for (let i = 0; i <
|
|
31
|
-
const heading =
|
|
36
|
+
for (let i = 0; i < flatHeadings.length; i++) {
|
|
37
|
+
const heading = flatHeadings[i];
|
|
32
38
|
if (!heading)
|
|
33
39
|
continue;
|
|
34
|
-
const nextHeading =
|
|
40
|
+
const nextHeading = flatHeadings[i + 1];
|
|
35
41
|
// Extract content between this heading and next (or end of file)
|
|
36
|
-
|
|
37
|
-
const
|
|
42
|
+
// Note: heading.line is 1-based, but array indices are 0-based
|
|
43
|
+
const headingLine = heading.line ?? 1; // 1-based line number
|
|
44
|
+
const nextHeadingLine = nextHeading?.line ?? (lines.length + 1); // 1-based line number
|
|
45
|
+
// Convert to 0-based array indices for slicing
|
|
46
|
+
const startIndex = headingLine - 1;
|
|
47
|
+
const endIndex = nextHeadingLine - 1;
|
|
38
48
|
const sectionContent = lines
|
|
39
|
-
.slice(
|
|
49
|
+
.slice(startIndex, endIndex)
|
|
40
50
|
.join('\n')
|
|
41
51
|
.trim();
|
|
42
52
|
if (sectionContent.length === 0) {
|
|
43
53
|
continue;
|
|
44
54
|
}
|
|
45
55
|
// Build heading path (hierarchy)
|
|
46
|
-
const headingPath = buildHeadingPath(
|
|
56
|
+
const headingPath = buildHeadingPath(flatHeadings, i);
|
|
47
57
|
// Chunk this section by tokens if needed
|
|
58
|
+
// Pass 1-based line numbers as metadata
|
|
48
59
|
const metadata = {
|
|
49
60
|
headingPath,
|
|
50
61
|
headingLevel: heading.level,
|
|
51
|
-
startLine,
|
|
52
|
-
endLine,
|
|
62
|
+
startLine: headingLine,
|
|
63
|
+
endLine: nextHeadingLine - 1, // Last line of this section
|
|
53
64
|
};
|
|
54
65
|
const sectionChunks = chunkByTokens(sectionContent, config, metadata);
|
|
55
66
|
rawChunks.push(...sectionChunks);
|
|
@@ -65,6 +76,33 @@ export function chunkResource(resource, config) {
|
|
|
65
76
|
};
|
|
66
77
|
return { chunks: rawChunks, stats };
|
|
67
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* Flatten nested heading tree into a flat array
|
|
81
|
+
*
|
|
82
|
+
* Converts hierarchical heading structure (with children) into a flat list
|
|
83
|
+
* sorted by line number, suitable for section extraction.
|
|
84
|
+
*
|
|
85
|
+
* @param headings - Hierarchical heading nodes
|
|
86
|
+
* @returns Flat array of headings sorted by line number
|
|
87
|
+
*/
|
|
88
|
+
function flattenHeadings(headings) {
|
|
89
|
+
const result = [];
|
|
90
|
+
function traverse(nodes) {
|
|
91
|
+
for (const node of nodes) {
|
|
92
|
+
result.push(node);
|
|
93
|
+
if (node.children && node.children.length > 0) {
|
|
94
|
+
traverse(node.children);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
traverse(headings);
|
|
99
|
+
// Sort by line number to ensure sections are in document order
|
|
100
|
+
return result.sort((a, b) => {
|
|
101
|
+
const lineA = a.line ?? 0;
|
|
102
|
+
const lineB = b.line ?? 0;
|
|
103
|
+
return lineA - lineB;
|
|
104
|
+
});
|
|
105
|
+
}
|
|
68
106
|
/**
|
|
69
107
|
* Build heading path from heading hierarchy
|
|
70
108
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk-resource.js","sourceRoot":"","sources":["../../src/chunking/chunk-resource.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAelE;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,aAAa,CAC3B,QAA2B,EAC3B,MAAsB;IAEtB,MAAM,SAAS,GAAe,EAAE,CAAC;IAEjC,
|
|
1
|
+
{"version":3,"file":"chunk-resource.js","sourceRoot":"","sources":["../../src/chunking/chunk-resource.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAelE;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,aAAa,CAC3B,QAA2B,EAC3B,MAAsB;IAEtB,MAAM,SAAS,GAAe,EAAE,CAAC;IAEjC,iDAAiD;IACjD,MAAM,YAAY,GAAG,eAAe,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAExD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,+CAA+C;QAC/C,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE;YACrD,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,KAAK,CAAC,MAAM;SACtB,CAAC,CAAC;QACH,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IAC5B,CAAC;SAAM,CAAC;QACN,mCAAmC;QACnC,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,MAAM,WAAW,GAAG,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAExC,iEAAiE;YACjE,+DAA+D;YAC/D,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,sBAAsB;YAC7D,MAAM,eAAe,GAAG,WAAW,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,sBAAsB;YAEvF,+CAA+C;YAC/C,MAAM,UAAU,GAAG,WAAW,GAAG,CAAC,CAAC;YACnC,MAAM,QAAQ,GAAG,eAAe,GAAG,CAAC,CAAC;YAErC,MAAM,cAAc,GAAG,KAAK;iBACzB,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC;iBAC3B,IAAI,CAAC,IAAI,CAAC;iBACV,IAAI,EAAE,CAAC;YAEV,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChC,SAAS;YACX,CAAC;YAED,iCAAiC;YACjC,MAAM,WAAW,GAAG,gBAAgB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;YAEtD,yCAAyC;YACzC,wCAAwC;YACxC,MAAM,QAAQ,GAAG;gBACf,WAAW;gBACX,YAAY,EAAE,OAAO,CAAC,KAAK;gBAC3B,SAAS,EAAE,WAAW;gBACtB,OAAO,EAAE,eAAe,GAAG,CAAC,EAAE,4BAA4B;aAC3D,CAAC;YAEF,MAAM,aAAa,GAAG,aAAa,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YACtE,SAAS,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/E,MAAM,KAAK,GAAG;QACZ,WAAW,EAAE,SAAS,CAAC,MAAM;QAC7B,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM;QAC5E,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC;QACnC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC;KACpC,CAAC;IAEF,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACtC,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,eAAe,CAAC,QAAuB;IAC9C,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,SAAS,QAAQ,CAAC,KAAoB;QACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClB,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9C,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEnB,+DAA+D;IAC/D,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC1B,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;QAC1B,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;QAC1B,OAAO,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,SAAS,gBAAgB,CACvB,QAAgD,EAChD,YAAoB;IAEpB,MAAM,OAAO,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;IACvC,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAExB,MAAM,IAAI,GAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtC,yCAAyC;IACzC,KAAK,IAAI,CAAC,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC;gBAAE,MAAM,CAAC,oBAAoB;QACtD,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,YAAY,CAC1B,SAAqB,EACrB,QAA2B,EAC3B,UAAsB,EACtB,cAAsB;IAEtB,MAAM,cAAc,GAAe,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;QAC9D,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAErD,OAAO;YACL,OAAO;YACP,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,WAAW;YACX,UAAU,EAAE,CAAC,EAAE,wBAAwB;YACvC,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAyB;YAC1D,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAuB;YACxD,KAAK,EAAE,QAAQ,CAAC,WAAW,CAAC,OAAO,CAAuB;YAC1D,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE;YAClC,cAAc;YACd,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,eAAe,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;YAChF,WAAW,EACT,KAAK,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;SACrF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AACxB,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Abstract RAG interfaces and shared implementations for vibe-agent-toolkit.
|
|
5
5
|
*/
|
|
6
|
-
export type { RAGQuery, RAGChunk, RAGResult, RAGStats, IndexResult, RAGQueryProvider, RAGAdminProvider, EmbeddingProvider, TokenCounter, } from './interfaces/index.js';
|
|
6
|
+
export type { RAGQuery, RAGChunk, RAGResult, RAGStats, IndexResult, IndexProgress, ProgressCallback, RAGQueryProvider, RAGAdminProvider, EmbeddingProvider, TokenCounter, } from './interfaces/index.js';
|
|
7
7
|
export { CoreRAGChunkSchema, type CoreRAGChunk, DefaultRAGMetadataSchema, type DefaultRAGMetadata, RAGChunkSchema, RAGQuerySchema, RAGResultSchema, RAGStatsSchema, IndexResultSchema, createCustomRAGChunkSchema, } from './schemas/index.js';
|
|
8
8
|
export { RAGChunkJsonSchema, RAGQueryJsonSchema, RAGResultJsonSchema, RAGStatsJsonSchema, IndexResultJsonSchema, jsonSchemas, } from './schemas/json-schema.js';
|
|
9
9
|
export { ApproximateTokenCounter, FastTokenCounter } from './token-counters/index.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,GACb,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EACL,kBAAkB,EAClB,KAAK,YAAY,EACjB,wBAAwB,EACxB,KAAK,kBAAkB,EACvB,cAAc,EACd,cAAc,EACd,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,0BAA0B,GAC3B,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,EACnB,kBAAkB,EAClB,qBAAqB,EACrB,WAAW,GACZ,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAGtF,OAAO,EACL,uBAAuB,EACvB,6BAA6B,EAC7B,KAAK,qBAAqB,EAC1B,KAAK,2BAA2B,GACjC,MAAM,gCAAgC,CAAC;AAGxC,OAAO,EACL,wBAAwB,EACxB,aAAa,EACb,aAAa,EACb,YAAY,EACZ,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,KAAK,iBAAiB,EACtB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,QAAQ,GACd,MAAM,qBAAqB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,aAAa,EACb,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,GACb,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EACL,kBAAkB,EAClB,KAAK,YAAY,EACjB,wBAAwB,EACxB,KAAK,kBAAkB,EACvB,cAAc,EACd,cAAc,EACd,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,0BAA0B,GAC3B,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,EACnB,kBAAkB,EAClB,qBAAqB,EACrB,WAAW,GACZ,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAGtF,OAAO,EACL,uBAAuB,EACvB,6BAA6B,EAC7B,KAAK,qBAAqB,EAC1B,KAAK,2BAA2B,GACjC,MAAM,gCAAgC,CAAC;AAGxC,OAAO,EACL,wBAAwB,EACxB,aAAa,EACb,aAAa,EACb,YAAY,EACZ,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,KAAK,iBAAiB,EACtB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,QAAQ,GACd,MAAM,qBAAqB,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAiBH,gBAAgB;AAChB,OAAO,EACL,kBAAkB,EAElB,wBAAwB,EAExB,cAAc,EACd,cAAc,EACd,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,0BAA0B,GAC3B,MAAM,oBAAoB,CAAC;AAE5B,eAAe;AACf,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,EACnB,kBAAkB,EAClB,qBAAqB,EACrB,WAAW,GACZ,MAAM,0BAA0B,CAAC;AAElC,iBAAiB;AACjB,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAEtF,sBAAsB;AACtB,OAAO,EACL,uBAAuB,EACvB,6BAA6B,GAG9B,MAAM,gCAAgC,CAAC;AAExC,WAAW;AACX,OAAO,EACL,wBAAwB,EACxB,aAAa,EACb,aAAa,EACb,YAAY,EACZ,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,GAKjB,MAAM,qBAAqB,CAAC"}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
export type { CoreRAGChunk } from '../schemas/core-chunk.js';
|
|
7
7
|
export type { DefaultRAGMetadata } from '../schemas/default-metadata.js';
|
|
8
8
|
export type { RAGChunk } from './chunk.js';
|
|
9
|
-
export type { RAGQuery, RAGResult, RAGStats, IndexResult, RAGQueryProvider, RAGAdminProvider, } from './provider.js';
|
|
9
|
+
export type { RAGQuery, RAGResult, RAGStats, IndexResult, IndexProgress, ProgressCallback, RAGQueryProvider, RAGAdminProvider, } from './provider.js';
|
|
10
10
|
export type { EmbeddingProvider } from './embedding.js';
|
|
11
11
|
export type { TokenCounter } from './token-counter.js';
|
|
12
12
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,YAAY,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACzE,YAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,YAAY,EACV,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,eAAe,CAAC;AAEvB,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACxD,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,YAAY,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACzE,YAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,YAAY,EACV,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,aAAa,EACb,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,eAAe,CAAC;AAEvB,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACxD,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC"}
|
|
@@ -71,6 +71,38 @@ export interface RAGStats {
|
|
|
71
71
|
embeddingModel: string;
|
|
72
72
|
lastIndexed: Date;
|
|
73
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Progress update during indexing operation
|
|
76
|
+
*/
|
|
77
|
+
export interface IndexProgress {
|
|
78
|
+
/** Current resource being processed (1-indexed) */
|
|
79
|
+
current: number;
|
|
80
|
+
/** Total resources to process */
|
|
81
|
+
total: number;
|
|
82
|
+
/** Resources indexed so far */
|
|
83
|
+
resourcesIndexed: number;
|
|
84
|
+
/** Resources skipped so far */
|
|
85
|
+
resourcesSkipped: number;
|
|
86
|
+
/** Resources updated so far */
|
|
87
|
+
resourcesUpdated: number;
|
|
88
|
+
/** Chunks created so far */
|
|
89
|
+
chunksCreated: number;
|
|
90
|
+
/** Elapsed time in milliseconds */
|
|
91
|
+
elapsedMs: number;
|
|
92
|
+
/** Estimated time remaining in milliseconds (null if not enough data) */
|
|
93
|
+
estimatedRemainingMs: number | null;
|
|
94
|
+
/** Current resource ID */
|
|
95
|
+
resourceId: string;
|
|
96
|
+
/** Errors encountered so far */
|
|
97
|
+
errors: Array<{
|
|
98
|
+
resourceId: string;
|
|
99
|
+
error: string;
|
|
100
|
+
}>;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Progress callback for indexing operations
|
|
104
|
+
*/
|
|
105
|
+
export type ProgressCallback = (progress: IndexProgress) => void;
|
|
74
106
|
/**
|
|
75
107
|
* Result from indexing operation
|
|
76
108
|
*/
|
|
@@ -115,8 +147,11 @@ export interface RAGAdminProvider<TMetadata extends Record<string, unknown> = De
|
|
|
115
147
|
* - Detects changes via content hash
|
|
116
148
|
* - Deletes old chunks for changed resources
|
|
117
149
|
* - Skips unchanged resources
|
|
150
|
+
*
|
|
151
|
+
* @param resources - Resources to index
|
|
152
|
+
* @param onProgress - Optional callback for progress updates (called after each resource)
|
|
118
153
|
*/
|
|
119
|
-
indexResources(resources: ResourceMetadata[]): Promise<IndexResult>;
|
|
154
|
+
indexResources(resources: ResourceMetadata[], onProgress?: ProgressCallback): Promise<IndexResult>;
|
|
120
155
|
/**
|
|
121
156
|
* Update a specific resource
|
|
122
157
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../src/interfaces/provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAEtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAEzE;;;;GAIG;AACH,MAAM,WAAW,QAAQ,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,kBAAkB;IACtF,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;IAEb,8CAA8C;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,cAAc;IACd,OAAO,CAAC,EAAE;QACR,+BAA+B;QAC/B,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAC/B,2BAA2B;QAC3B,SAAS,CAAC,EAAE;YAAE,KAAK,EAAE,IAAI,CAAC;YAAC,GAAG,EAAE,IAAI,CAAA;SAAE,CAAC;QAEvC,sDAAsD;QACtD,QAAQ,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;KAC/B,CAAC;IAEF,qDAAqD;IACrD,YAAY,CAAC,EAAE;QACb,OAAO,EAAE,OAAO,CAAC;QACjB,iEAAiE;QACjE,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACH;AAED;;;;;GAKG;AACH,YAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C;;;;GAIG;AACH,MAAM,WAAW,SAAS,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,kBAAkB;IACvF,0CAA0C;IAC1C,MAAM,EAAE,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;IAExC,wBAAwB;IACxB,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,SAAS,CAAC,EAAE;YACV,KAAK,EAAE,MAAM,CAAC;YACd,UAAU,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,IAAI,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvD;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,kBAAkB;IAC9F;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;IAEjE;;OAEG;IACH,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC;CAC/B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,kBAAkB,CAC9F,SAAQ,gBAAgB,CAAC,SAAS,CAAC;IACnC
|
|
1
|
+
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../src/interfaces/provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAEtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAEzE;;;;GAIG;AACH,MAAM,WAAW,QAAQ,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,kBAAkB;IACtF,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;IAEb,8CAA8C;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,cAAc;IACd,OAAO,CAAC,EAAE;QACR,+BAA+B;QAC/B,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAC/B,2BAA2B;QAC3B,SAAS,CAAC,EAAE;YAAE,KAAK,EAAE,IAAI,CAAC;YAAC,GAAG,EAAE,IAAI,CAAA;SAAE,CAAC;QAEvC,sDAAsD;QACtD,QAAQ,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;KAC/B,CAAC;IAEF,qDAAqD;IACrD,YAAY,CAAC,EAAE;QACb,OAAO,EAAE,OAAO,CAAC;QACjB,iEAAiE;QACjE,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACH;AAED;;;;;GAKG;AACH,YAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C;;;;GAIG;AACH,MAAM,WAAW,SAAS,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,kBAAkB;IACvF,0CAA0C;IAC1C,MAAM,EAAE,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;IAExC,wBAAwB;IACxB,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,SAAS,CAAC,EAAE;YACV,KAAK,EAAE,MAAM,CAAC;YACd,UAAU,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,IAAI,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,mDAAmD;IACnD,OAAO,EAAE,MAAM,CAAC;IAChB,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,+BAA+B;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,+BAA+B;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,4BAA4B;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,0BAA0B;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,MAAM,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACtD;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;AAEjE;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvD;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,kBAAkB;IAC9F;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;IAEjE;;OAEG;IACH,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC;CAC/B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,kBAAkB,CAC9F,SAAQ,gBAAgB,CAAC,SAAS,CAAC;IACnC;;;;;;;;OAQG;IACH,cAAc,CAAC,SAAS,EAAE,gBAAgB,EAAE,EAAE,UAAU,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEnG;;OAEG;IACH,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElD;;OAEG;IACH,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElD;;OAEG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;OAEG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB"}
|
package/dist/schemas/chunk.d.ts
CHANGED
|
@@ -22,6 +22,8 @@ export declare const RAGChunkSchema: z.ZodObject<{
|
|
|
22
22
|
embeddedAt: z.ZodDate;
|
|
23
23
|
previousChunkId: z.ZodOptional<z.ZodString>;
|
|
24
24
|
nextChunkId: z.ZodOptional<z.ZodString>;
|
|
25
|
+
_distance: z.ZodOptional<z.ZodNumber>;
|
|
26
|
+
score: z.ZodOptional<z.ZodNumber>;
|
|
25
27
|
} & {
|
|
26
28
|
filePath: z.ZodString;
|
|
27
29
|
tags: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
@@ -44,6 +46,8 @@ export declare const RAGChunkSchema: z.ZodObject<{
|
|
|
44
46
|
type?: string | undefined;
|
|
45
47
|
previousChunkId?: string | undefined;
|
|
46
48
|
nextChunkId?: string | undefined;
|
|
49
|
+
_distance?: number | undefined;
|
|
50
|
+
score?: number | undefined;
|
|
47
51
|
tags?: string[] | undefined;
|
|
48
52
|
title?: string | undefined;
|
|
49
53
|
headingPath?: string | undefined;
|
|
@@ -63,6 +67,8 @@ export declare const RAGChunkSchema: z.ZodObject<{
|
|
|
63
67
|
type?: string | undefined;
|
|
64
68
|
previousChunkId?: string | undefined;
|
|
65
69
|
nextChunkId?: string | undefined;
|
|
70
|
+
_distance?: number | undefined;
|
|
71
|
+
score?: number | undefined;
|
|
66
72
|
tags?: string[] | undefined;
|
|
67
73
|
title?: string | undefined;
|
|
68
74
|
headingPath?: string | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../src/schemas/chunk.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAGrD;;;;;GAKG;AACH,eAAO,MAAM,cAAc
|
|
1
|
+
{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../src/schemas/chunk.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAGrD;;;;;GAKG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAAqD,CAAC;AAEjF;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,0BAA0B,CAAC,CAAC,SAAS,CAAC,CAAC,WAAW,EAChE,cAAc,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAC7B,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,OAAO,kBAAkB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAE3E"}
|
|
@@ -21,6 +21,8 @@ export declare const CoreRAGChunkSchema: z.ZodObject<{
|
|
|
21
21
|
embeddedAt: z.ZodDate;
|
|
22
22
|
previousChunkId: z.ZodOptional<z.ZodString>;
|
|
23
23
|
nextChunkId: z.ZodOptional<z.ZodString>;
|
|
24
|
+
_distance: z.ZodOptional<z.ZodNumber>;
|
|
25
|
+
score: z.ZodOptional<z.ZodNumber>;
|
|
24
26
|
}, "strip", z.ZodTypeAny, {
|
|
25
27
|
chunkId: string;
|
|
26
28
|
resourceId: string;
|
|
@@ -32,6 +34,8 @@ export declare const CoreRAGChunkSchema: z.ZodObject<{
|
|
|
32
34
|
embeddedAt: Date;
|
|
33
35
|
previousChunkId?: string | undefined;
|
|
34
36
|
nextChunkId?: string | undefined;
|
|
37
|
+
_distance?: number | undefined;
|
|
38
|
+
score?: number | undefined;
|
|
35
39
|
}, {
|
|
36
40
|
chunkId: string;
|
|
37
41
|
resourceId: string;
|
|
@@ -43,6 +47,8 @@ export declare const CoreRAGChunkSchema: z.ZodObject<{
|
|
|
43
47
|
embeddedAt: Date;
|
|
44
48
|
previousChunkId?: string | undefined;
|
|
45
49
|
nextChunkId?: string | undefined;
|
|
50
|
+
_distance?: number | undefined;
|
|
51
|
+
score?: number | undefined;
|
|
46
52
|
}>;
|
|
47
53
|
/**
|
|
48
54
|
* CoreRAGChunk TypeScript type (inferred from schema)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"core-chunk.d.ts","sourceRoot":"","sources":["../../src/schemas/core-chunk.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB
|
|
1
|
+
{"version":3,"file":"core-chunk.d.ts","sourceRoot":"","sources":["../../src/schemas/core-chunk.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsB7B,CAAC;AAEH;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC"}
|
|
@@ -25,5 +25,8 @@ export const CoreRAGChunkSchema = z.object({
|
|
|
25
25
|
// Context linking
|
|
26
26
|
previousChunkId: z.string().optional().describe('Previous chunk in document (for context expansion)'),
|
|
27
27
|
nextChunkId: z.string().optional().describe('Next chunk in document (for context expansion)'),
|
|
28
|
+
// Search result metrics (optional, only present in query results)
|
|
29
|
+
_distance: z.number().optional().describe('Raw distance metric from vector search (lower is more similar)'),
|
|
30
|
+
score: z.number().optional().describe('Computed similarity score 0-1 (higher is more similar)'),
|
|
28
31
|
});
|
|
29
32
|
//# sourceMappingURL=core-chunk.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"core-chunk.js","sourceRoot":"","sources":["../../src/schemas/core-chunk.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,WAAW;IACX,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gCAAgC,CAAC;IAC9D,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IAE3E,UAAU;IACV,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACrD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC;IACxE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wCAAwC,CAAC;IAEzE,UAAU;IACV,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAC3D,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gEAAgE,CAAC;IACrG,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IAE7D,kBAAkB;IAClB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oDAAoD,CAAC;IACrG,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gDAAgD,CAAC;
|
|
1
|
+
{"version":3,"file":"core-chunk.js","sourceRoot":"","sources":["../../src/schemas/core-chunk.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,WAAW;IACX,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gCAAgC,CAAC;IAC9D,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IAE3E,UAAU;IACV,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACrD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC;IACxE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wCAAwC,CAAC;IAEzE,UAAU;IACV,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAC3D,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gEAAgE,CAAC;IACrG,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IAE7D,kBAAkB;IAClB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oDAAoD,CAAC;IACrG,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gDAAgD,CAAC;IAE7F,kEAAkE;IAClE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gEAAgE,CAAC;IAC3G,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wDAAwD,CAAC;CAChG,CAAC,CAAC"}
|
package/dist/schemas/query.d.ts
CHANGED
|
@@ -120,6 +120,8 @@ export declare const RAGResultSchema: z.ZodObject<{
|
|
|
120
120
|
embeddedAt: z.ZodDate;
|
|
121
121
|
previousChunkId: z.ZodOptional<z.ZodString>;
|
|
122
122
|
nextChunkId: z.ZodOptional<z.ZodString>;
|
|
123
|
+
_distance: z.ZodOptional<z.ZodNumber>;
|
|
124
|
+
score: z.ZodOptional<z.ZodNumber>;
|
|
123
125
|
} & {
|
|
124
126
|
filePath: z.ZodString;
|
|
125
127
|
tags: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
@@ -142,6 +144,8 @@ export declare const RAGResultSchema: z.ZodObject<{
|
|
|
142
144
|
type?: string | undefined;
|
|
143
145
|
previousChunkId?: string | undefined;
|
|
144
146
|
nextChunkId?: string | undefined;
|
|
147
|
+
_distance?: number | undefined;
|
|
148
|
+
score?: number | undefined;
|
|
145
149
|
tags?: string[] | undefined;
|
|
146
150
|
title?: string | undefined;
|
|
147
151
|
headingPath?: string | undefined;
|
|
@@ -161,6 +165,8 @@ export declare const RAGResultSchema: z.ZodObject<{
|
|
|
161
165
|
type?: string | undefined;
|
|
162
166
|
previousChunkId?: string | undefined;
|
|
163
167
|
nextChunkId?: string | undefined;
|
|
168
|
+
_distance?: number | undefined;
|
|
169
|
+
score?: number | undefined;
|
|
164
170
|
tags?: string[] | undefined;
|
|
165
171
|
title?: string | undefined;
|
|
166
172
|
headingPath?: string | undefined;
|
|
@@ -211,6 +217,8 @@ export declare const RAGResultSchema: z.ZodObject<{
|
|
|
211
217
|
type?: string | undefined;
|
|
212
218
|
previousChunkId?: string | undefined;
|
|
213
219
|
nextChunkId?: string | undefined;
|
|
220
|
+
_distance?: number | undefined;
|
|
221
|
+
score?: number | undefined;
|
|
214
222
|
tags?: string[] | undefined;
|
|
215
223
|
title?: string | undefined;
|
|
216
224
|
headingPath?: string | undefined;
|
|
@@ -240,6 +248,8 @@ export declare const RAGResultSchema: z.ZodObject<{
|
|
|
240
248
|
type?: string | undefined;
|
|
241
249
|
previousChunkId?: string | undefined;
|
|
242
250
|
nextChunkId?: string | undefined;
|
|
251
|
+
_distance?: number | undefined;
|
|
252
|
+
score?: number | undefined;
|
|
243
253
|
tags?: string[] | undefined;
|
|
244
254
|
title?: string | undefined;
|
|
245
255
|
headingPath?: string | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../../src/schemas/query.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB;;;;GAIG;AACH,eAAO,MAAM,cAAc;IACzB,wBAAwB;;IAGxB,8CAA8C;;IAG9C,uBAAuB;;QAErB,+BAA+B;;QAE/B,qBAAqB;;QAErB,8BAA8B;;QAE9B,6BAA6B;;QAE7B,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAO7B,kCAAkC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKlC,CAAC;AAEH;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD;;;;GAIG;AACH,eAAO,MAAM,eAAe;IAC1B,0CAA0C
|
|
1
|
+
{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../../src/schemas/query.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB;;;;GAIG;AACH,eAAO,MAAM,cAAc;IACzB,wBAAwB;;IAGxB,8CAA8C;;IAG9C,uBAAuB;;QAErB,+BAA+B;;QAE/B,qBAAqB;;QAErB,8BAA8B;;QAE9B,6BAA6B;;QAE7B,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAO7B,kCAAkC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKlC,CAAC;AAEH;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD;;;;GAIG;AACH,eAAO,MAAM,eAAe;IAC1B,0CAA0C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAG1C,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EASxB,CAAC;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vibe-agent-toolkit/rag",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.9",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Abstract RAG (Retrieval-Augmented Generation) interfaces and shared implementations",
|
|
6
6
|
"keywords": [
|
|
@@ -31,8 +31,8 @@
|
|
|
31
31
|
"typecheck": "tsc --noEmit"
|
|
32
32
|
},
|
|
33
33
|
"dependencies": {
|
|
34
|
-
"@vibe-agent-toolkit/resources": "0.1.
|
|
35
|
-
"@vibe-agent-toolkit/utils": "0.1.
|
|
34
|
+
"@vibe-agent-toolkit/resources": "0.1.9",
|
|
35
|
+
"@vibe-agent-toolkit/utils": "0.1.9",
|
|
36
36
|
"@xenova/transformers": "^2.17.0",
|
|
37
37
|
"gpt-tokenizer": "^2.6.0",
|
|
38
38
|
"zod": "^3.22.0",
|