@cogitator-ai/rag 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunkers/recursive-chunker.d.ts +2 -2
- package/dist/chunkers/recursive-chunker.d.ts.map +1 -1
- package/dist/chunkers/recursive-chunker.js +49 -88
- package/dist/chunkers/recursive-chunker.js.map +1 -1
- package/dist/chunkers/semantic-chunker.d.ts +2 -0
- package/dist/chunkers/semantic-chunker.d.ts.map +1 -1
- package/dist/chunkers/semantic-chunker.js +68 -37
- package/dist/chunkers/semantic-chunker.js.map +1 -1
- package/dist/loaders/csv-loader.d.ts.map +1 -1
- package/dist/loaders/csv-loader.js +3 -0
- package/dist/loaders/csv-loader.js.map +1 -1
- package/dist/loaders/json-loader.d.ts +1 -0
- package/dist/loaders/json-loader.d.ts.map +1 -1
- package/dist/loaders/json-loader.js +15 -3
- package/dist/loaders/json-loader.js.map +1 -1
- package/dist/loaders/markdown-loader.js +1 -1
- package/dist/loaders/markdown-loader.js.map +1 -1
- package/dist/loaders/pdf-loader.d.ts +1 -0
- package/dist/loaders/pdf-loader.d.ts.map +1 -1
- package/dist/loaders/pdf-loader.js +22 -3
- package/dist/loaders/pdf-loader.js.map +1 -1
- package/dist/loaders/web-loader.d.ts +5 -0
- package/dist/loaders/web-loader.d.ts.map +1 -1
- package/dist/loaders/web-loader.js +127 -8
- package/dist/loaders/web-loader.js.map +1 -1
- package/dist/rag-builder.d.ts.map +1 -1
- package/dist/rag-builder.js +3 -1
- package/dist/rag-builder.js.map +1 -1
- package/dist/rag-pipeline.d.ts +1 -1
- package/dist/rag-pipeline.d.ts.map +1 -1
- package/dist/rag-pipeline.js +5 -6
- package/dist/rag-pipeline.js.map +1 -1
- package/dist/rerankers/cohere-reranker.d.ts +2 -2
- package/dist/rerankers/cohere-reranker.d.ts.map +1 -1
- package/dist/rerankers/cohere-reranker.js +5 -0
- package/dist/rerankers/cohere-reranker.js.map +1 -1
- package/dist/rerankers/llm-reranker.d.ts +1 -1
- package/dist/rerankers/llm-reranker.d.ts.map +1 -1
- package/dist/rerankers/llm-reranker.js +26 -13
- package/dist/rerankers/llm-reranker.js.map +1 -1
- package/dist/retrievers/create-retriever.d.ts +1 -0
- package/dist/retrievers/create-retriever.d.ts.map +1 -1
- package/dist/retrievers/create-retriever.js +5 -0
- package/dist/retrievers/create-retriever.js.map +1 -1
- package/dist/retrievers/mmr-retriever.d.ts.map +1 -1
- package/dist/retrievers/mmr-retriever.js +11 -7
- package/dist/retrievers/mmr-retriever.js.map +1 -1
- package/dist/retrievers/multi-query-retriever.d.ts +2 -0
- package/dist/retrievers/multi-query-retriever.d.ts.map +1 -1
- package/dist/retrievers/multi-query-retriever.js +10 -5
- package/dist/retrievers/multi-query-retriever.js.map +1 -1
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +16 -2
- package/dist/schema.js.map +1 -1
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +6 -4
- package/dist/tools.js.map +1 -1
- package/package.json +4 -4
|
@@ -10,9 +10,9 @@ export declare class RecursiveChunker implements Chunker {
|
|
|
10
10
|
private readonly separators;
|
|
11
11
|
constructor(options: RecursiveChunkerOptions);
|
|
12
12
|
chunk(text: string, documentId: string): DocumentChunk[];
|
|
13
|
+
private mapPiecePositions;
|
|
14
|
+
private mergePieces;
|
|
13
15
|
private splitText;
|
|
14
16
|
private charSplit;
|
|
15
|
-
private mergeWithOverlap;
|
|
16
|
-
private mergeWithOverlapSimple;
|
|
17
17
|
}
|
|
18
18
|
//# sourceMappingURL=recursive-chunker.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recursive-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;gBAE1B,OAAO,EAAE,uBAAuB;IAe5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;
|
|
1
|
+
{"version":3,"file":"recursive-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;gBAE1B,OAAO,EAAE,uBAAuB;IAe5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;IAUxD,OAAO,CAAC,iBAAiB;IAiBzB,OAAO,CAAC,WAAW;IA8CnB,OAAO,CAAC,SAAS;IA+CjB,OAAO,CAAC,SAAS;CAOlB"}
|
|
@@ -21,8 +21,55 @@ export class RecursiveChunker {
|
|
|
21
21
|
chunk(text, documentId) {
|
|
22
22
|
if (text.length === 0)
|
|
23
23
|
return [];
|
|
24
|
-
const
|
|
25
|
-
|
|
24
|
+
const pieces = this.splitText(text, 0);
|
|
25
|
+
const piecePositions = this.mapPiecePositions(pieces, text);
|
|
26
|
+
return this.mergePieces(pieces, piecePositions, text, documentId);
|
|
27
|
+
}
|
|
28
|
+
mapPiecePositions(pieces, originalText) {
|
|
29
|
+
const positions = [];
|
|
30
|
+
let searchFrom = 0;
|
|
31
|
+
for (const piece of pieces) {
|
|
32
|
+
const idx = originalText.indexOf(piece, searchFrom);
|
|
33
|
+
const start = idx >= 0 ? idx : searchFrom;
|
|
34
|
+
positions.push({ start, end: start + piece.length });
|
|
35
|
+
searchFrom = start + piece.length;
|
|
36
|
+
}
|
|
37
|
+
return positions;
|
|
38
|
+
}
|
|
39
|
+
mergePieces(pieces, positions, originalText, documentId) {
|
|
40
|
+
const chunks = [];
|
|
41
|
+
let order = 0;
|
|
42
|
+
let i = 0;
|
|
43
|
+
while (i < pieces.length) {
|
|
44
|
+
const chunkStart = positions[i].start;
|
|
45
|
+
let j = i + 1;
|
|
46
|
+
while (j < pieces.length && positions[j].end - chunkStart <= this.chunkSize) {
|
|
47
|
+
j++;
|
|
48
|
+
}
|
|
49
|
+
const chunkEnd = positions[j - 1].end;
|
|
50
|
+
chunks.push({
|
|
51
|
+
id: nanoid(),
|
|
52
|
+
documentId,
|
|
53
|
+
content: originalText.slice(chunkStart, chunkEnd),
|
|
54
|
+
startOffset: chunkStart,
|
|
55
|
+
endOffset: chunkEnd,
|
|
56
|
+
order: order++,
|
|
57
|
+
});
|
|
58
|
+
if (j >= pieces.length)
|
|
59
|
+
break;
|
|
60
|
+
if (this.chunkOverlap > 0) {
|
|
61
|
+
const overlapTarget = chunkEnd - this.chunkOverlap;
|
|
62
|
+
let k = j;
|
|
63
|
+
while (k > i + 1 && positions[k - 1].start >= overlapTarget) {
|
|
64
|
+
k--;
|
|
65
|
+
}
|
|
66
|
+
i = Math.max(k, i + 1);
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
i = j;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return chunks;
|
|
26
73
|
}
|
|
27
74
|
splitText(text, separatorIndex) {
|
|
28
75
|
if (text.length <= this.chunkSize)
|
|
@@ -71,91 +118,5 @@ export class RecursiveChunker {
|
|
|
71
118
|
}
|
|
72
119
|
return result;
|
|
73
120
|
}
|
|
74
|
-
mergeWithOverlap(texts, documentId) {
|
|
75
|
-
if (this.chunkOverlap === 0) {
|
|
76
|
-
let offset = 0;
|
|
77
|
-
return texts.map((content, i) => {
|
|
78
|
-
const chunk = {
|
|
79
|
-
id: nanoid(),
|
|
80
|
-
documentId,
|
|
81
|
-
content,
|
|
82
|
-
startOffset: offset,
|
|
83
|
-
endOffset: offset + content.length,
|
|
84
|
-
order: i,
|
|
85
|
-
};
|
|
86
|
-
offset += content.length;
|
|
87
|
-
return chunk;
|
|
88
|
-
});
|
|
89
|
-
}
|
|
90
|
-
const joined = texts.join('');
|
|
91
|
-
const chunks = [];
|
|
92
|
-
let order = 0;
|
|
93
|
-
let textIndex = 0;
|
|
94
|
-
let i = 0;
|
|
95
|
-
while (i < texts.length) {
|
|
96
|
-
let content = texts[i];
|
|
97
|
-
let consumed = 1;
|
|
98
|
-
while (i + consumed < texts.length) {
|
|
99
|
-
const next = content + texts[i + consumed];
|
|
100
|
-
if (next.length <= this.chunkSize) {
|
|
101
|
-
content = next;
|
|
102
|
-
consumed++;
|
|
103
|
-
}
|
|
104
|
-
else {
|
|
105
|
-
break;
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
const startOffset = joined.indexOf(content, textIndex > 0 ? Math.max(0, textIndex - this.chunkOverlap) : 0);
|
|
109
|
-
chunks.push({
|
|
110
|
-
id: nanoid(),
|
|
111
|
-
documentId,
|
|
112
|
-
content,
|
|
113
|
-
startOffset: startOffset >= 0 ? startOffset : textIndex,
|
|
114
|
-
endOffset: (startOffset >= 0 ? startOffset : textIndex) + content.length,
|
|
115
|
-
order: order++,
|
|
116
|
-
});
|
|
117
|
-
textIndex += content.length;
|
|
118
|
-
const overlapChars = Math.min(this.chunkOverlap, content.length);
|
|
119
|
-
const overlapText = content.slice(content.length - overlapChars);
|
|
120
|
-
i += consumed;
|
|
121
|
-
if (i < texts.length) {
|
|
122
|
-
const remaining = texts.slice(i);
|
|
123
|
-
const withOverlap = overlapText + remaining.join('');
|
|
124
|
-
if (withOverlap.length <= this.chunkSize) {
|
|
125
|
-
chunks.push({
|
|
126
|
-
id: nanoid(),
|
|
127
|
-
documentId,
|
|
128
|
-
content: withOverlap,
|
|
129
|
-
startOffset: textIndex - overlapChars,
|
|
130
|
-
endOffset: textIndex - overlapChars + withOverlap.length,
|
|
131
|
-
order: order++,
|
|
132
|
-
});
|
|
133
|
-
break;
|
|
134
|
-
}
|
|
135
|
-
const reSplit = this.splitText(withOverlap, 0);
|
|
136
|
-
const subChunks = this.mergeWithOverlapSimple(reSplit, documentId, textIndex - overlapChars, order);
|
|
137
|
-
chunks.push(...subChunks);
|
|
138
|
-
break;
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
return chunks;
|
|
142
|
-
}
|
|
143
|
-
mergeWithOverlapSimple(texts, documentId, baseOffset, startOrder) {
|
|
144
|
-
const chunks = [];
|
|
145
|
-
let offset = baseOffset;
|
|
146
|
-
let order = startOrder;
|
|
147
|
-
for (const content of texts) {
|
|
148
|
-
chunks.push({
|
|
149
|
-
id: nanoid(),
|
|
150
|
-
documentId,
|
|
151
|
-
content,
|
|
152
|
-
startOffset: offset,
|
|
153
|
-
endOffset: offset + content.length,
|
|
154
|
-
order: order++,
|
|
155
|
-
});
|
|
156
|
-
offset += content.length;
|
|
157
|
-
}
|
|
158
|
-
return chunks;
|
|
159
|
-
}
|
|
160
121
|
}
|
|
161
122
|
//# sourceMappingURL=recursive-chunker.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,
|
|
1
|
+
{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEvC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE5D,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpE,CAAC;IAEO,iBAAiB,CACvB,MAAgB,EAChB,YAAoB;QAEpB,MAAM,SAAS,GAA0C,EAAE,CAAC;QAC5D,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;YACpD,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;YAC1C,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACrD,UAAU,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QACpC,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,WAAW,CACjB,MAAgB,EAChB,SAAgD,EAChD,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAEtC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC5E,CAAC,EAAE,CAAC;YACN,CAAC;YAED,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEtC,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC;gBACjD,WAAW,EAAE,UAAU;gBACvB,SAAS,EAAE,QAAQ;gBACnB,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM;gBAAE,MAAM;YAE9B,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,aAAa,GAAG,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;gBACnD,IAAI,CAAC,GAAG,CAAC,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;oBAC5D,CAAC,EAAE,CAAC;gBACN,CAAC;gBACD,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,CAAC,GAAG,CAAC,CAAC;YACR,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,cAAsB;QACpD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAEjD,IAAI,cAAc,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAElD,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEhC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAE9D,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACvC,OAAO,GAAG,SAAS,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,IAAI,OAAO;oBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAElC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;oBAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;oBACzB,OAAO,GAAG,EAAE,CAAC;gBACf,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
|
@@ -13,7 +13,9 @@ export declare class SemanticChunker implements AsyncChunker {
|
|
|
13
13
|
constructor(options: SemanticChunkerOptions);
|
|
14
14
|
chunk(text: string, documentId: string): Promise<DocumentChunk[]>;
|
|
15
15
|
private mergeSmallGroups;
|
|
16
|
+
private findSentenceOffset;
|
|
16
17
|
private buildChunks;
|
|
17
18
|
private splitLargeGroup;
|
|
19
|
+
private emitSentenceGroup;
|
|
18
20
|
}
|
|
19
21
|
//# sourceMappingURL=semantic-chunker.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;
|
|
1
|
+
{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AA8BD,qBAAa,eAAgB,YAAW,YAAY;IAClD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,sBAAsB;IAqBrC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAkCvE,OAAO,CAAC,gBAAgB;IAqCxB,OAAO,CAAC,kBAAkB;IAK1B,OAAO,CAAC,WAAW;IA4CnB,OAAO,CAAC,eAAe;IA2EvB,OAAO,CAAC,iBAAiB;CAqB1B"}
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { nanoid } from 'nanoid';
|
|
2
2
|
function cosineSimilarity(a, b) {
|
|
3
|
-
|
|
3
|
+
if (a.length !== b.length) {
|
|
4
|
+
throw new Error(`Embedding dimension mismatch: ${a.length} vs ${b.length}. ` +
|
|
5
|
+
'Ensure all embeddings come from the same model.');
|
|
6
|
+
}
|
|
4
7
|
let dot = 0;
|
|
5
8
|
let normA = 0;
|
|
6
9
|
let normB = 0;
|
|
7
|
-
for (let i = 0; i <
|
|
10
|
+
for (let i = 0; i < a.length; i++) {
|
|
8
11
|
dot += a[i] * b[i];
|
|
9
12
|
normA += a[i] * a[i];
|
|
10
13
|
normB += b[i] * b[i];
|
|
@@ -25,10 +28,19 @@ export class SemanticChunker {
|
|
|
25
28
|
minChunkSize;
|
|
26
29
|
maxChunkSize;
|
|
27
30
|
constructor(options) {
|
|
31
|
+
if (options.breakpointThreshold !== undefined &&
|
|
32
|
+
(options.breakpointThreshold < 0 || options.breakpointThreshold > 1)) {
|
|
33
|
+
throw new Error('breakpointThreshold must be between 0 and 1');
|
|
34
|
+
}
|
|
35
|
+
const maxChunkSize = options.maxChunkSize ?? 2000;
|
|
36
|
+
const minChunkSize = options.minChunkSize ?? Math.min(100, maxChunkSize);
|
|
37
|
+
if (minChunkSize > maxChunkSize) {
|
|
38
|
+
throw new Error('minChunkSize must not exceed maxChunkSize');
|
|
39
|
+
}
|
|
28
40
|
this.embeddingService = options.embeddingService;
|
|
29
41
|
this.breakpointThreshold = options.breakpointThreshold ?? 0.5;
|
|
30
|
-
this.minChunkSize =
|
|
31
|
-
this.maxChunkSize =
|
|
42
|
+
this.minChunkSize = minChunkSize;
|
|
43
|
+
this.maxChunkSize = maxChunkSize;
|
|
32
44
|
}
|
|
33
45
|
async chunk(text, documentId) {
|
|
34
46
|
if (!text || text.trim().length === 0)
|
|
@@ -94,21 +106,25 @@ export class SemanticChunker {
|
|
|
94
106
|
}
|
|
95
107
|
return result;
|
|
96
108
|
}
|
|
109
|
+
findSentenceOffset(sentence, originalText, searchFrom) {
|
|
110
|
+
const idx = originalText.indexOf(sentence, searchFrom);
|
|
111
|
+
return idx >= 0 ? idx : searchFrom;
|
|
112
|
+
}
|
|
97
113
|
buildChunks(groups, originalText, documentId) {
|
|
98
114
|
const chunks = [];
|
|
99
115
|
let searchFrom = 0;
|
|
100
116
|
for (const group of groups) {
|
|
101
|
-
const
|
|
117
|
+
const startOffset = this.findSentenceOffset(group[0], originalText, searchFrom);
|
|
118
|
+
const lastSentence = group[group.length - 1];
|
|
119
|
+
const lastStart = this.findSentenceOffset(lastSentence, originalText, startOffset);
|
|
120
|
+
const endOffset = lastStart + lastSentence.length;
|
|
121
|
+
const content = originalText.slice(startOffset, endOffset);
|
|
102
122
|
if (content.length <= this.maxChunkSize) {
|
|
103
|
-
const startOffset = originalText.indexOf(group[0], searchFrom);
|
|
104
|
-
const lastSentence = group[group.length - 1];
|
|
105
|
-
const lastStart = originalText.indexOf(lastSentence, startOffset >= 0 ? startOffset : 0);
|
|
106
|
-
const endOffset = lastStart >= 0 ? lastStart + lastSentence.length : startOffset + content.length;
|
|
107
123
|
chunks.push({
|
|
108
124
|
id: nanoid(),
|
|
109
125
|
documentId,
|
|
110
126
|
content,
|
|
111
|
-
startOffset
|
|
127
|
+
startOffset,
|
|
112
128
|
endOffset,
|
|
113
129
|
order: chunks.length,
|
|
114
130
|
});
|
|
@@ -131,22 +147,35 @@ export class SemanticChunker {
|
|
|
131
147
|
let order = startOrder;
|
|
132
148
|
let offset = searchFrom;
|
|
133
149
|
for (const sentence of sentences) {
|
|
150
|
+
if (sentence.length > this.maxChunkSize) {
|
|
151
|
+
if (currentSentences.length > 0) {
|
|
152
|
+
const chunk = this.emitSentenceGroup(currentSentences, originalText, documentId, offset, order++);
|
|
153
|
+
chunks.push(chunk);
|
|
154
|
+
offset = chunk.endOffset;
|
|
155
|
+
currentSentences = [];
|
|
156
|
+
currentLength = 0;
|
|
157
|
+
}
|
|
158
|
+
const sentenceStart = this.findSentenceOffset(sentence, originalText, offset);
|
|
159
|
+
for (let pos = 0; pos < sentence.length; pos += this.maxChunkSize) {
|
|
160
|
+
const sliceEnd = Math.min(pos + this.maxChunkSize, sentence.length);
|
|
161
|
+
const slice = sentence.slice(pos, sliceEnd);
|
|
162
|
+
chunks.push({
|
|
163
|
+
id: nanoid(),
|
|
164
|
+
documentId,
|
|
165
|
+
content: slice,
|
|
166
|
+
startOffset: sentenceStart + pos,
|
|
167
|
+
endOffset: sentenceStart + sliceEnd,
|
|
168
|
+
order: order++,
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
offset = sentenceStart + sentence.length;
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
134
174
|
const newLength = currentLength + (currentSentences.length > 0 ? 1 : 0) + sentence.length;
|
|
135
175
|
if (newLength > this.maxChunkSize && currentSentences.length > 0) {
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
const lastStart = originalText.indexOf(lastSentence, startOffset >= 0 ? startOffset : 0);
|
|
140
|
-
const endOffset = lastStart >= 0 ? lastStart + lastSentence.length : startOffset + content.length;
|
|
141
|
-
chunks.push({
|
|
142
|
-
id: nanoid(),
|
|
143
|
-
documentId,
|
|
144
|
-
content,
|
|
145
|
-
startOffset: Math.max(0, startOffset),
|
|
146
|
-
endOffset,
|
|
147
|
-
order: order++,
|
|
148
|
-
});
|
|
149
|
-
offset = endOffset;
|
|
176
|
+
const chunk = this.emitSentenceGroup(currentSentences, originalText, documentId, offset, order++);
|
|
177
|
+
chunks.push(chunk);
|
|
178
|
+
offset = chunk.endOffset;
|
|
150
179
|
currentSentences = [sentence];
|
|
151
180
|
currentLength = sentence.length;
|
|
152
181
|
}
|
|
@@ -156,21 +185,23 @@ export class SemanticChunker {
|
|
|
156
185
|
}
|
|
157
186
|
}
|
|
158
187
|
if (currentSentences.length > 0) {
|
|
159
|
-
|
|
160
|
-
const startOffset = originalText.indexOf(currentSentences[0], offset);
|
|
161
|
-
const lastSentence = currentSentences[currentSentences.length - 1];
|
|
162
|
-
const lastStart = originalText.indexOf(lastSentence, startOffset >= 0 ? startOffset : 0);
|
|
163
|
-
const endOffset = lastStart >= 0 ? lastStart + lastSentence.length : startOffset + content.length;
|
|
164
|
-
chunks.push({
|
|
165
|
-
id: nanoid(),
|
|
166
|
-
documentId,
|
|
167
|
-
content,
|
|
168
|
-
startOffset: Math.max(0, startOffset),
|
|
169
|
-
endOffset,
|
|
170
|
-
order: order++,
|
|
171
|
-
});
|
|
188
|
+
chunks.push(this.emitSentenceGroup(currentSentences, originalText, documentId, offset, order++));
|
|
172
189
|
}
|
|
173
190
|
return chunks;
|
|
174
191
|
}
|
|
192
|
+
emitSentenceGroup(sentences, originalText, documentId, searchFrom, order) {
|
|
193
|
+
const startOffset = this.findSentenceOffset(sentences[0], originalText, searchFrom);
|
|
194
|
+
const lastSentence = sentences[sentences.length - 1];
|
|
195
|
+
const lastStart = this.findSentenceOffset(lastSentence, originalText, startOffset);
|
|
196
|
+
const endOffset = lastStart + lastSentence.length;
|
|
197
|
+
return {
|
|
198
|
+
id: nanoid(),
|
|
199
|
+
documentId,
|
|
200
|
+
content: originalText.slice(startOffset, endOffset),
|
|
201
|
+
startOffset,
|
|
202
|
+
endOffset,
|
|
203
|
+
order,
|
|
204
|
+
};
|
|
205
|
+
}
|
|
175
206
|
}
|
|
176
207
|
//# sourceMappingURL=semantic-chunker.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,
|
|
1
|
+
{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,iCAAiC,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,IAAI;YAC1D,iDAAiD,CACpD,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IACE,OAAO,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,OAAO,CAAC,mBAAmB,GAAG,CAAC,IAAI,OAAO,CAAC,mBAAmB,GAAG,CAAC,CAAC,EACpE,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,YAAY,GAAG,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAErE,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,kBAAkB,CAAC,QAAgB,EAAE,YAAoB,EAAE,UAAkB;QACnF,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACvD,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;IACrC,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;YAChF,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;YACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;YAElD,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBACH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;oBACzB,gBAAgB,GAAG,EAAE,CAAC;oBACtB,aAAa,GAAG,CAAC,CAAC;gBACpB,CAAC;gBAED,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9E,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,MAAM,EAAE;wBACZ,UAAU;wBACV,OAAO,EAAE,KAAK;wBACd,WAAW,EAAE,aAAa,GAAG,GAAG;wBAChC,SAAS,EAAE,aAAa,GAAG,QAAQ;wBACnC,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,GAAG,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CACpF,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,iBAAiB,CACvB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,KAAa;QAEb,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;QAElD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,UAAU;YACV,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC;YACnD,WAAW;YACX,SAAS;YACT,KAAK;SACN,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAUvE,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAW;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAS;gBAExB,OAAO,CAAC,EAAE,gBAAgB;IAMhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAUvE,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAW;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAS;gBAExB,OAAO,CAAC,EAAE,gBAAgB;IAMhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAkClD,OAAO,CAAC,eAAe;CAWxB"}
|
|
@@ -30,6 +30,9 @@ export class CSVLoader {
|
|
|
30
30
|
});
|
|
31
31
|
const headers = result.meta.fields ?? [];
|
|
32
32
|
const contentCol = this.contentColumn ?? headers[0];
|
|
33
|
+
if (this.contentColumn && !headers.includes(this.contentColumn)) {
|
|
34
|
+
throw new Error(`CSVLoader: contentColumn "${this.contentColumn}" not found in CSV headers: ${headers.join(', ')}`);
|
|
35
|
+
}
|
|
33
36
|
return result.data.map((row) => {
|
|
34
37
|
const content = row[contentCol] ?? '';
|
|
35
38
|
const metadata = this.extractMetadata(row);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,KAAK,UAAU,aAAa;IAC1B,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAQD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,aAAa,CAAU;IACvB,eAAe,CAAY;IAC3B,SAAS,CAAU;IAEpC,YAAY,OAA0B;QACpC,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,IAAI,GAAG,MAAM,aAAa,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAyB,GAAG,EAAE;YACrD,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QAEpD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YAE3C,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO;gBACP,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,KAAc;gBAC1B,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;aAClE,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe,CAAC,GAA2B;QACjD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;gBACf,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,KAAK,UAAU,aAAa;IAC1B,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAQD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,aAAa,CAAU;IACvB,eAAe,CAAY;IAC3B,SAAS,CAAU;IAEpC,YAAY,OAA0B;QACpC,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,IAAI,GAAG,MAAM,aAAa,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAyB,GAAG,EAAE;YACrD,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QAEpD,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,CAAC,aAAa,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnG,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YAE3C,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO;gBACP,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,KAAc;gBAC1B,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;aAClE,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe,CAAC,GAA2B;QACjD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;gBACf,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -11,6 +11,7 @@ export declare class JSONLoader implements DocumentLoader {
|
|
|
11
11
|
load(source: string): Promise<RAGDocument[]>;
|
|
12
12
|
private itemToDocument;
|
|
13
13
|
private extractContent;
|
|
14
|
+
private valueToString;
|
|
14
15
|
private extractMetadata;
|
|
15
16
|
}
|
|
16
17
|
//# sourceMappingURL=json-loader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAIvE,MAAM,WAAW,iBAAiB;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAW;gBAE/B,OAAO,CAAC,EAAE,iBAAiB;IAKjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"json-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAIvE,MAAM,WAAW,iBAAiB;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAW;gBAE/B,OAAO,CAAC,EAAE,iBAAiB;IAKjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAgBlD,OAAO,CAAC,cAAc;IAsBtB,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,eAAe;CAWxB"}
|
|
@@ -13,7 +13,14 @@ export class JSONLoader {
|
|
|
13
13
|
async load(source) {
|
|
14
14
|
const filePath = resolve(source);
|
|
15
15
|
const raw = await readFile(filePath, 'utf-8');
|
|
16
|
-
|
|
16
|
+
let data;
|
|
17
|
+
try {
|
|
18
|
+
data = JSON.parse(raw);
|
|
19
|
+
}
|
|
20
|
+
catch (err) {
|
|
21
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
22
|
+
throw new Error(`JSONLoader: failed to parse "${filePath}": ${message}`);
|
|
23
|
+
}
|
|
17
24
|
const items = Array.isArray(data) ? data : [data];
|
|
18
25
|
return items.map((item) => this.itemToDocument(item, filePath));
|
|
19
26
|
}
|
|
@@ -39,15 +46,20 @@ export class JSONLoader {
|
|
|
39
46
|
}
|
|
40
47
|
extractContent(obj) {
|
|
41
48
|
if (this.contentField && this.contentField in obj) {
|
|
42
|
-
return
|
|
49
|
+
return this.valueToString(obj[this.contentField]);
|
|
43
50
|
}
|
|
44
51
|
for (const field of CONTENT_FIELDS) {
|
|
45
52
|
if (field in obj) {
|
|
46
|
-
return
|
|
53
|
+
return this.valueToString(obj[field]);
|
|
47
54
|
}
|
|
48
55
|
}
|
|
49
56
|
return JSON.stringify(obj, null, 2);
|
|
50
57
|
}
|
|
58
|
+
valueToString(value) {
|
|
59
|
+
if (typeof value === 'string')
|
|
60
|
+
return value;
|
|
61
|
+
return JSON.stringify(value, null, 2);
|
|
62
|
+
}
|
|
51
63
|
extractMetadata(obj) {
|
|
52
64
|
if (!this.metadataFields)
|
|
53
65
|
return undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAOnD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,YAAY,CAAU;IACtB,cAAc,CAAY;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAOnD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,YAAY,CAAU;IACtB,cAAc,CAAY;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,MAAM,OAAO,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;IAClE,CAAC;IAEO,cAAc,CAAC,IAAa,EAAE,MAAc;QAClD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC;gBACrB,MAAM;gBACN,UAAU,EAAE,MAAM;aACnB,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE3C,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,MAAM;YAClB,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SAClE,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,GAA4B;QACjD,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,GAAG,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACpD,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC;IAEO,eAAe,CAAC,GAA4B;QAClD,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -2,7 +2,7 @@ import { readFile, readdir, stat } from 'node:fs/promises';
|
|
|
2
2
|
import { join, resolve } from 'node:path';
|
|
3
3
|
import { nanoid } from 'nanoid';
|
|
4
4
|
const MD_EXTENSIONS = new Set(['md', 'mdx']);
|
|
5
|
-
const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n
|
|
5
|
+
const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/;
|
|
6
6
|
export class MarkdownLoader {
|
|
7
7
|
supportedTypes = ['md', 'mdx'];
|
|
8
8
|
stripFrontmatter;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7C,MAAM,cAAc,GAAG,
|
|
1
|
+
{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7C,MAAM,cAAc,GAAG,wCAAwC,CAAC;AAMhE,MAAM,OAAO,cAAc;IAChB,cAAc,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvB,gBAAgB,CAAU;IAE3C,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,QAA6C,CAAC;QAElD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
|
|
@@ -9,6 +9,7 @@ export declare class PDFLoader implements DocumentLoader {
|
|
|
9
9
|
load(source: string): Promise<RAGDocument[]>;
|
|
10
10
|
private loadSingleDocument;
|
|
11
11
|
private loadSplitPages;
|
|
12
|
+
private parseWithErrorContext;
|
|
12
13
|
private buildPageDoc;
|
|
13
14
|
}
|
|
14
15
|
//# sourceMappingURL=pdf-loader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AA4BvE,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;gBAEzB,OAAO,CAAC,EAAE,gBAAgB;IAIhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAYpC,kBAAkB;YAsBlB,cAAc;YA6Bd,qBAAqB;IAcnC,OAAO,CAAC,YAAY;CAkBrB"}
|
|
@@ -26,7 +26,7 @@ export class PDFLoader {
|
|
|
26
26
|
return this.loadSingleDocument(pdfParse, buffer, filePath);
|
|
27
27
|
}
|
|
28
28
|
async loadSingleDocument(pdfParse, buffer, source) {
|
|
29
|
-
const result = await pdfParse
|
|
29
|
+
const result = await this.parseWithErrorContext(pdfParse, buffer, source);
|
|
30
30
|
const title = result.info?.Title;
|
|
31
31
|
const metadata = { pages: result.numpages };
|
|
32
32
|
if (title)
|
|
@@ -42,14 +42,33 @@ export class PDFLoader {
|
|
|
42
42
|
];
|
|
43
43
|
}
|
|
44
44
|
async loadSplitPages(pdfParse, buffer, source) {
|
|
45
|
-
const
|
|
45
|
+
const pageTexts = [];
|
|
46
|
+
const result = await this.parseWithErrorContext(pdfParse, buffer, source, {
|
|
47
|
+
pagerender: async (pageData) => {
|
|
48
|
+
if (typeof pageData.getTextContent !== 'function')
|
|
49
|
+
return '';
|
|
50
|
+
const textContent = await pageData.getTextContent();
|
|
51
|
+
const text = textContent.items.map((item) => item.str).join('');
|
|
52
|
+
pageTexts.push(text);
|
|
53
|
+
return text;
|
|
54
|
+
},
|
|
55
|
+
});
|
|
46
56
|
const title = result.info?.Title;
|
|
47
|
-
const pages =
|
|
57
|
+
const pages = pageTexts.filter((p) => p.trim().length > 0);
|
|
48
58
|
if (pages.length === 0) {
|
|
49
59
|
return [this.buildPageDoc(result.text, source, 1, result.numpages, title)];
|
|
50
60
|
}
|
|
51
61
|
return pages.map((text, i) => this.buildPageDoc(text.trim(), source, i + 1, result.numpages, title));
|
|
52
62
|
}
|
|
63
|
+
async parseWithErrorContext(pdfParse, buffer, source, options) {
|
|
64
|
+
try {
|
|
65
|
+
return await pdfParse(buffer, options);
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
69
|
+
throw new Error(`PDFLoader: failed to parse "${source}": ${message}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
53
72
|
buildPageDoc(content, source, pageNumber, totalPages, title) {
|
|
54
73
|
const metadata = { pageNumber, totalPages };
|
|
55
74
|
if (title)
|