@meaningfully/core 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Meaningfully.d.ts +3 -2
- package/dist/Meaningfully.d.ts.map +1 -1
- package/dist/Meaningfully.js +8 -4
- package/dist/Meaningfully.js.map +1 -1
- package/dist/api/embedding.d.ts +1 -1
- package/dist/api/embedding.d.ts.map +1 -1
- package/dist/api/embedding.js +3 -3
- package/dist/api/embedding.js.map +1 -1
- package/dist/services/embeddings.d.ts +0 -1
- package/dist/services/embeddings.d.ts.map +1 -1
- package/dist/services/embeddings.js +51 -72
- package/dist/services/embeddings.js.map +1 -1
- package/dist/types/index.d.ts +2 -0
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +5 -2
- package/src/Meaningfully.ts +9 -5
- package/{tests → src/__tests__}/MetadataManager.test.ts +1 -1
- package/{tests → src/api/__tests__}/embedding.test.ts +20 -20
- package/src/api/embedding.ts +3 -3
- package/{tests → src/services/__tests__}/csvLoader.test.ts +1 -1
- package/{tests → src/services/__tests__}/embeddings.test.ts +2 -2
- package/{tests → src/services/__tests__}/sentenceSplitter.test.ts +1 -1
- package/src/services/embeddings.d.ts +0 -1
- package/src/services/embeddings.ts +57 -81
- package/src/types/index.ts +2 -0
- package/tsconfig.json +1 -1
- package/dist/api/embedding.test.d.ts +0 -2
- package/dist/api/embedding.test.d.ts.map +0 -1
- package/dist/api/embedding.test.js +0 -340
- package/dist/api/embedding.test.js.map +0 -1
- package/dist/services/csvLoader.test.d.ts +0 -2
- package/dist/services/csvLoader.test.d.ts.map +0 -1
- package/dist/services/csvLoader.test.js +0 -75
- package/dist/services/csvLoader.test.js.map +0 -1
- package/dist/services/embeddings.test.d.ts +0 -2
- package/dist/services/embeddings.test.d.ts.map +0 -1
- package/dist/services/embeddings.test.js +0 -115
- package/dist/services/embeddings.test.js.map +0 -1
- package/dist/services/sentenceSplitter.test.d.ts +0 -2
- package/dist/services/sentenceSplitter.test.d.ts.map +0 -1
- package/dist/services/sentenceSplitter.test.js +0 -68
- package/dist/services/sentenceSplitter.test.js.map +0 -1
- package/src/api/embedding.d.ts +0 -6
- package/tests/csvLoader.test.d.ts +0 -1
- package/tests/embedding.test.d.ts +0 -1
- package/tests/embeddings.test.d.ts +0 -1
- package/tests/sentenceSplitter.test.d.ts +0 -1
|
@@ -1,340 +0,0 @@
|
|
|
1
|
-
//@ts-nocheck
|
|
2
|
-
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
3
|
-
import { createEmbeddings, previewResults, getDocStore, getIndex, search } from './embedding';
|
|
4
|
-
import { loadDocumentsFromCsv } from '../services/csvLoader';
|
|
5
|
-
import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, getExistingDocStore } from '../services/embeddings';
|
|
6
|
-
// filepath: /Users/jeremybmerrill/code/meaningfully/src/main/api/embedding.test.ts
|
|
7
|
-
vi.mock('../services/csvLoader');
|
|
8
|
-
vi.mock('../services/embeddings');
|
|
9
|
-
describe('embedding.ts', () => {
|
|
10
|
-
describe('createEmbeddings', () => {
|
|
11
|
-
it('should create embeddings and return success', async () => {
|
|
12
|
-
const mockDocuments = [{ text: 'doc1' }, { text: 'doc2' }];
|
|
13
|
-
const mockNodes = [{ node: 'node1' }, { node: 'node2' }];
|
|
14
|
-
const mockIndex = 'index1';
|
|
15
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
16
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
17
|
-
persistNodes.mockResolvedValue(mockIndex);
|
|
18
|
-
const result = await createEmbeddings('path/to/csv', 'text', {}, {});
|
|
19
|
-
expect(result).toEqual({ success: true, index: mockIndex });
|
|
20
|
-
});
|
|
21
|
-
it('should return error on failure', async () => {
|
|
22
|
-
loadDocumentsFromCsv.mockRejectedValue(new Error('Failed to load documents'));
|
|
23
|
-
const result = await createEmbeddings('path/to/csv', 'text', {}, {});
|
|
24
|
-
expect(result).toEqual({ success: false, error: 'Failed to load documents' });
|
|
25
|
-
});
|
|
26
|
-
it('should handle empty documents', async () => {
|
|
27
|
-
loadDocumentsFromCsv.mockResolvedValue([]);
|
|
28
|
-
const result = await createEmbeddings('path/to/csv', 'text', {}, {});
|
|
29
|
-
expect(result).toEqual({ success: false, error: 'That CSV does not appear to contain any documents. Please check the file and try again.' });
|
|
30
|
-
});
|
|
31
|
-
});
|
|
32
|
-
describe('previewResults', () => {
|
|
33
|
-
it('should return preview results and estimated cost', async () => {
|
|
34
|
-
const mockDocuments = Array(20).fill({ text: 'doc' });
|
|
35
|
-
const mockNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
|
|
36
|
-
const mockPreviewNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
|
|
37
|
-
const mockEstimate = { estimatedPrice: 10, tokenCount: 100, pricePer1M: 0.01 };
|
|
38
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
39
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
40
|
-
estimateCost.mockReturnValue(mockEstimate);
|
|
41
|
-
const result = await previewResults('path/to/csv', 'text', {});
|
|
42
|
-
expect(result).toEqual({
|
|
43
|
-
success: true,
|
|
44
|
-
nodes: mockPreviewNodes,
|
|
45
|
-
...mockEstimate
|
|
46
|
-
});
|
|
47
|
-
});
|
|
48
|
-
it('should return error on failure', async () => {
|
|
49
|
-
loadDocumentsFromCsv.mockRejectedValue(new Error('Failed to load documents'));
|
|
50
|
-
const result = await previewResults('path/to/csv', 'text', {});
|
|
51
|
-
expect(result).toEqual({ success: false, error: 'Failed to load documents' });
|
|
52
|
-
});
|
|
53
|
-
it('should handle empty documents', async () => {
|
|
54
|
-
loadDocumentsFromCsv.mockResolvedValue([]);
|
|
55
|
-
const result = await previewResults('path/to/csv', 'text', {});
|
|
56
|
-
expect(result).toEqual({ success: false, error: 'That CSV does not appear to contain any documents. Please check the file and try again.' });
|
|
57
|
-
});
|
|
58
|
-
});
|
|
59
|
-
describe('getDocStore', () => {
|
|
60
|
-
it('should return existing doc store', async () => {
|
|
61
|
-
const mockDocStore = 'docStore';
|
|
62
|
-
getExistingDocStore.mockResolvedValue(mockDocStore);
|
|
63
|
-
const result = await getDocStore({});
|
|
64
|
-
expect(result).toBe(mockDocStore);
|
|
65
|
-
});
|
|
66
|
-
});
|
|
67
|
-
describe('getIndex', () => {
|
|
68
|
-
it('should return existing vector store index', async () => {
|
|
69
|
-
const mockIndex = 'index';
|
|
70
|
-
getExistingVectorStoreIndex.mockResolvedValue(mockIndex);
|
|
71
|
-
const result = await getIndex({}, {});
|
|
72
|
-
expect(result).toBe(mockIndex);
|
|
73
|
-
});
|
|
74
|
-
});
|
|
75
|
-
describe('search', () => {
|
|
76
|
-
it('should return search results', async () => {
|
|
77
|
-
const mockResults = [
|
|
78
|
-
{ node: { getContent: () => 'content1', metadata: {} }, score: 1 },
|
|
79
|
-
{ node: { getContent: () => 'content2', metadata: {} }, score: 2 }
|
|
80
|
-
];
|
|
81
|
-
searchDocuments.mockResolvedValue(mockResults);
|
|
82
|
-
const result = await search('index', 'query');
|
|
83
|
-
expect(result).toEqual([
|
|
84
|
-
{ text: 'content1', score: 1, metadata: {} },
|
|
85
|
-
{ text: 'content2', score: 2, metadata: {} }
|
|
86
|
-
]);
|
|
87
|
-
});
|
|
88
|
-
it('should handle no search results', async () => {
|
|
89
|
-
searchDocuments.mockResolvedValue([]);
|
|
90
|
-
const result = await search('index', 'query');
|
|
91
|
-
expect(result).toEqual([]);
|
|
92
|
-
});
|
|
93
|
-
it('should handle search results with null scores', async () => {
|
|
94
|
-
const mockResults = [
|
|
95
|
-
{ node: { getContent: () => 'content1', metadata: {} }, score: null },
|
|
96
|
-
{ node: { getContent: () => 'content2', metadata: {} }, score: null }
|
|
97
|
-
];
|
|
98
|
-
searchDocuments.mockResolvedValue(mockResults);
|
|
99
|
-
const result = await search('index', 'query');
|
|
100
|
-
expect(result).toEqual([
|
|
101
|
-
{ text: 'content1', score: 0, metadata: {} },
|
|
102
|
-
{ text: 'content2', score: 0, metadata: {} }
|
|
103
|
-
]);
|
|
104
|
-
});
|
|
105
|
-
});
|
|
106
|
-
});
|
|
107
|
-
describe('previewResults', () => {
|
|
108
|
-
it('should return preview results and estimated cost', async () => {
|
|
109
|
-
const mockDocuments = Array(20).fill({ text: 'doc' });
|
|
110
|
-
const mockNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
|
|
111
|
-
const mockPreviewNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
|
|
112
|
-
const mockEstimate = { estimatedPrice: 10, tokenCount: 100, pricePer1M: 0.01 };
|
|
113
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
114
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
115
|
-
estimateCost.mockReturnValue(mockEstimate);
|
|
116
|
-
const result = await previewResults('path/to/csv', 'text', {});
|
|
117
|
-
expect(result).toEqual({
|
|
118
|
-
success: true,
|
|
119
|
-
nodes: mockPreviewNodes,
|
|
120
|
-
...mockEstimate
|
|
121
|
-
});
|
|
122
|
-
});
|
|
123
|
-
it('should return error on failure', async () => {
|
|
124
|
-
loadDocumentsFromCsv.mockRejectedValue(new Error('Failed to load documents'));
|
|
125
|
-
const result = await previewResults('path/to/csv', 'text', {});
|
|
126
|
-
expect(result).toEqual({ success: false, error: 'Failed to load documents' });
|
|
127
|
-
});
|
|
128
|
-
});
|
|
129
|
-
describe('getDocStore', () => {
|
|
130
|
-
it('should return existing doc store', async () => {
|
|
131
|
-
const mockDocStore = 'docStore';
|
|
132
|
-
getExistingDocStore.mockResolvedValue(mockDocStore);
|
|
133
|
-
const result = await getDocStore({});
|
|
134
|
-
expect(result).toBe(mockDocStore);
|
|
135
|
-
});
|
|
136
|
-
});
|
|
137
|
-
describe('getIndex', () => {
|
|
138
|
-
it('should return existing vector store index', async () => {
|
|
139
|
-
const mockIndex = 'index';
|
|
140
|
-
getExistingVectorStoreIndex.mockResolvedValue(mockIndex);
|
|
141
|
-
const result = await getIndex({}, {});
|
|
142
|
-
expect(result).toBe(mockIndex);
|
|
143
|
-
});
|
|
144
|
-
});
|
|
145
|
-
describe('search', () => {
|
|
146
|
-
it('should return search results', async () => {
|
|
147
|
-
const mockResults = [
|
|
148
|
-
{ node: { getContent: () => 'content1', metadata: {} }, score: 1 },
|
|
149
|
-
{ node: { getContent: () => 'content2', metadata: {} }, score: 2 }
|
|
150
|
-
];
|
|
151
|
-
searchDocuments.mockResolvedValue(mockResults);
|
|
152
|
-
const result = await search('index', 'query');
|
|
153
|
-
expect(result).toEqual([
|
|
154
|
-
{ text: 'content1', score: 1, metadata: {} },
|
|
155
|
-
{ text: 'content2', score: 2, metadata: {} }
|
|
156
|
-
]);
|
|
157
|
-
});
|
|
158
|
-
it('should handle no search results', async () => {
|
|
159
|
-
searchDocuments.mockResolvedValue([]);
|
|
160
|
-
const result = await search('index', 'query');
|
|
161
|
-
expect(result).toEqual([]);
|
|
162
|
-
});
|
|
163
|
-
it('should handle search results with null scores', async () => {
|
|
164
|
-
const mockResults = [
|
|
165
|
-
{ node: { getContent: () => 'content1', metadata: {} }, score: null },
|
|
166
|
-
{ node: { getContent: () => 'content2', metadata: {} }, score: null }
|
|
167
|
-
];
|
|
168
|
-
searchDocuments.mockResolvedValue(mockResults);
|
|
169
|
-
const result = await search('index', 'query');
|
|
170
|
-
expect(result).toEqual([
|
|
171
|
-
{ text: 'content1', score: 0, metadata: {} },
|
|
172
|
-
{ text: 'content2', score: 0, metadata: {} }
|
|
173
|
-
]);
|
|
174
|
-
});
|
|
175
|
-
});
|
|
176
|
-
describe('createEmbeddings with progress tracking', () => {
|
|
177
|
-
beforeEach(() => {
|
|
178
|
-
vi.clearAllMocks();
|
|
179
|
-
});
|
|
180
|
-
it('should pass progress callback to persistNodes', async () => {
|
|
181
|
-
// Setup mocks
|
|
182
|
-
const mockDocuments = [{ text: 'doc1' }, { text: 'doc2' }];
|
|
183
|
-
const mockNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
|
|
184
|
-
const mockIndex = 'index1';
|
|
185
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
186
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
187
|
-
persistNodes.mockImplementation((nodes, config, settings, clients, callback) => {
|
|
188
|
-
// Call the callback with sample progress
|
|
189
|
-
if (callback) {
|
|
190
|
-
callback(1, 2); // 50% progress
|
|
191
|
-
callback(2, 2); // 100% progress
|
|
192
|
-
}
|
|
193
|
-
return Promise.resolve(mockIndex);
|
|
194
|
-
});
|
|
195
|
-
// Execute
|
|
196
|
-
const result = await createEmbeddings('path/to/csv', 'text', { modelName: 'test-model' }, {}, {});
|
|
197
|
-
// Verify
|
|
198
|
-
expect(persistNodes).toHaveBeenCalledTimes(1);
|
|
199
|
-
expect(persistNodes.mock.calls[0][4]).toBeInstanceOf(Function); // Verify callback was passed
|
|
200
|
-
expect(result).toEqual({ success: true, index: mockIndex });
|
|
201
|
-
});
|
|
202
|
-
it('should correctly track progress through ProgressManager', async () => {
|
|
203
|
-
// Setup
|
|
204
|
-
vi.mock('../services/progressManager', () => {
|
|
205
|
-
const mockInstance = {
|
|
206
|
-
startOperation: vi.fn(),
|
|
207
|
-
updateProgress: vi.fn(),
|
|
208
|
-
completeOperation: vi.fn(),
|
|
209
|
-
clearOperation: vi.fn()
|
|
210
|
-
};
|
|
211
|
-
return {
|
|
212
|
-
ProgressManager: {
|
|
213
|
-
getInstance: () => mockInstance
|
|
214
|
-
}
|
|
215
|
-
};
|
|
216
|
-
});
|
|
217
|
-
// Re-import to use mocked version
|
|
218
|
-
const { createEmbeddings } = await import('./embedding');
|
|
219
|
-
const { ProgressManager } = await import('../services/progressManager');
|
|
220
|
-
const mockDocuments = [{ text: 'doc1' }, { text: 'doc2' }];
|
|
221
|
-
const mockNodes = [{ text: 'node1', metadata: {} }];
|
|
222
|
-
const mockIndex = 'testIndex';
|
|
223
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
224
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
225
|
-
persistNodes.mockImplementation((nodes, config, settings, clients, callback) => {
|
|
226
|
-
if (callback)
|
|
227
|
-
callback(1, 2); // Call with 50% progress
|
|
228
|
-
return Promise.resolve(mockIndex);
|
|
229
|
-
});
|
|
230
|
-
// Execute
|
|
231
|
-
await createEmbeddings('path/to/csv', 'text', {}, {}, {});
|
|
232
|
-
// Verify
|
|
233
|
-
const progressManager = ProgressManager.getInstance();
|
|
234
|
-
expect(progressManager.startOperation).toHaveBeenCalledWith(expect.stringMatching(/^embed-\d+$/), 100);
|
|
235
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 5);
|
|
236
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), expect.any(Number));
|
|
237
|
-
expect(progressManager.completeOperation).toHaveBeenCalledWith(expect.any(String));
|
|
238
|
-
});
|
|
239
|
-
it('should properly calculate percentage in progress callback', async () => {
|
|
240
|
-
// Setup mocks with spy on updateProgress
|
|
241
|
-
vi.mock('../services/progressManager', () => {
|
|
242
|
-
const mockInstance = {
|
|
243
|
-
startOperation: vi.fn(),
|
|
244
|
-
updateProgress: vi.fn(),
|
|
245
|
-
completeOperation: vi.fn(),
|
|
246
|
-
clearOperation: vi.fn()
|
|
247
|
-
};
|
|
248
|
-
return {
|
|
249
|
-
ProgressManager: {
|
|
250
|
-
getInstance: () => mockInstance
|
|
251
|
-
}
|
|
252
|
-
};
|
|
253
|
-
});
|
|
254
|
-
// Re-import to use mocked version
|
|
255
|
-
const { createEmbeddings } = await import('./embedding');
|
|
256
|
-
const { ProgressManager } = await import('../services/progressManager');
|
|
257
|
-
const mockDocuments = [{ text: 'doc1' }];
|
|
258
|
-
const mockNodes = [{ text: 'node1', metadata: {} }];
|
|
259
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
260
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
261
|
-
// Simulate persistNodes calling the callback with various progress values
|
|
262
|
-
persistNodes.mockImplementation((nodes, config, settings, clients, callback) => {
|
|
263
|
-
if (callback) {
|
|
264
|
-
callback(0, 10); // 0% progress
|
|
265
|
-
callback(5, 10); // 50% progress
|
|
266
|
-
callback(10, 10); // 100% progress
|
|
267
|
-
}
|
|
268
|
-
return Promise.resolve('mockIndex');
|
|
269
|
-
});
|
|
270
|
-
// Execute
|
|
271
|
-
await createEmbeddings('path/to/csv', 'text', {}, {}, {});
|
|
272
|
-
// Verify percentage calculations
|
|
273
|
-
// Initial update at 5%
|
|
274
|
-
const progressManager = ProgressManager.getInstance();
|
|
275
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 5);
|
|
276
|
-
// Progress updates: 0%, 50%, 100% mapped to 5-95% range
|
|
277
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 5); // 0% -> 5%
|
|
278
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 50); // 50% -> 50%
|
|
279
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 95); // 100% -> 95%
|
|
280
|
-
});
|
|
281
|
-
it('should clear operation on empty documents', async () => {
|
|
282
|
-
// Setup
|
|
283
|
-
vi.mock('../services/progressManager', () => {
|
|
284
|
-
const mockInstance = {
|
|
285
|
-
startOperation: vi.fn(),
|
|
286
|
-
updateProgress: vi.fn(),
|
|
287
|
-
completeOperation: vi.fn(),
|
|
288
|
-
clearOperation: vi.fn()
|
|
289
|
-
};
|
|
290
|
-
return {
|
|
291
|
-
ProgressManager: {
|
|
292
|
-
getInstance: () => mockInstance
|
|
293
|
-
}
|
|
294
|
-
};
|
|
295
|
-
});
|
|
296
|
-
// Re-import to use mocked version
|
|
297
|
-
const { createEmbeddings } = await import('./embedding');
|
|
298
|
-
const { ProgressManager } = await import('../services/progressManager');
|
|
299
|
-
loadDocumentsFromCsv.mockResolvedValue([]);
|
|
300
|
-
// Execute
|
|
301
|
-
const result = await createEmbeddings('path/to/csv', 'text', {}, {}, {});
|
|
302
|
-
// Verify
|
|
303
|
-
const progressManager = ProgressManager.getInstance();
|
|
304
|
-
expect(progressManager.clearOperation).toHaveBeenCalled();
|
|
305
|
-
expect(result).toEqual({
|
|
306
|
-
success: false,
|
|
307
|
-
error: "That CSV does not appear to contain any documents. Please check the file and try again."
|
|
308
|
-
});
|
|
309
|
-
});
|
|
310
|
-
it('shoulde complete operation on successful embedding', async () => {
|
|
311
|
-
// Setup
|
|
312
|
-
vi.mock('../services/progressManager', () => {
|
|
313
|
-
const mockInstance = {
|
|
314
|
-
startOperation: vi.fn(),
|
|
315
|
-
updateProgress: vi.fn(),
|
|
316
|
-
completeOperation: vi.fn(),
|
|
317
|
-
clearOperation: vi.fn()
|
|
318
|
-
};
|
|
319
|
-
return {
|
|
320
|
-
ProgressManager: {
|
|
321
|
-
getInstance: () => mockInstance
|
|
322
|
-
}
|
|
323
|
-
};
|
|
324
|
-
});
|
|
325
|
-
// Re-import to use mocked version
|
|
326
|
-
const { createEmbeddings } = await import('./embedding');
|
|
327
|
-
const { ProgressManager } = await import('../services/progressManager');
|
|
328
|
-
const mockDocuments = [{ text: 'doc1' }];
|
|
329
|
-
const mockNodes = [{ text: 'node1', metadata: {} }];
|
|
330
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
331
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
332
|
-
persistNodes.mockResolvedValue('mockIndex');
|
|
333
|
-
// Execute
|
|
334
|
-
await createEmbeddings('path/to/csv', 'text', {}, {}, {});
|
|
335
|
-
// Verify
|
|
336
|
-
const progressManager = ProgressManager.getInstance();
|
|
337
|
-
expect(progressManager.completeOperation).toHaveBeenCalled();
|
|
338
|
-
});
|
|
339
|
-
});
|
|
340
|
-
//# sourceMappingURL=embedding.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embedding.test.js","sourceRoot":"","sources":["../../src/api/embedding.test.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC9F,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAGlK,mFAAmF;AAGnF,EAAE,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;AACjC,EAAE,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;AAElC,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC1B,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACzD,MAAM,SAAS,GAAG,QAAQ,CAAC;YAC3B,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvD,YAAY,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAE1C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC5C,oBAAoB,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;YAE9E,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC3C,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,yFAAyF,EAAE,CAAC,CAAC;QACjJ,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,aAAa,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACtD,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YACrF,MAAM,gBAAgB,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC5F,MAAM,YAAY,GAAG,EAAE,cAAc,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;YAC/E,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvD,YAAY,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACnB,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,gBAAgB;gBACvB,GAAG,YAAY;aAClB,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC5C,oBAAoB,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;YAE9E,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC3C,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,yFAAyF,EAAE,CAAC,CAAC;QACjJ,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,YAAY,GAAG,UAAU,CAAC;YAChC,mBAAmB,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAEpD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,CAAC;YAErC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,SAAS,GAAG,OAAO,CAAC;YAC1B,2BAA2B,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEzD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAEtC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACpB,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,WAAW,GAAG;gBAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;gBAClE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;aACrE,CAAC;YACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;gBAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC7C,eAAe,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAEtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,WAAW,GAAG;gBAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;gBACrE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;aACxE,CAAC;YACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;gBAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAED,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,aAAa,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,MAAM,gBAAgB,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5F,MAAM,YAAY,GAAG,EAAE,cAAc,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QAC/E,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,gBAAgB;YACvB,GAAG,YAAY;SAChB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,oBAAoB,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;QAE9E,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,YAAY,GAAG,UAAU,CAAC;QAChC,mBAAmB,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,CAAC;QAErC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;IACxB,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,OAAO,CAAC;QAC1B,2BAA2B,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAEzD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAEtC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AACD,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;IACpB,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,WAAW,GAAG;YAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;YAClE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;SACrE,CAAC;QACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;SAC/C,CAAC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;QACjD,eAAe,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAEtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,WAAW,GAAG;YAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;YACrE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SACxE,CAAC;QACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;SAC/C,CAAC,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AACH,QAAQ,CAAC,yCAAyC,EAAE,GAAG,EAAE;IACvD,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC7D,cAAc;QACd,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,MAAM,SAAS,GAAG,QAAQ,CAAC;QAC3B,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;YAC7E,yCAAyC;YACzC,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,eAAe;gBAChC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,gBAAgB;YACnC,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,UAAU;QACV,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAElG,SAAS;QACT,MAAM,CAAC,YAAY,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;QAC7F,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACvE,QAAQ;QACR,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACpD,MAAM,SAAS,GAAG,WAAW,CAAC;QAE9B,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;YAC7E,IAAI,QAAQ;gBAAE,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,yBAAyB;YACvD,OAAO,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,UAAU;QACV,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1D,SAAS;QACT,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC,aAAa,CAAC,EAAE,GAAG,CAAC,CAAC;QACvG,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACpG,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACrF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,yCAAyC;QACzC,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACpD,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAEvD,0EAA0E;QAC1E,YAAY,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;YAC7E,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,cAAc;gBAChC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,eAAe;gBACjC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB;YACpC,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,UAAU;QACV,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1D,iCAAiC;QACjC,uBAAuB;QACvB,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAEnF,wDAAwD;QACxD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAG,WAAW;QACjG,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,aAAa;QACnG,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,cAAc;IACtG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,QAAQ;QACR,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAE3C,UAAU;QACV,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAEzE,SAAS;QACT,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,yFAAyF;SACjG,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,QAAQ;QACR,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACpD,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAE5C,UAAU;QACV,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1D,SAAS;QACT,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,gBAAgB,EAAE,CAAC;IAC/D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"csvLoader.test.d.ts","sourceRoot":"","sources":["../../src/services/csvLoader.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
//@ts-nocheck
|
|
2
|
-
import { describe, it, expect, vi } from 'vitest';
|
|
3
|
-
import { readFileSync } from 'fs';
|
|
4
|
-
import { loadDocumentsFromCsv } from './csvLoader';
|
|
5
|
-
import { Document } from 'llamaindex';
|
|
6
|
-
import Papa from 'papaparse';
|
|
7
|
-
vi.mock('fs');
|
|
8
|
-
vi.mock('papaparse');
|
|
9
|
-
describe('csvLoader.ts', () => {
|
|
10
|
-
describe('loadDocumentsFromCsv', () => {
|
|
11
|
-
it('should load documents from CSV and return Document instances', async () => {
|
|
12
|
-
const mockFileContent = 'text,metadata1,metadata2\ncontent1,meta1,meta2\ncontent2,meta3,meta4';
|
|
13
|
-
const mockParsedData = {
|
|
14
|
-
data: [
|
|
15
|
-
{ text: 'content1', metadata1: 'meta1', metadata2: 'meta2' },
|
|
16
|
-
{ text: 'content2', metadata1: 'meta3', metadata2: 'meta4' }
|
|
17
|
-
]
|
|
18
|
-
};
|
|
19
|
-
readFileSync.mockReturnValue(mockFileContent);
|
|
20
|
-
Papa.parse.mockReturnValue(mockParsedData);
|
|
21
|
-
const result = await loadDocumentsFromCsv('path/to/csv', 'text');
|
|
22
|
-
expect(remove_id(result)).toEqual(remove_id([
|
|
23
|
-
new Document({ text: 'content1', metadata: { metadata1: 'meta1', metadata2: 'meta2' } }),
|
|
24
|
-
new Document({ text: 'content2', metadata: { metadata1: 'meta3', metadata2: 'meta4' } })
|
|
25
|
-
]));
|
|
26
|
-
});
|
|
27
|
-
it('should handle empty CSV file', async () => {
|
|
28
|
-
const mockFileContent = '';
|
|
29
|
-
const mockParsedData = { data: [] };
|
|
30
|
-
readFileSync.mockReturnValue(mockFileContent);
|
|
31
|
-
Papa.parse.mockReturnValue(mockParsedData);
|
|
32
|
-
const result = await loadDocumentsFromCsv('path/to/csv', 'text');
|
|
33
|
-
expect(result).toEqual([]);
|
|
34
|
-
});
|
|
35
|
-
it('should handle missing text column', async () => {
|
|
36
|
-
const mockFileContent = 'metadata1,metadata2\nmeta1,meta2\nmeta3,meta4';
|
|
37
|
-
const mockParsedData = {
|
|
38
|
-
data: [
|
|
39
|
-
{ metadata1: 'meta1', metadata2: 'meta2' },
|
|
40
|
-
{ metadata1: 'meta3', metadata2: 'meta4' }
|
|
41
|
-
]
|
|
42
|
-
};
|
|
43
|
-
readFileSync.mockReturnValue(mockFileContent);
|
|
44
|
-
Papa.parse.mockReturnValue(mockParsedData);
|
|
45
|
-
const result = await loadDocumentsFromCsv('path/to/csv', 'text');
|
|
46
|
-
expect(remove_id(result)).toEqual(remove_id([
|
|
47
|
-
new Document({ text: undefined, metadata: { metadata1: 'meta1', metadata2: 'meta2' } }),
|
|
48
|
-
new Document({ text: undefined, metadata: { metadata1: 'meta3', metadata2: 'meta4' } })
|
|
49
|
-
]));
|
|
50
|
-
});
|
|
51
|
-
it('should handle null values in metadata', async () => {
|
|
52
|
-
const mockFileContent = 'text,metadata1,metadata2\ncontent1,,meta2\ncontent2,meta3,';
|
|
53
|
-
const mockParsedData = {
|
|
54
|
-
data: [
|
|
55
|
-
{ text: 'content1', metadata1: null, metadata2: 'meta2' },
|
|
56
|
-
{ text: 'content2', metadata1: 'meta3', metadata2: null }
|
|
57
|
-
]
|
|
58
|
-
};
|
|
59
|
-
readFileSync.mockReturnValue(mockFileContent);
|
|
60
|
-
Papa.parse.mockReturnValue(mockParsedData);
|
|
61
|
-
const result = await loadDocumentsFromCsv('path/to/csv', 'text');
|
|
62
|
-
expect(remove_id(result)).toEqual(remove_id([
|
|
63
|
-
new Document({ text: 'content1', metadata: { metadata1: '', metadata2: 'meta2' } }),
|
|
64
|
-
new Document({ text: 'content2', metadata: { metadata1: 'meta3', metadata2: '' } })
|
|
65
|
-
]));
|
|
66
|
-
});
|
|
67
|
-
});
|
|
68
|
-
});
|
|
69
|
-
function remove_id(list_of_documents) {
|
|
70
|
-
return list_of_documents.map((doc) => {
|
|
71
|
-
const { id_, ...doc_without_id } = doc;
|
|
72
|
-
return doc_without_id;
|
|
73
|
-
});
|
|
74
|
-
}
|
|
75
|
-
//# sourceMappingURL=csvLoader.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"csvLoader.test.js","sourceRoot":"","sources":["../../src/services/csvLoader.test.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAErB,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,eAAe,GAAG,sEAAsE,CAAC;YAC/F,MAAM,cAAc,GAAG;gBACrB,IAAI,EAAE;oBACJ,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;oBAC5D,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;iBAC7D;aACF,CAAC;YACF,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEjE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;gBAC1C,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;gBACxF,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;aACzF,CAAC,CAAC,CAAC;QACN,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,eAAe,GAAG,EAAE,CAAC;YAC3B,MAAM,cAAc,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;YACpC,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEjE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,eAAe,GAAG,+CAA+C,CAAC;YACxE,MAAM,cAAc,GAAG;gBACrB,IAAI,EAAE;oBACJ,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;oBAC1C,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;iBAC3C;aACF,CAAC;YACF,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEjE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;gBAC1C,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;gBACvF,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;aACxF,CAAC,CAAC,CAAC;QACN,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,eAAe,GAAG,4DAA4D,CAAC;YACrF,MAAM,cAAc,GAAG;gBACnB,IAAI,EAAE;oBACN,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE;oBACzD,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE;iBACxD;aACJ,CAAC;YACF,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YACjE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;gBACxC,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;gBACnF,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,CAAC;aACtF,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,SAAS,SAAS,CAAC,iBAAiB;IAClC,OAAO,iBAAiB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACnC,MAAM,EAAE,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,GAAG,CAAC;QACvC,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACH,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.test.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
//@ts-nocheck
|
|
2
|
-
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
3
|
-
import { Document, TextNode } from 'llamaindex';
|
|
4
|
-
// First, set up the mock before importing the module
|
|
5
|
-
vi.mock(import("./embeddings"), async (importOriginal) => {
|
|
6
|
-
const actual = await importOriginal();
|
|
7
|
-
return {
|
|
8
|
-
...actual,
|
|
9
|
-
// your mocked methods
|
|
10
|
-
estimateCost: vi.fn(),
|
|
11
|
-
getExistingVectorStoreIndex: vi.fn(),
|
|
12
|
-
persistNodes: vi.fn(),
|
|
13
|
-
persistDocuments: vi.fn(),
|
|
14
|
-
getExistingDocStore: vi.fn(),
|
|
15
|
-
searchDocuments: vi.fn()
|
|
16
|
-
};
|
|
17
|
-
});
|
|
18
|
-
// Now import the mocked functions
|
|
19
|
-
import { transformDocumentsToNodes, getEmbedModel } from './embeddings';
|
|
20
|
-
describe('transformDocumentsToNodes', () => {
|
|
21
|
-
beforeEach(() => {
|
|
22
|
-
vi.clearAllMocks();
|
|
23
|
-
});
|
|
24
|
-
const mockConfig = {
|
|
25
|
-
chunkSize: 100,
|
|
26
|
-
chunkOverlap: 10,
|
|
27
|
-
combineSentencesIntoChunks: true,
|
|
28
|
-
sploderMaxSize: 500,
|
|
29
|
-
modelProvider: 'mock',
|
|
30
|
-
modelName: 'text-embedding-3-small',
|
|
31
|
-
vectorStoreType: "simple",
|
|
32
|
-
storagePath: './storage',
|
|
33
|
-
projectName: 'test_project',
|
|
34
|
-
splitIntoSentences: true,
|
|
35
|
-
};
|
|
36
|
-
const mockSettings = {
|
|
37
|
-
openAIKey: 'mock-api-key',
|
|
38
|
-
oLlamaBaseURL: 'http://localhost',
|
|
39
|
-
azureOpenAIKey: null,
|
|
40
|
-
azureOpenAIEndpoint: null,
|
|
41
|
-
azureOpenAIApiVersion: null,
|
|
42
|
-
mistralApiKey: null,
|
|
43
|
-
geminiApiKey: null,
|
|
44
|
-
};
|
|
45
|
-
it('should process documents and return nodes', async () => {
|
|
46
|
-
const mockDocuments = [
|
|
47
|
-
new Document({ text: 'Document 1', metadata: { key1: 'value1' } }),
|
|
48
|
-
new Document({ text: 'Document 2', metadata: { key2: 'value2' } }),
|
|
49
|
-
];
|
|
50
|
-
const mockNodes = [
|
|
51
|
-
new TextNode({ text: 'Document 1' }),
|
|
52
|
-
new TextNode({ text: 'Document 2' }),
|
|
53
|
-
];
|
|
54
|
-
const result = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings);
|
|
55
|
-
expect(result.map((node) => node.text)).toEqual(mockNodes.map((node) => node.text));
|
|
56
|
-
});
|
|
57
|
-
it('should filter out documents with null, undefined, or zero-length text', async () => {
|
|
58
|
-
const mockDocuments = [
|
|
59
|
-
new Document({ text: 'Valid Document', metadata: { key1: 'value1' } }),
|
|
60
|
-
new Document({ text: undefined, metadata: { key3: 'value3' } }),
|
|
61
|
-
new Document({ text: '', metadata: { key4: 'value4' } }),
|
|
62
|
-
];
|
|
63
|
-
const filteredDocuments = [mockDocuments[0]];
|
|
64
|
-
const mockNodes = [new TextNode({ text: 'Valid Document' })];
|
|
65
|
-
// (transformDocumentsToNodes as vi.Mock).mockResolvedValue(mockNodes);
|
|
66
|
-
const result = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings);
|
|
67
|
-
expect(result.map((n) => n.text)).toEqual(mockNodes.map((n) => n.text));
|
|
68
|
-
// TODO: I can't get these to work. Apparently you can't spyOn a function that is imported from the same file.
|
|
69
|
-
// all well and good but ... why did CoPilot generate a test that can't work?
|
|
70
|
-
// expect(transformDocumentsToNodes).toHaveBeenCalledWith(filteredDocuments, expect.any(Array));
|
|
71
|
-
});
|
|
72
|
-
it('should exclude all metadata keys from embedding', async () => {
|
|
73
|
-
const mockDocuments = [
|
|
74
|
-
new Document({ text: 'Document 1', metadata: { key1: 'value1', key2: 'value2' } }),
|
|
75
|
-
];
|
|
76
|
-
const nodes = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings);
|
|
77
|
-
expect(nodes[0].excludedEmbedMetadataKeys).toEqual(['key1', 'key2']);
|
|
78
|
-
});
|
|
79
|
-
});
|
|
80
|
-
describe('getEmbedModel', () => {
|
|
81
|
-
const mockConfig = {
|
|
82
|
-
chunkSize: 100,
|
|
83
|
-
chunkOverlap: 10,
|
|
84
|
-
combineSentencesIntoChunks: true,
|
|
85
|
-
sploderMaxSize: 500,
|
|
86
|
-
modelProvider: 'openai',
|
|
87
|
-
modelName: 'text-embedding-3-small',
|
|
88
|
-
vectorStoreType: "simple",
|
|
89
|
-
storagePath: './storage',
|
|
90
|
-
projectName: 'test_project',
|
|
91
|
-
splitIntoSentences: true,
|
|
92
|
-
};
|
|
93
|
-
const mockSettings = {
|
|
94
|
-
openAIKey: 'mock-api-key',
|
|
95
|
-
oLlamaBaseURL: 'http://localhost',
|
|
96
|
-
azureOpenAIKey: null,
|
|
97
|
-
azureOpenAIEndpoint: null,
|
|
98
|
-
azureOpenAIApiVersion: null,
|
|
99
|
-
mistralApiKey: null,
|
|
100
|
-
geminiApiKey: null,
|
|
101
|
-
};
|
|
102
|
-
it('should handle different model providers correctly', () => {
|
|
103
|
-
// Test with 'ollama' provider
|
|
104
|
-
const ollamaModel = getEmbedModel({ ...mockConfig, modelProvider: 'ollama' }, mockSettings);
|
|
105
|
-
expect(ollamaModel).toBeDefined();
|
|
106
|
-
// Test with 'mock' provider
|
|
107
|
-
const mockModel = getEmbedModel({ ...mockConfig, modelProvider: 'mock' }, mockSettings);
|
|
108
|
-
expect(mockModel).toBeDefined();
|
|
109
|
-
// Test with invalid provider
|
|
110
|
-
expect(() => {
|
|
111
|
-
getEmbedModel({ ...mockConfig, modelProvider: 'invalid' }, mockSettings);
|
|
112
|
-
}).toThrow('Unsupported embedding model provider: invalid');
|
|
113
|
-
});
|
|
114
|
-
});
|
|
115
|
-
//# sourceMappingURL=embeddings.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.test.js","sourceRoot":"","sources":["../../src/services/embeddings.test.ts"],"names":[],"mappings":"AAAA,aAAa;AAEb,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEhD,qDAAqD;AACrD,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE;IACvD,MAAM,MAAM,GAAG,MAAM,cAAc,EAAE,CAAA;IACrC,OAAO;QACL,GAAG,MAAM;QACT,sBAAsB;QACtB,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;QACrB,2BAA2B,EAAE,EAAE,CAAC,EAAE,EAAE;QACpC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;QACrB,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE;QACzB,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE;QAC5B,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE;KACzB,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,kCAAkC;AAClC,OAAO,EAAE,yBAAyB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAExE,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG;QACjB,SAAS,EAAE,GAAG;QACd,YAAY,EAAE,EAAE;QAChB,0BAA0B,EAAE,IAAI;QAChC,cAAc,EAAE,GAAG;QACnB,aAAa,EAAE,MAAM;QACrB,SAAS,EAAE,wBAAwB;QACnC,eAAe,EAAE,QAAoB;QACrC,WAAW,EAAE,WAAW;QACxB,WAAW,EAAE,cAAc;QAC3B,kBAAkB,EAAE,IAAI;KACzB,CAAC;IAEF,MAAM,YAAY,GAAG;QACnB,SAAS,EAAE,cAAc;QACzB,aAAa,EAAE,kBAAkB;QACjC,cAAc,EAAE,IAAI;QACpB,mBAAmB,EAAE,IAAI;QACzB,qBAAqB,EAAE,IAAI;QAC3B,aAAa,EAAE,IAAI;QACnB,YAAY,EAAE,IAAI;KACnB,CAAC;IAEF,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,aAAa,GAAG;YACpB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;YAClE,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;SACnE,CAAC;QACF,MAAM,SAAS,GAAG;YAChB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;YACpC,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;SACrC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;QAExF,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uEAAuE,EAAE,KAAK,IAAI,EAAE;QACrF,MAAM,aAAa,GAAG;YACpB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;YACtE,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC/D,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;SACzD,CAAC;QACF,MAAM,iBAAiB,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC;QAE7D,uEAAuE;QAEvE,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;QACxF,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAExE,8GAA8G;QAC9G,6EAA6E;QAC7E,gGAAgG;IAClG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,aAAa,GAAG;YACpB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;SACnF,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAC,CAAA;QACtF,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,MAAM,UAAU,GAAG;QACjB,SAAS,EAAE,GAAG;QACd,YAAY,EAAE,EAAE;QAChB,0BAA0B,EAAE,IAAI;QAChC,cAAc,EAAE,GAAG;QACnB,aAAa,EAAE,QAAQ;QACvB,SAAS,EAAE,wBAAwB;QACnC,eAAe,EAAE,QAAoB;QACrC,WAAW,EAAE,WAAW;QACxB,WAAW,EAAE,cAAc;QAC3B,kBAAkB,EAAE,IAAI;KACzB,CAAC;IAEF,MAAM,YAAY,GAAG;QACnB,SAAS,EAAE,cAAc;QACzB,aAAa,EAAE,kBAAkB;QACjC,cAAc,EAAE,IAAI;QACpB,mBAAmB,EAAE,IAAI;QACzB,qBAAqB,EAAE,IAAI;QAC3B,aAAa,EAAE,IAAI;QACnB,YAAY,EAAE,IAAI;KACnB,CAAC;IAGF,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,8BAA8B;QAC9B,MAAM,WAAW,GAAG,aAAa,CAC/B,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,QAAQ,EAAE,EAC1C,YAAY,CACb,CAAC;QACF,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;QAElC,4BAA4B;QAC5B,MAAM,SAAS,GAAG,aAAa,CAC7B,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,MAAM,EAAE,EACxC,YAAY,CACb,CAAC;QACF,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QAEhC,6BAA6B;QAC7B,MAAM,CAAC,GAAG,EAAE;YACV,aAAa,CACX,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,SAAgB,EAAE,EAClD,YAAY,CACb,CAAC;QACJ,CAAC,CAAC,CAAC,OAAO,CAAC,+CAA+C,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sentenceSplitter.test.d.ts","sourceRoot":"","sources":["../../src/services/sentenceSplitter.test.ts"],"names":[],"mappings":""}
|