@meaningfully/core 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Meaningfully.d.ts +1 -1
- package/package.json +3 -1
- package/{tests → src/__tests__}/MetadataManager.test.ts +1 -1
- package/{tests → src/api/__tests__}/embedding.test.ts +17 -17
- package/{tests → src/services/__tests__}/csvLoader.test.ts +1 -1
- package/{tests → src/services/__tests__}/embeddings.test.ts +2 -2
- package/{tests → src/services/__tests__}/sentenceSplitter.test.ts +1 -1
- package/tsconfig.json +1 -1
- package/dist/api/embedding.test.d.ts +0 -2
- package/dist/api/embedding.test.d.ts.map +0 -1
- package/dist/api/embedding.test.js +0 -340
- package/dist/api/embedding.test.js.map +0 -1
- package/dist/services/csvLoader.test.d.ts +0 -2
- package/dist/services/csvLoader.test.d.ts.map +0 -1
- package/dist/services/csvLoader.test.js +0 -75
- package/dist/services/csvLoader.test.js.map +0 -1
- package/dist/services/embeddings.test.d.ts +0 -2
- package/dist/services/embeddings.test.d.ts.map +0 -1
- package/dist/services/embeddings.test.js +0 -115
- package/dist/services/embeddings.test.js.map +0 -1
- package/dist/services/sentenceSplitter.test.d.ts +0 -2
- package/dist/services/sentenceSplitter.test.d.ts.map +0 -1
- package/dist/services/sentenceSplitter.test.js +0 -68
- package/dist/services/sentenceSplitter.test.js.map +0 -1
- package/src/api/embedding.d.ts +0 -6
- package/tests/csvLoader.test.d.ts +0 -1
- package/tests/embedding.test.d.ts +0 -1
- package/tests/embeddings.test.d.ts +0 -1
- package/tests/sentenceSplitter.test.d.ts +0 -1
package/dist/Meaningfully.d.ts
CHANGED
|
@@ -23,7 +23,7 @@ export declare class MeaningfullyAPI {
|
|
|
23
23
|
deleteDocumentSet(documentSetId: number): Promise<{
|
|
24
24
|
success: boolean;
|
|
25
25
|
}>;
|
|
26
|
-
getVectorStoreType(): "
|
|
26
|
+
getVectorStoreType(): "simple" | "weaviate";
|
|
27
27
|
generatePreviewData(data: DocumentSetParamsFilePath): Promise<import("./types/index.js").PreviewResult>;
|
|
28
28
|
uploadCsv(data: DocumentSetParamsFilePath): Promise<{
|
|
29
29
|
success: boolean;
|
package/package.json
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@meaningfully/core",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Core functionality for meaningfully semantic search",
|
|
5
|
+
"repo": "https://github.com/jeremybmerrill/meaningfully-core",
|
|
5
6
|
"license": "MIT",
|
|
6
7
|
"type": "module",
|
|
7
8
|
"main": "./dist/index.js",
|
|
@@ -15,6 +16,7 @@
|
|
|
15
16
|
},
|
|
16
17
|
"scripts": {
|
|
17
18
|
"build": "tsc",
|
|
19
|
+
"prepare": "npm run build",
|
|
18
20
|
"watch": "tsc --watch",
|
|
19
21
|
"test": "vitest --silent=false --disable-console-intercept"
|
|
20
22
|
},
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
//@ts-nocheck
|
|
2
2
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
3
|
-
import { createEmbeddings, previewResults, getDocStore, getIndex, search } from '../
|
|
4
|
-
import { loadDocumentsFromCsv } from '
|
|
5
|
-
import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, getExistingDocStore } from '
|
|
3
|
+
import { createEmbeddings, previewResults, getDocStore, getIndex, search } from '../embedding.js';
|
|
4
|
+
import { loadDocumentsFromCsv } from '../../services/csvLoader.js';
|
|
5
|
+
import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, getExistingDocStore } from '../../services/embeddings.js';
|
|
6
6
|
import { MetadataMode } from 'llamaindex';
|
|
7
7
|
|
|
8
8
|
// filepath: /Users/jeremybmerrill/code/meaningfully/src/main/api/embedding.test.ts
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
vi.mock('
|
|
12
|
-
vi.mock('
|
|
11
|
+
vi.mock('../../services/csvLoader');
|
|
12
|
+
vi.mock('../../services/embeddings');
|
|
13
13
|
|
|
14
14
|
describe('embedding.ts', () => {
|
|
15
15
|
describe('createEmbeddings', () => {
|
|
@@ -262,7 +262,7 @@ describe('embedding.ts', () => {
|
|
|
262
262
|
|
|
263
263
|
it('should correctly track progress through ProgressManager', async () => {
|
|
264
264
|
// Setup
|
|
265
|
-
vi.mock('
|
|
265
|
+
vi.mock('../../services/progressManager', () => {
|
|
266
266
|
const mockInstance = {
|
|
267
267
|
startOperation: vi.fn(),
|
|
268
268
|
updateProgress: vi.fn(),
|
|
@@ -278,8 +278,8 @@ describe('embedding.ts', () => {
|
|
|
278
278
|
});
|
|
279
279
|
|
|
280
280
|
// Re-import to use mocked version
|
|
281
|
-
const { createEmbeddings } = await import('../
|
|
282
|
-
const { ProgressManager } = await import('
|
|
281
|
+
const { createEmbeddings } = await import('../embedding.js');
|
|
282
|
+
const { ProgressManager } = await import('../../services/progressManager.js');
|
|
283
283
|
|
|
284
284
|
const mockDocuments = [{ text: 'doc1' }, { text: 'doc2' }];
|
|
285
285
|
const mockNodes = [{ text: 'node1', metadata: {} }];
|
|
@@ -305,7 +305,7 @@ describe('embedding.ts', () => {
|
|
|
305
305
|
|
|
306
306
|
it('should properly calculate percentage in progress callback', async () => {
|
|
307
307
|
// Setup mocks with spy on updateProgress
|
|
308
|
-
vi.mock('
|
|
308
|
+
vi.mock('../../services/progressManager', () => {
|
|
309
309
|
const mockInstance = {
|
|
310
310
|
startOperation: vi.fn(),
|
|
311
311
|
updateProgress: vi.fn(),
|
|
@@ -321,8 +321,8 @@ describe('embedding.ts', () => {
|
|
|
321
321
|
});
|
|
322
322
|
|
|
323
323
|
// Re-import to use mocked version
|
|
324
|
-
const { createEmbeddings } = await import('../
|
|
325
|
-
const { ProgressManager } = await import('
|
|
324
|
+
const { createEmbeddings } = await import('../embedding.js');
|
|
325
|
+
const { ProgressManager } = await import('../../services/progressManager.js');
|
|
326
326
|
|
|
327
327
|
const mockDocuments = [{ text: 'doc1' }];
|
|
328
328
|
const mockNodes = [{ text: 'node1', metadata: {} }];
|
|
@@ -355,7 +355,7 @@ describe('embedding.ts', () => {
|
|
|
355
355
|
|
|
356
356
|
it('should clear operation on empty documents', async () => {
|
|
357
357
|
// Setup
|
|
358
|
-
vi.mock('
|
|
358
|
+
vi.mock('../../services/progressManager', () => {
|
|
359
359
|
const mockInstance = {
|
|
360
360
|
startOperation: vi.fn(),
|
|
361
361
|
updateProgress: vi.fn(),
|
|
@@ -371,8 +371,8 @@ describe('embedding.ts', () => {
|
|
|
371
371
|
});
|
|
372
372
|
|
|
373
373
|
// Re-import to use mocked version
|
|
374
|
-
const { createEmbeddings } = await import('../
|
|
375
|
-
const { ProgressManager } = await import('
|
|
374
|
+
const { createEmbeddings } = await import('../embedding.js');
|
|
375
|
+
const { ProgressManager } = await import('../../services/progressManager.js');
|
|
376
376
|
|
|
377
377
|
loadDocumentsFromCsv.mockResolvedValue([]);
|
|
378
378
|
|
|
@@ -390,7 +390,7 @@ describe('embedding.ts', () => {
|
|
|
390
390
|
|
|
391
391
|
it('shoulde complete operation on successful embedding', async () => {
|
|
392
392
|
// Setup
|
|
393
|
-
vi.mock('
|
|
393
|
+
vi.mock('../../services/progressManager', () => {
|
|
394
394
|
const mockInstance = {
|
|
395
395
|
startOperation: vi.fn(),
|
|
396
396
|
updateProgress: vi.fn(),
|
|
@@ -406,8 +406,8 @@ describe('embedding.ts', () => {
|
|
|
406
406
|
});
|
|
407
407
|
|
|
408
408
|
// Re-import to use mocked version
|
|
409
|
-
const { createEmbeddings } = await import('../
|
|
410
|
-
const { ProgressManager } = await import('
|
|
409
|
+
const { createEmbeddings } = await import('../embedding.js');
|
|
410
|
+
const { ProgressManager } = await import('../../services/progressManager.js');
|
|
411
411
|
|
|
412
412
|
const mockDocuments = [{ text: 'doc1' }];
|
|
413
413
|
const mockNodes = [{ text: 'node1', metadata: {} }];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
//@ts-nocheck
|
|
2
2
|
import { describe, it, expect, vi } from 'vitest';
|
|
3
3
|
import { readFileSync } from 'fs';
|
|
4
|
-
import { loadDocumentsFromCsv } from '../
|
|
4
|
+
import { loadDocumentsFromCsv } from '../csvLoader.js';
|
|
5
5
|
import { Document } from 'llamaindex';
|
|
6
6
|
import Papa from 'papaparse';
|
|
7
7
|
|
|
@@ -4,7 +4,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
|
4
4
|
import { Document, TextNode } from 'llamaindex';
|
|
5
5
|
|
|
6
6
|
// First, set up the mock before importing the module
|
|
7
|
-
vi.mock(import("../
|
|
7
|
+
vi.mock(import("../embeddings.js"), async (importOriginal) => {
|
|
8
8
|
const actual = await importOriginal()
|
|
9
9
|
return {
|
|
10
10
|
...actual,
|
|
@@ -19,7 +19,7 @@ vi.mock(import("../src/services/embeddings.js"), async (importOriginal) => {
|
|
|
19
19
|
})
|
|
20
20
|
|
|
21
21
|
// Now import the mocked functions
|
|
22
|
-
import { transformDocumentsToNodes, getEmbedModel } from '../
|
|
22
|
+
import { transformDocumentsToNodes, getEmbedModel } from '../embeddings.js';
|
|
23
23
|
|
|
24
24
|
describe('transformDocumentsToNodes', () => {
|
|
25
25
|
beforeEach(() => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
//@ts-nocheck
|
|
2
2
|
import { expect, test } from 'vitest'
|
|
3
|
-
import { CustomSentenceSplitter } from '../
|
|
3
|
+
import { CustomSentenceSplitter } from '../sentenceSplitter.js'
|
|
4
4
|
import { SentenceSplitter, IngestionPipeline, Document } from "llamaindex";
|
|
5
5
|
|
|
6
6
|
// do these tests just to make sure that we can factor out my hacky fixes when llamaindex is fixed.
|
package/tsconfig.json
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embedding.test.d.ts","sourceRoot":"","sources":["../../src/api/embedding.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,340 +0,0 @@
|
|
|
1
|
-
//@ts-nocheck
|
|
2
|
-
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
3
|
-
import { createEmbeddings, previewResults, getDocStore, getIndex, search } from './embedding';
|
|
4
|
-
import { loadDocumentsFromCsv } from '../services/csvLoader';
|
|
5
|
-
import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, getExistingDocStore } from '../services/embeddings';
|
|
6
|
-
// filepath: /Users/jeremybmerrill/code/meaningfully/src/main/api/embedding.test.ts
|
|
7
|
-
vi.mock('../services/csvLoader');
|
|
8
|
-
vi.mock('../services/embeddings');
|
|
9
|
-
describe('embedding.ts', () => {
|
|
10
|
-
describe('createEmbeddings', () => {
|
|
11
|
-
it('should create embeddings and return success', async () => {
|
|
12
|
-
const mockDocuments = [{ text: 'doc1' }, { text: 'doc2' }];
|
|
13
|
-
const mockNodes = [{ node: 'node1' }, { node: 'node2' }];
|
|
14
|
-
const mockIndex = 'index1';
|
|
15
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
16
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
17
|
-
persistNodes.mockResolvedValue(mockIndex);
|
|
18
|
-
const result = await createEmbeddings('path/to/csv', 'text', {}, {});
|
|
19
|
-
expect(result).toEqual({ success: true, index: mockIndex });
|
|
20
|
-
});
|
|
21
|
-
it('should return error on failure', async () => {
|
|
22
|
-
loadDocumentsFromCsv.mockRejectedValue(new Error('Failed to load documents'));
|
|
23
|
-
const result = await createEmbeddings('path/to/csv', 'text', {}, {});
|
|
24
|
-
expect(result).toEqual({ success: false, error: 'Failed to load documents' });
|
|
25
|
-
});
|
|
26
|
-
it('should handle empty documents', async () => {
|
|
27
|
-
loadDocumentsFromCsv.mockResolvedValue([]);
|
|
28
|
-
const result = await createEmbeddings('path/to/csv', 'text', {}, {});
|
|
29
|
-
expect(result).toEqual({ success: false, error: 'That CSV does not appear to contain any documents. Please check the file and try again.' });
|
|
30
|
-
});
|
|
31
|
-
});
|
|
32
|
-
describe('previewResults', () => {
|
|
33
|
-
it('should return preview results and estimated cost', async () => {
|
|
34
|
-
const mockDocuments = Array(20).fill({ text: 'doc' });
|
|
35
|
-
const mockNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
|
|
36
|
-
const mockPreviewNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
|
|
37
|
-
const mockEstimate = { estimatedPrice: 10, tokenCount: 100, pricePer1M: 0.01 };
|
|
38
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
39
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
40
|
-
estimateCost.mockReturnValue(mockEstimate);
|
|
41
|
-
const result = await previewResults('path/to/csv', 'text', {});
|
|
42
|
-
expect(result).toEqual({
|
|
43
|
-
success: true,
|
|
44
|
-
nodes: mockPreviewNodes,
|
|
45
|
-
...mockEstimate
|
|
46
|
-
});
|
|
47
|
-
});
|
|
48
|
-
it('should return error on failure', async () => {
|
|
49
|
-
loadDocumentsFromCsv.mockRejectedValue(new Error('Failed to load documents'));
|
|
50
|
-
const result = await previewResults('path/to/csv', 'text', {});
|
|
51
|
-
expect(result).toEqual({ success: false, error: 'Failed to load documents' });
|
|
52
|
-
});
|
|
53
|
-
it('should handle empty documents', async () => {
|
|
54
|
-
loadDocumentsFromCsv.mockResolvedValue([]);
|
|
55
|
-
const result = await previewResults('path/to/csv', 'text', {});
|
|
56
|
-
expect(result).toEqual({ success: false, error: 'That CSV does not appear to contain any documents. Please check the file and try again.' });
|
|
57
|
-
});
|
|
58
|
-
});
|
|
59
|
-
describe('getDocStore', () => {
|
|
60
|
-
it('should return existing doc store', async () => {
|
|
61
|
-
const mockDocStore = 'docStore';
|
|
62
|
-
getExistingDocStore.mockResolvedValue(mockDocStore);
|
|
63
|
-
const result = await getDocStore({});
|
|
64
|
-
expect(result).toBe(mockDocStore);
|
|
65
|
-
});
|
|
66
|
-
});
|
|
67
|
-
describe('getIndex', () => {
|
|
68
|
-
it('should return existing vector store index', async () => {
|
|
69
|
-
const mockIndex = 'index';
|
|
70
|
-
getExistingVectorStoreIndex.mockResolvedValue(mockIndex);
|
|
71
|
-
const result = await getIndex({}, {});
|
|
72
|
-
expect(result).toBe(mockIndex);
|
|
73
|
-
});
|
|
74
|
-
});
|
|
75
|
-
describe('search', () => {
|
|
76
|
-
it('should return search results', async () => {
|
|
77
|
-
const mockResults = [
|
|
78
|
-
{ node: { getContent: () => 'content1', metadata: {} }, score: 1 },
|
|
79
|
-
{ node: { getContent: () => 'content2', metadata: {} }, score: 2 }
|
|
80
|
-
];
|
|
81
|
-
searchDocuments.mockResolvedValue(mockResults);
|
|
82
|
-
const result = await search('index', 'query');
|
|
83
|
-
expect(result).toEqual([
|
|
84
|
-
{ text: 'content1', score: 1, metadata: {} },
|
|
85
|
-
{ text: 'content2', score: 2, metadata: {} }
|
|
86
|
-
]);
|
|
87
|
-
});
|
|
88
|
-
it('should handle no search results', async () => {
|
|
89
|
-
searchDocuments.mockResolvedValue([]);
|
|
90
|
-
const result = await search('index', 'query');
|
|
91
|
-
expect(result).toEqual([]);
|
|
92
|
-
});
|
|
93
|
-
it('should handle search results with null scores', async () => {
|
|
94
|
-
const mockResults = [
|
|
95
|
-
{ node: { getContent: () => 'content1', metadata: {} }, score: null },
|
|
96
|
-
{ node: { getContent: () => 'content2', metadata: {} }, score: null }
|
|
97
|
-
];
|
|
98
|
-
searchDocuments.mockResolvedValue(mockResults);
|
|
99
|
-
const result = await search('index', 'query');
|
|
100
|
-
expect(result).toEqual([
|
|
101
|
-
{ text: 'content1', score: 0, metadata: {} },
|
|
102
|
-
{ text: 'content2', score: 0, metadata: {} }
|
|
103
|
-
]);
|
|
104
|
-
});
|
|
105
|
-
});
|
|
106
|
-
});
|
|
107
|
-
describe('previewResults', () => {
|
|
108
|
-
it('should return preview results and estimated cost', async () => {
|
|
109
|
-
const mockDocuments = Array(20).fill({ text: 'doc' });
|
|
110
|
-
const mockNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
|
|
111
|
-
const mockPreviewNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
|
|
112
|
-
const mockEstimate = { estimatedPrice: 10, tokenCount: 100, pricePer1M: 0.01 };
|
|
113
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
114
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
115
|
-
estimateCost.mockReturnValue(mockEstimate);
|
|
116
|
-
const result = await previewResults('path/to/csv', 'text', {});
|
|
117
|
-
expect(result).toEqual({
|
|
118
|
-
success: true,
|
|
119
|
-
nodes: mockPreviewNodes,
|
|
120
|
-
...mockEstimate
|
|
121
|
-
});
|
|
122
|
-
});
|
|
123
|
-
it('should return error on failure', async () => {
|
|
124
|
-
loadDocumentsFromCsv.mockRejectedValue(new Error('Failed to load documents'));
|
|
125
|
-
const result = await previewResults('path/to/csv', 'text', {});
|
|
126
|
-
expect(result).toEqual({ success: false, error: 'Failed to load documents' });
|
|
127
|
-
});
|
|
128
|
-
});
|
|
129
|
-
describe('getDocStore', () => {
|
|
130
|
-
it('should return existing doc store', async () => {
|
|
131
|
-
const mockDocStore = 'docStore';
|
|
132
|
-
getExistingDocStore.mockResolvedValue(mockDocStore);
|
|
133
|
-
const result = await getDocStore({});
|
|
134
|
-
expect(result).toBe(mockDocStore);
|
|
135
|
-
});
|
|
136
|
-
});
|
|
137
|
-
describe('getIndex', () => {
|
|
138
|
-
it('should return existing vector store index', async () => {
|
|
139
|
-
const mockIndex = 'index';
|
|
140
|
-
getExistingVectorStoreIndex.mockResolvedValue(mockIndex);
|
|
141
|
-
const result = await getIndex({}, {});
|
|
142
|
-
expect(result).toBe(mockIndex);
|
|
143
|
-
});
|
|
144
|
-
});
|
|
145
|
-
describe('search', () => {
|
|
146
|
-
it('should return search results', async () => {
|
|
147
|
-
const mockResults = [
|
|
148
|
-
{ node: { getContent: () => 'content1', metadata: {} }, score: 1 },
|
|
149
|
-
{ node: { getContent: () => 'content2', metadata: {} }, score: 2 }
|
|
150
|
-
];
|
|
151
|
-
searchDocuments.mockResolvedValue(mockResults);
|
|
152
|
-
const result = await search('index', 'query');
|
|
153
|
-
expect(result).toEqual([
|
|
154
|
-
{ text: 'content1', score: 1, metadata: {} },
|
|
155
|
-
{ text: 'content2', score: 2, metadata: {} }
|
|
156
|
-
]);
|
|
157
|
-
});
|
|
158
|
-
it('should handle no search results', async () => {
|
|
159
|
-
searchDocuments.mockResolvedValue([]);
|
|
160
|
-
const result = await search('index', 'query');
|
|
161
|
-
expect(result).toEqual([]);
|
|
162
|
-
});
|
|
163
|
-
it('should handle search results with null scores', async () => {
|
|
164
|
-
const mockResults = [
|
|
165
|
-
{ node: { getContent: () => 'content1', metadata: {} }, score: null },
|
|
166
|
-
{ node: { getContent: () => 'content2', metadata: {} }, score: null }
|
|
167
|
-
];
|
|
168
|
-
searchDocuments.mockResolvedValue(mockResults);
|
|
169
|
-
const result = await search('index', 'query');
|
|
170
|
-
expect(result).toEqual([
|
|
171
|
-
{ text: 'content1', score: 0, metadata: {} },
|
|
172
|
-
{ text: 'content2', score: 0, metadata: {} }
|
|
173
|
-
]);
|
|
174
|
-
});
|
|
175
|
-
});
|
|
176
|
-
describe('createEmbeddings with progress tracking', () => {
|
|
177
|
-
beforeEach(() => {
|
|
178
|
-
vi.clearAllMocks();
|
|
179
|
-
});
|
|
180
|
-
it('should pass progress callback to persistNodes', async () => {
|
|
181
|
-
// Setup mocks
|
|
182
|
-
const mockDocuments = [{ text: 'doc1' }, { text: 'doc2' }];
|
|
183
|
-
const mockNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
|
|
184
|
-
const mockIndex = 'index1';
|
|
185
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
186
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
187
|
-
persistNodes.mockImplementation((nodes, config, settings, clients, callback) => {
|
|
188
|
-
// Call the callback with sample progress
|
|
189
|
-
if (callback) {
|
|
190
|
-
callback(1, 2); // 50% progress
|
|
191
|
-
callback(2, 2); // 100% progress
|
|
192
|
-
}
|
|
193
|
-
return Promise.resolve(mockIndex);
|
|
194
|
-
});
|
|
195
|
-
// Execute
|
|
196
|
-
const result = await createEmbeddings('path/to/csv', 'text', { modelName: 'test-model' }, {}, {});
|
|
197
|
-
// Verify
|
|
198
|
-
expect(persistNodes).toHaveBeenCalledTimes(1);
|
|
199
|
-
expect(persistNodes.mock.calls[0][4]).toBeInstanceOf(Function); // Verify callback was passed
|
|
200
|
-
expect(result).toEqual({ success: true, index: mockIndex });
|
|
201
|
-
});
|
|
202
|
-
it('should correctly track progress through ProgressManager', async () => {
|
|
203
|
-
// Setup
|
|
204
|
-
vi.mock('../services/progressManager', () => {
|
|
205
|
-
const mockInstance = {
|
|
206
|
-
startOperation: vi.fn(),
|
|
207
|
-
updateProgress: vi.fn(),
|
|
208
|
-
completeOperation: vi.fn(),
|
|
209
|
-
clearOperation: vi.fn()
|
|
210
|
-
};
|
|
211
|
-
return {
|
|
212
|
-
ProgressManager: {
|
|
213
|
-
getInstance: () => mockInstance
|
|
214
|
-
}
|
|
215
|
-
};
|
|
216
|
-
});
|
|
217
|
-
// Re-import to use mocked version
|
|
218
|
-
const { createEmbeddings } = await import('./embedding');
|
|
219
|
-
const { ProgressManager } = await import('../services/progressManager');
|
|
220
|
-
const mockDocuments = [{ text: 'doc1' }, { text: 'doc2' }];
|
|
221
|
-
const mockNodes = [{ text: 'node1', metadata: {} }];
|
|
222
|
-
const mockIndex = 'testIndex';
|
|
223
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
224
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
225
|
-
persistNodes.mockImplementation((nodes, config, settings, clients, callback) => {
|
|
226
|
-
if (callback)
|
|
227
|
-
callback(1, 2); // Call with 50% progress
|
|
228
|
-
return Promise.resolve(mockIndex);
|
|
229
|
-
});
|
|
230
|
-
// Execute
|
|
231
|
-
await createEmbeddings('path/to/csv', 'text', {}, {}, {});
|
|
232
|
-
// Verify
|
|
233
|
-
const progressManager = ProgressManager.getInstance();
|
|
234
|
-
expect(progressManager.startOperation).toHaveBeenCalledWith(expect.stringMatching(/^embed-\d+$/), 100);
|
|
235
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 5);
|
|
236
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), expect.any(Number));
|
|
237
|
-
expect(progressManager.completeOperation).toHaveBeenCalledWith(expect.any(String));
|
|
238
|
-
});
|
|
239
|
-
it('should properly calculate percentage in progress callback', async () => {
|
|
240
|
-
// Setup mocks with spy on updateProgress
|
|
241
|
-
vi.mock('../services/progressManager', () => {
|
|
242
|
-
const mockInstance = {
|
|
243
|
-
startOperation: vi.fn(),
|
|
244
|
-
updateProgress: vi.fn(),
|
|
245
|
-
completeOperation: vi.fn(),
|
|
246
|
-
clearOperation: vi.fn()
|
|
247
|
-
};
|
|
248
|
-
return {
|
|
249
|
-
ProgressManager: {
|
|
250
|
-
getInstance: () => mockInstance
|
|
251
|
-
}
|
|
252
|
-
};
|
|
253
|
-
});
|
|
254
|
-
// Re-import to use mocked version
|
|
255
|
-
const { createEmbeddings } = await import('./embedding');
|
|
256
|
-
const { ProgressManager } = await import('../services/progressManager');
|
|
257
|
-
const mockDocuments = [{ text: 'doc1' }];
|
|
258
|
-
const mockNodes = [{ text: 'node1', metadata: {} }];
|
|
259
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
260
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
261
|
-
// Simulate persistNodes calling the callback with various progress values
|
|
262
|
-
persistNodes.mockImplementation((nodes, config, settings, clients, callback) => {
|
|
263
|
-
if (callback) {
|
|
264
|
-
callback(0, 10); // 0% progress
|
|
265
|
-
callback(5, 10); // 50% progress
|
|
266
|
-
callback(10, 10); // 100% progress
|
|
267
|
-
}
|
|
268
|
-
return Promise.resolve('mockIndex');
|
|
269
|
-
});
|
|
270
|
-
// Execute
|
|
271
|
-
await createEmbeddings('path/to/csv', 'text', {}, {}, {});
|
|
272
|
-
// Verify percentage calculations
|
|
273
|
-
// Initial update at 5%
|
|
274
|
-
const progressManager = ProgressManager.getInstance();
|
|
275
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 5);
|
|
276
|
-
// Progress updates: 0%, 50%, 100% mapped to 5-95% range
|
|
277
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 5); // 0% -> 5%
|
|
278
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 50); // 50% -> 50%
|
|
279
|
-
expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 95); // 100% -> 95%
|
|
280
|
-
});
|
|
281
|
-
it('should clear operation on empty documents', async () => {
|
|
282
|
-
// Setup
|
|
283
|
-
vi.mock('../services/progressManager', () => {
|
|
284
|
-
const mockInstance = {
|
|
285
|
-
startOperation: vi.fn(),
|
|
286
|
-
updateProgress: vi.fn(),
|
|
287
|
-
completeOperation: vi.fn(),
|
|
288
|
-
clearOperation: vi.fn()
|
|
289
|
-
};
|
|
290
|
-
return {
|
|
291
|
-
ProgressManager: {
|
|
292
|
-
getInstance: () => mockInstance
|
|
293
|
-
}
|
|
294
|
-
};
|
|
295
|
-
});
|
|
296
|
-
// Re-import to use mocked version
|
|
297
|
-
const { createEmbeddings } = await import('./embedding');
|
|
298
|
-
const { ProgressManager } = await import('../services/progressManager');
|
|
299
|
-
loadDocumentsFromCsv.mockResolvedValue([]);
|
|
300
|
-
// Execute
|
|
301
|
-
const result = await createEmbeddings('path/to/csv', 'text', {}, {}, {});
|
|
302
|
-
// Verify
|
|
303
|
-
const progressManager = ProgressManager.getInstance();
|
|
304
|
-
expect(progressManager.clearOperation).toHaveBeenCalled();
|
|
305
|
-
expect(result).toEqual({
|
|
306
|
-
success: false,
|
|
307
|
-
error: "That CSV does not appear to contain any documents. Please check the file and try again."
|
|
308
|
-
});
|
|
309
|
-
});
|
|
310
|
-
it('shoulde complete operation on successful embedding', async () => {
|
|
311
|
-
// Setup
|
|
312
|
-
vi.mock('../services/progressManager', () => {
|
|
313
|
-
const mockInstance = {
|
|
314
|
-
startOperation: vi.fn(),
|
|
315
|
-
updateProgress: vi.fn(),
|
|
316
|
-
completeOperation: vi.fn(),
|
|
317
|
-
clearOperation: vi.fn()
|
|
318
|
-
};
|
|
319
|
-
return {
|
|
320
|
-
ProgressManager: {
|
|
321
|
-
getInstance: () => mockInstance
|
|
322
|
-
}
|
|
323
|
-
};
|
|
324
|
-
});
|
|
325
|
-
// Re-import to use mocked version
|
|
326
|
-
const { createEmbeddings } = await import('./embedding');
|
|
327
|
-
const { ProgressManager } = await import('../services/progressManager');
|
|
328
|
-
const mockDocuments = [{ text: 'doc1' }];
|
|
329
|
-
const mockNodes = [{ text: 'node1', metadata: {} }];
|
|
330
|
-
loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
|
|
331
|
-
transformDocumentsToNodes.mockResolvedValue(mockNodes);
|
|
332
|
-
persistNodes.mockResolvedValue('mockIndex');
|
|
333
|
-
// Execute
|
|
334
|
-
await createEmbeddings('path/to/csv', 'text', {}, {}, {});
|
|
335
|
-
// Verify
|
|
336
|
-
const progressManager = ProgressManager.getInstance();
|
|
337
|
-
expect(progressManager.completeOperation).toHaveBeenCalled();
|
|
338
|
-
});
|
|
339
|
-
});
|
|
340
|
-
//# sourceMappingURL=embedding.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embedding.test.js","sourceRoot":"","sources":["../../src/api/embedding.test.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC9F,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAGlK,mFAAmF;AAGnF,EAAE,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;AACjC,EAAE,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;AAElC,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC1B,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACzD,MAAM,SAAS,GAAG,QAAQ,CAAC;YAC3B,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvD,YAAY,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAE1C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC5C,oBAAoB,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;YAE9E,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC3C,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,yFAAyF,EAAE,CAAC,CAAC;QACjJ,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,aAAa,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACtD,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YACrF,MAAM,gBAAgB,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC5F,MAAM,YAAY,GAAG,EAAE,cAAc,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;YAC/E,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvD,YAAY,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACnB,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,gBAAgB;gBACvB,GAAG,YAAY;aAClB,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC5C,oBAAoB,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;YAE9E,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC3C,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,yFAAyF,EAAE,CAAC,CAAC;QACjJ,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,YAAY,GAAG,UAAU,CAAC;YAChC,mBAAmB,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAEpD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,CAAC;YAErC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,SAAS,GAAG,OAAO,CAAC;YAC1B,2BAA2B,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEzD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAEtC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACpB,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,WAAW,GAAG;gBAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;gBAClE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;aACrE,CAAC;YACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;gBAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC7C,eAAe,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAEtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,WAAW,GAAG;gBAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;gBACrE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;aACxE,CAAC;YACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;gBAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAED,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,aAAa,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,MAAM,gBAAgB,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5F,MAAM,YAAY,GAAG,EAAE,cAAc,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QAC/E,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,gBAAgB;YACvB,GAAG,YAAY;SAChB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,oBAAoB,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;QAE9E,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,YAAY,GAAG,UAAU,CAAC;QAChC,mBAAmB,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,CAAC;QAErC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;IACxB,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,OAAO,CAAC;QAC1B,2BAA2B,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAEzD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAEtC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AACD,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;IACpB,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,WAAW,GAAG;YAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;YAClE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;SACrE,CAAC;QACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;SAC/C,CAAC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;QACjD,eAAe,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAEtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,WAAW,GAAG;YAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;YACrE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SACxE,CAAC;QACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;SAC/C,CAAC,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AACH,QAAQ,CAAC,yCAAyC,EAAE,GAAG,EAAE;IACvD,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC7D,cAAc;QACd,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,MAAM,SAAS,GAAG,QAAQ,CAAC;QAC3B,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;YAC7E,yCAAyC;YACzC,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,eAAe;gBAChC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,gBAAgB;YACnC,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,UAAU;QACV,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAElG,SAAS;QACT,MAAM,CAAC,YAAY,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;QAC7F,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACvE,QAAQ;QACR,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACpD,MAAM,SAAS,GAAG,WAAW,CAAC;QAE9B,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;YAC7E,IAAI,QAAQ;gBAAE,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,yBAAyB;YACvD,OAAO,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,UAAU;QACV,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1D,SAAS;QACT,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC,aAAa,CAAC,EAAE,GAAG,CAAC,CAAC;QACvG,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACpG,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACrF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,yCAAyC;QACzC,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACpD,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAEvD,0EAA0E;QAC1E,YAAY,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;YAC7E,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,cAAc;gBAChC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,eAAe;gBACjC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB;YACpC,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,UAAU;QACV,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1D,iCAAiC;QACjC,uBAAuB;QACvB,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAEnF,wDAAwD;QACxD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAG,WAAW;QACjG,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,aAAa;QACnG,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,cAAc;IACtG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,QAAQ;QACR,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAE3C,UAAU;QACV,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAEzE,SAAS;QACT,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,yFAAyF;SACjG,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,QAAQ;QACR,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACpD,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAE5C,UAAU;QACV,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1D,SAAS;QACT,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,gBAAgB,EAAE,CAAC;IAC/D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"csvLoader.test.d.ts","sourceRoot":"","sources":["../../src/services/csvLoader.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
//@ts-nocheck
|
|
2
|
-
import { describe, it, expect, vi } from 'vitest';
|
|
3
|
-
import { readFileSync } from 'fs';
|
|
4
|
-
import { loadDocumentsFromCsv } from './csvLoader';
|
|
5
|
-
import { Document } from 'llamaindex';
|
|
6
|
-
import Papa from 'papaparse';
|
|
7
|
-
vi.mock('fs');
|
|
8
|
-
vi.mock('papaparse');
|
|
9
|
-
describe('csvLoader.ts', () => {
|
|
10
|
-
describe('loadDocumentsFromCsv', () => {
|
|
11
|
-
it('should load documents from CSV and return Document instances', async () => {
|
|
12
|
-
const mockFileContent = 'text,metadata1,metadata2\ncontent1,meta1,meta2\ncontent2,meta3,meta4';
|
|
13
|
-
const mockParsedData = {
|
|
14
|
-
data: [
|
|
15
|
-
{ text: 'content1', metadata1: 'meta1', metadata2: 'meta2' },
|
|
16
|
-
{ text: 'content2', metadata1: 'meta3', metadata2: 'meta4' }
|
|
17
|
-
]
|
|
18
|
-
};
|
|
19
|
-
readFileSync.mockReturnValue(mockFileContent);
|
|
20
|
-
Papa.parse.mockReturnValue(mockParsedData);
|
|
21
|
-
const result = await loadDocumentsFromCsv('path/to/csv', 'text');
|
|
22
|
-
expect(remove_id(result)).toEqual(remove_id([
|
|
23
|
-
new Document({ text: 'content1', metadata: { metadata1: 'meta1', metadata2: 'meta2' } }),
|
|
24
|
-
new Document({ text: 'content2', metadata: { metadata1: 'meta3', metadata2: 'meta4' } })
|
|
25
|
-
]));
|
|
26
|
-
});
|
|
27
|
-
it('should handle empty CSV file', async () => {
|
|
28
|
-
const mockFileContent = '';
|
|
29
|
-
const mockParsedData = { data: [] };
|
|
30
|
-
readFileSync.mockReturnValue(mockFileContent);
|
|
31
|
-
Papa.parse.mockReturnValue(mockParsedData);
|
|
32
|
-
const result = await loadDocumentsFromCsv('path/to/csv', 'text');
|
|
33
|
-
expect(result).toEqual([]);
|
|
34
|
-
});
|
|
35
|
-
it('should handle missing text column', async () => {
|
|
36
|
-
const mockFileContent = 'metadata1,metadata2\nmeta1,meta2\nmeta3,meta4';
|
|
37
|
-
const mockParsedData = {
|
|
38
|
-
data: [
|
|
39
|
-
{ metadata1: 'meta1', metadata2: 'meta2' },
|
|
40
|
-
{ metadata1: 'meta3', metadata2: 'meta4' }
|
|
41
|
-
]
|
|
42
|
-
};
|
|
43
|
-
readFileSync.mockReturnValue(mockFileContent);
|
|
44
|
-
Papa.parse.mockReturnValue(mockParsedData);
|
|
45
|
-
const result = await loadDocumentsFromCsv('path/to/csv', 'text');
|
|
46
|
-
expect(remove_id(result)).toEqual(remove_id([
|
|
47
|
-
new Document({ text: undefined, metadata: { metadata1: 'meta1', metadata2: 'meta2' } }),
|
|
48
|
-
new Document({ text: undefined, metadata: { metadata1: 'meta3', metadata2: 'meta4' } })
|
|
49
|
-
]));
|
|
50
|
-
});
|
|
51
|
-
it('should handle null values in metadata', async () => {
|
|
52
|
-
const mockFileContent = 'text,metadata1,metadata2\ncontent1,,meta2\ncontent2,meta3,';
|
|
53
|
-
const mockParsedData = {
|
|
54
|
-
data: [
|
|
55
|
-
{ text: 'content1', metadata1: null, metadata2: 'meta2' },
|
|
56
|
-
{ text: 'content2', metadata1: 'meta3', metadata2: null }
|
|
57
|
-
]
|
|
58
|
-
};
|
|
59
|
-
readFileSync.mockReturnValue(mockFileContent);
|
|
60
|
-
Papa.parse.mockReturnValue(mockParsedData);
|
|
61
|
-
const result = await loadDocumentsFromCsv('path/to/csv', 'text');
|
|
62
|
-
expect(remove_id(result)).toEqual(remove_id([
|
|
63
|
-
new Document({ text: 'content1', metadata: { metadata1: '', metadata2: 'meta2' } }),
|
|
64
|
-
new Document({ text: 'content2', metadata: { metadata1: 'meta3', metadata2: '' } })
|
|
65
|
-
]));
|
|
66
|
-
});
|
|
67
|
-
});
|
|
68
|
-
});
|
|
69
|
-
function remove_id(list_of_documents) {
|
|
70
|
-
return list_of_documents.map((doc) => {
|
|
71
|
-
const { id_, ...doc_without_id } = doc;
|
|
72
|
-
return doc_without_id;
|
|
73
|
-
});
|
|
74
|
-
}
|
|
75
|
-
//# sourceMappingURL=csvLoader.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"csvLoader.test.js","sourceRoot":"","sources":["../../src/services/csvLoader.test.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAErB,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,eAAe,GAAG,sEAAsE,CAAC;YAC/F,MAAM,cAAc,GAAG;gBACrB,IAAI,EAAE;oBACJ,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;oBAC5D,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;iBAC7D;aACF,CAAC;YACF,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEjE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;gBAC1C,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;gBACxF,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;aACzF,CAAC,CAAC,CAAC;QACN,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,eAAe,GAAG,EAAE,CAAC;YAC3B,MAAM,cAAc,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;YACpC,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEjE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,eAAe,GAAG,+CAA+C,CAAC;YACxE,MAAM,cAAc,GAAG;gBACrB,IAAI,EAAE;oBACJ,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;oBAC1C,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;iBAC3C;aACF,CAAC;YACF,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEjE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;gBAC1C,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;gBACvF,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;aACxF,CAAC,CAAC,CAAC;QACN,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,eAAe,GAAG,4DAA4D,CAAC;YACrF,MAAM,cAAc,GAAG;gBACnB,IAAI,EAAE;oBACN,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE;oBACzD,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE;iBACxD;aACJ,CAAC;YACF,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YACjE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;gBACxC,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;gBACnF,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,CAAC;aACtF,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,SAAS,SAAS,CAAC,iBAAiB;IAClC,OAAO,iBAAiB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACnC,MAAM,EAAE,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,GAAG,CAAC;QACvC,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACH,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.test.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
//@ts-nocheck
|
|
2
|
-
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
3
|
-
import { Document, TextNode } from 'llamaindex';
|
|
4
|
-
// First, set up the mock before importing the module
|
|
5
|
-
vi.mock(import("./embeddings"), async (importOriginal) => {
|
|
6
|
-
const actual = await importOriginal();
|
|
7
|
-
return {
|
|
8
|
-
...actual,
|
|
9
|
-
// your mocked methods
|
|
10
|
-
estimateCost: vi.fn(),
|
|
11
|
-
getExistingVectorStoreIndex: vi.fn(),
|
|
12
|
-
persistNodes: vi.fn(),
|
|
13
|
-
persistDocuments: vi.fn(),
|
|
14
|
-
getExistingDocStore: vi.fn(),
|
|
15
|
-
searchDocuments: vi.fn()
|
|
16
|
-
};
|
|
17
|
-
});
|
|
18
|
-
// Now import the mocked functions
|
|
19
|
-
import { transformDocumentsToNodes, getEmbedModel } from './embeddings';
|
|
20
|
-
describe('transformDocumentsToNodes', () => {
|
|
21
|
-
beforeEach(() => {
|
|
22
|
-
vi.clearAllMocks();
|
|
23
|
-
});
|
|
24
|
-
const mockConfig = {
|
|
25
|
-
chunkSize: 100,
|
|
26
|
-
chunkOverlap: 10,
|
|
27
|
-
combineSentencesIntoChunks: true,
|
|
28
|
-
sploderMaxSize: 500,
|
|
29
|
-
modelProvider: 'mock',
|
|
30
|
-
modelName: 'text-embedding-3-small',
|
|
31
|
-
vectorStoreType: "simple",
|
|
32
|
-
storagePath: './storage',
|
|
33
|
-
projectName: 'test_project',
|
|
34
|
-
splitIntoSentences: true,
|
|
35
|
-
};
|
|
36
|
-
const mockSettings = {
|
|
37
|
-
openAIKey: 'mock-api-key',
|
|
38
|
-
oLlamaBaseURL: 'http://localhost',
|
|
39
|
-
azureOpenAIKey: null,
|
|
40
|
-
azureOpenAIEndpoint: null,
|
|
41
|
-
azureOpenAIApiVersion: null,
|
|
42
|
-
mistralApiKey: null,
|
|
43
|
-
geminiApiKey: null,
|
|
44
|
-
};
|
|
45
|
-
it('should process documents and return nodes', async () => {
|
|
46
|
-
const mockDocuments = [
|
|
47
|
-
new Document({ text: 'Document 1', metadata: { key1: 'value1' } }),
|
|
48
|
-
new Document({ text: 'Document 2', metadata: { key2: 'value2' } }),
|
|
49
|
-
];
|
|
50
|
-
const mockNodes = [
|
|
51
|
-
new TextNode({ text: 'Document 1' }),
|
|
52
|
-
new TextNode({ text: 'Document 2' }),
|
|
53
|
-
];
|
|
54
|
-
const result = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings);
|
|
55
|
-
expect(result.map((node) => node.text)).toEqual(mockNodes.map((node) => node.text));
|
|
56
|
-
});
|
|
57
|
-
it('should filter out documents with null, undefined, or zero-length text', async () => {
|
|
58
|
-
const mockDocuments = [
|
|
59
|
-
new Document({ text: 'Valid Document', metadata: { key1: 'value1' } }),
|
|
60
|
-
new Document({ text: undefined, metadata: { key3: 'value3' } }),
|
|
61
|
-
new Document({ text: '', metadata: { key4: 'value4' } }),
|
|
62
|
-
];
|
|
63
|
-
const filteredDocuments = [mockDocuments[0]];
|
|
64
|
-
const mockNodes = [new TextNode({ text: 'Valid Document' })];
|
|
65
|
-
// (transformDocumentsToNodes as vi.Mock).mockResolvedValue(mockNodes);
|
|
66
|
-
const result = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings);
|
|
67
|
-
expect(result.map((n) => n.text)).toEqual(mockNodes.map((n) => n.text));
|
|
68
|
-
// TODO: I can't get these to work. Apparently you can't spyOn a function that is imported from the same file.
|
|
69
|
-
// all well and good but ... why did CoPilot generate a test that can't work?
|
|
70
|
-
// expect(transformDocumentsToNodes).toHaveBeenCalledWith(filteredDocuments, expect.any(Array));
|
|
71
|
-
});
|
|
72
|
-
it('should exclude all metadata keys from embedding', async () => {
|
|
73
|
-
const mockDocuments = [
|
|
74
|
-
new Document({ text: 'Document 1', metadata: { key1: 'value1', key2: 'value2' } }),
|
|
75
|
-
];
|
|
76
|
-
const nodes = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings);
|
|
77
|
-
expect(nodes[0].excludedEmbedMetadataKeys).toEqual(['key1', 'key2']);
|
|
78
|
-
});
|
|
79
|
-
});
|
|
80
|
-
describe('getEmbedModel', () => {
|
|
81
|
-
const mockConfig = {
|
|
82
|
-
chunkSize: 100,
|
|
83
|
-
chunkOverlap: 10,
|
|
84
|
-
combineSentencesIntoChunks: true,
|
|
85
|
-
sploderMaxSize: 500,
|
|
86
|
-
modelProvider: 'openai',
|
|
87
|
-
modelName: 'text-embedding-3-small',
|
|
88
|
-
vectorStoreType: "simple",
|
|
89
|
-
storagePath: './storage',
|
|
90
|
-
projectName: 'test_project',
|
|
91
|
-
splitIntoSentences: true,
|
|
92
|
-
};
|
|
93
|
-
const mockSettings = {
|
|
94
|
-
openAIKey: 'mock-api-key',
|
|
95
|
-
oLlamaBaseURL: 'http://localhost',
|
|
96
|
-
azureOpenAIKey: null,
|
|
97
|
-
azureOpenAIEndpoint: null,
|
|
98
|
-
azureOpenAIApiVersion: null,
|
|
99
|
-
mistralApiKey: null,
|
|
100
|
-
geminiApiKey: null,
|
|
101
|
-
};
|
|
102
|
-
it('should handle different model providers correctly', () => {
|
|
103
|
-
// Test with 'ollama' provider
|
|
104
|
-
const ollamaModel = getEmbedModel({ ...mockConfig, modelProvider: 'ollama' }, mockSettings);
|
|
105
|
-
expect(ollamaModel).toBeDefined();
|
|
106
|
-
// Test with 'mock' provider
|
|
107
|
-
const mockModel = getEmbedModel({ ...mockConfig, modelProvider: 'mock' }, mockSettings);
|
|
108
|
-
expect(mockModel).toBeDefined();
|
|
109
|
-
// Test with invalid provider
|
|
110
|
-
expect(() => {
|
|
111
|
-
getEmbedModel({ ...mockConfig, modelProvider: 'invalid' }, mockSettings);
|
|
112
|
-
}).toThrow('Unsupported embedding model provider: invalid');
|
|
113
|
-
});
|
|
114
|
-
});
|
|
115
|
-
//# sourceMappingURL=embeddings.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.test.js","sourceRoot":"","sources":["../../src/services/embeddings.test.ts"],"names":[],"mappings":"AAAA,aAAa;AAEb,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEhD,qDAAqD;AACrD,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE;IACvD,MAAM,MAAM,GAAG,MAAM,cAAc,EAAE,CAAA;IACrC,OAAO;QACL,GAAG,MAAM;QACT,sBAAsB;QACtB,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;QACrB,2BAA2B,EAAE,EAAE,CAAC,EAAE,EAAE;QACpC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;QACrB,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE;QACzB,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE;QAC5B,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE;KACzB,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,kCAAkC;AAClC,OAAO,EAAE,yBAAyB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAExE,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG;QACjB,SAAS,EAAE,GAAG;QACd,YAAY,EAAE,EAAE;QAChB,0BAA0B,EAAE,IAAI;QAChC,cAAc,EAAE,GAAG;QACnB,aAAa,EAAE,MAAM;QACrB,SAAS,EAAE,wBAAwB;QACnC,eAAe,EAAE,QAAoB;QACrC,WAAW,EAAE,WAAW;QACxB,WAAW,EAAE,cAAc;QAC3B,kBAAkB,EAAE,IAAI;KACzB,CAAC;IAEF,MAAM,YAAY,GAAG;QACnB,SAAS,EAAE,cAAc;QACzB,aAAa,EAAE,kBAAkB;QACjC,cAAc,EAAE,IAAI;QACpB,mBAAmB,EAAE,IAAI;QACzB,qBAAqB,EAAE,IAAI;QAC3B,aAAa,EAAE,IAAI;QACnB,YAAY,EAAE,IAAI;KACnB,CAAC;IAEF,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,aAAa,GAAG;YACpB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;YAClE,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;SACnE,CAAC;QACF,MAAM,SAAS,GAAG;YAChB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;YACpC,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;SACrC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;QAExF,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uEAAuE,EAAE,KAAK,IAAI,EAAE;QACrF,MAAM,aAAa,GAAG;YACpB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;YACtE,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC/D,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;SACzD,CAAC;QACF,MAAM,iBAAiB,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC;QAE7D,uEAAuE;QAEvE,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;QACxF,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAExE,8GAA8G;QAC9G,6EAA6E;QAC7E,gGAAgG;IAClG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,aAAa,GAAG;YACpB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;SACnF,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAC,CAAA;QACtF,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,MAAM,UAAU,GAAG;QACjB,SAAS,EAAE,GAAG;QACd,YAAY,EAAE,EAAE;QAChB,0BAA0B,EAAE,IAAI;QAChC,cAAc,EAAE,GAAG;QACnB,aAAa,EAAE,QAAQ;QACvB,SAAS,EAAE,wBAAwB;QACnC,eAAe,EAAE,QAAoB;QACrC,WAAW,EAAE,WAAW;QACxB,WAAW,EAAE,cAAc;QAC3B,kBAAkB,EAAE,IAAI;KACzB,CAAC;IAEF,MAAM,YAAY,GAAG;QACnB,SAAS,EAAE,cAAc;QACzB,aAAa,EAAE,kBAAkB;QACjC,cAAc,EAAE,IAAI;QACpB,mBAAmB,EAAE,IAAI;QACzB,qBAAqB,EAAE,IAAI;QAC3B,aAAa,EAAE,IAAI;QACnB,YAAY,EAAE,IAAI;KACnB,CAAC;IAGF,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,8BAA8B;QAC9B,MAAM,WAAW,GAAG,aAAa,CAC/B,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,QAAQ,EAAE,EAC1C,YAAY,CACb,CAAC;QACF,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;QAElC,4BAA4B;QAC5B,MAAM,SAAS,GAAG,aAAa,CAC7B,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,MAAM,EAAE,EACxC,YAAY,CACb,CAAC;QACF,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QAEhC,6BAA6B;QAC7B,MAAM,CAAC,GAAG,EAAE;YACV,aAAa,CACX,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,SAAgB,EAAE,EAClD,YAAY,CACb,CAAC;QACJ,CAAC,CAAC,CAAC,OAAO,CAAC,+CAA+C,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sentenceSplitter.test.d.ts","sourceRoot":"","sources":["../../src/services/sentenceSplitter.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
//@ts-nocheck
|
|
2
|
-
import { expect, test } from 'vitest';
|
|
3
|
-
import { CustomSentenceSplitter } from './sentenceSplitter';
|
|
4
|
-
import { SentenceSplitter, IngestionPipeline, Document } from "llamaindex";
|
|
5
|
-
// do these tests just to make sure that we can factor out my hacky fixes when llamaindex is fixed.
|
|
6
|
-
// test that original sentenceSplitter splits on abbreviations
|
|
7
|
-
// test that original sentenceSplitter splits on abbreviations even when specified
|
|
8
|
-
// test that my modified sentenceSplitter excludes metadata when arg is specified
|
|
9
|
-
// test that my modified sentenceSplitter includes metadata when arg is specified the other way
|
|
10
|
-
let documents = [
|
|
11
|
-
new Document({ text: "JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee. Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019. He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time." }),
|
|
12
|
-
];
|
|
13
|
-
let originalSentenceSplitterPipeline = new IngestionPipeline({
|
|
14
|
-
transformations: [
|
|
15
|
-
new SentenceSplitter({ chunkSize: 50, chunkOverlap: 10 }),
|
|
16
|
-
],
|
|
17
|
-
});
|
|
18
|
-
let customSentenceSplitterPipeline = new IngestionPipeline({
|
|
19
|
-
transformations: [
|
|
20
|
-
new CustomSentenceSplitter({ chunkSize: 50, chunkOverlap: 10 }),
|
|
21
|
-
],
|
|
22
|
-
});
|
|
23
|
-
test("my modified sentenceSplitter doesn't eliminate spaces", () => {
|
|
24
|
-
customSentenceSplitterPipeline.run({ documents: documents }).then((nodes) => {
|
|
25
|
-
expect(nodes.some((node) => node["text"].indexOf("Co.elected") > -1)).toEqual(false);
|
|
26
|
-
expect(nodes.some((node) => node["text"].indexOf("Mr.Weinberger") > -1)).toEqual(false);
|
|
27
|
-
expect(nodes.some((node) => node["text"].indexOf("A.and") > -1)).toEqual(false);
|
|
28
|
-
});
|
|
29
|
-
});
|
|
30
|
-
// test("original sentenceSplitter does eliminate spaces", () => {
|
|
31
|
-
// originalSentenceSplitterPipeline.run({documents: documents}).then((nodes) => {
|
|
32
|
-
// expect(nodes.some((node) => node["text"].indexOf("Co.elected") > -1)).toEqual(true);
|
|
33
|
-
// expect(nodes.some((node) => node["text"].indexOf("Mr.Weinberger") > -1)).toEqual(true);
|
|
34
|
-
// expect(nodes.some((node) => node["text"].indexOf("A.and") > -1)).toEqual(true);
|
|
35
|
-
// });
|
|
36
|
-
// });
|
|
37
|
-
let noAbbrevsCustomSentenceSplitterPipeline = new IngestionPipeline({
|
|
38
|
-
transformations: [
|
|
39
|
-
new CustomSentenceSplitter({ chunkSize: 50, chunkOverlap: 10, abbreviations: [] }),
|
|
40
|
-
],
|
|
41
|
-
});
|
|
42
|
-
test("my modified sentenceSplitter doesn't split on specified abbreviations", () => {
|
|
43
|
-
customSentenceSplitterPipeline.run({ documents: documents }).then((nodes) => {
|
|
44
|
-
expect(nodes.map((node) => !!node["text"].match(/Mr\.$/))).not.toContainEqual(true);
|
|
45
|
-
});
|
|
46
|
-
});
|
|
47
|
-
// this is only a problem on branch fix/sentence-splitter-spaces
|
|
48
|
-
// where the chunker is eliminated entirely in favor of just splitting by sentences with natural.
|
|
49
|
-
test("original sentenceSplitter splits in silly places, like Mr", () => {
|
|
50
|
-
noAbbrevsCustomSentenceSplitterPipeline.run({ documents: documents }).then((nodes) => {
|
|
51
|
-
expect(nodes.map((node) => !!node["text"].match(/Mr\.$/))).toContainEqual(true);
|
|
52
|
-
});
|
|
53
|
-
});
|
|
54
|
-
const testcases = [
|
|
55
|
-
["USA v. 4227 JENIFER STREET N.W. WASHINGTON, D.C., AND ELECTRONIC DEVICES THEREIN UNDER RULE 41", "USA v. 4227 JENIFER STREET N.W. WASHINGTON, D.C., AND ELECTRONIC DEVICES THEREIN UNDER RULE 41"],
|
|
56
|
-
["JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee.", "JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee."],
|
|
57
|
-
["Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019.", "Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019."],
|
|
58
|
-
["He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time.", "He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time."],
|
|
59
|
-
];
|
|
60
|
-
testcases.forEach(([testcase_input, testcase_expected_output]) => {
|
|
61
|
-
test(`my sentenceSplitter correctly handles short sentence ${testcase_input}`, () => {
|
|
62
|
-
customSentenceSplitterPipeline.run({ documents: [new Document({ text: testcase_input })] }).then((nodes) => {
|
|
63
|
-
expect(nodes.length).toEqual(1);
|
|
64
|
-
expect(nodes[0]["text"]).toEqual(testcase_expected_output);
|
|
65
|
-
});
|
|
66
|
-
});
|
|
67
|
-
});
|
|
68
|
-
//# sourceMappingURL=sentenceSplitter.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sentenceSplitter.test.js","sourceRoot":"","sources":["../../src/services/sentenceSplitter.test.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAA;AACrC,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAA;AAC3D,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3E,mGAAmG;AACnG,8DAA8D;AAC9D,kFAAkF;AAElF,iFAAiF;AACjF,+FAA+F;AAI/F,IAAI,SAAS,GAAG;IACZ,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,oeAAoe,EAAE,CAAC;CAC/f,CAAC;AAEF,IAAI,gCAAgC,GAAG,IAAI,iBAAiB,CAAC;IACzD,eAAe,EAAE;QACb,IAAI,gBAAgB,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;KACxD;CACJ,CAAC,CAAC;AACP,IAAI,8BAA8B,GAAG,IAAI,iBAAiB,CAAC;IACvD,eAAe,EAAE;QACf,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;KAChE;CACF,CAAC,CAAC;AAEL,IAAI,CAAC,uDAAuD,EAAE,GAAG,EAAE;IAC/D,8BAA8B,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;QACtE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACxF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,kEAAkE;AAClE,qFAAqF;AACrF,+FAA+F;AAC/F,kGAAkG;AAClG,0FAA0F;AAC1F,UAAU;AACV,MAAM;AAEN,IAAI,uCAAuC,GAAG,IAAI,iBAAiB,CAAC;IAChE,eAAe,EAAE;QACf,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAC,CAAC;KAClF;CACF,CAAC,CAAC;AAGH,IAAI,CAAC,uEAAuE,EAAE,GAAG,EAAE;IACjF,8BAA8B,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;QACtE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IACxF,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,gEAAgE;AAChE,iGAAiG;AACjG,IAAI,CAAC,2DAA2D,EAAE,GAAG,EAAE;IACnE,uCAAuC,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;QAC/E,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG;IACd,CAAC,gGAAgG,EAAE,gGAAgG,CAAC;IACpM,CAAC,sKAAsK,EAAE,sKAAsK,CAAC;IAChV,CAAC,oGAAoG,EAAE,oGAAoG,CAAC;IAC5M,CAAC,0NAA0N,EAAE,0NAA0N,CAAC;CAE3b,CAAC;AACF,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,EAAE,wBAAwB,CAAC,EAAE,EAAE;IAC7D,IAAI,CAAC,wDAAwD,cAAc,EAAE,EAAE,GAAG,EAAE;QAChF,8BAA8B,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,CAAC,IAAI,QAAQ,CAAC,EAAC,IAAI,EAAE,cAAc,EAAC,CAAC,CAAC,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACnG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAA;AACN,CAAC,CAAC,CAAC"}
|
package/src/api/embedding.d.ts
DELETED
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import type { EmbeddingConfig, EmbeddingResult, SearchResult, PreviewResult, Settings, MetadataFilter, Clients } from "../types";
|
|
2
|
-
export declare function createEmbeddings(csvPath: string, textColumnName: string, config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<EmbeddingResult>;
|
|
3
|
-
export declare function previewResults(csvPath: string, textColumnName: string, config: EmbeddingConfig): Promise<PreviewResult>;
|
|
4
|
-
export declare function getDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
|
|
5
|
-
export declare function getIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<import("llamaindex").VectorStoreIndex>;
|
|
6
|
-
export declare function search(index: any, query: string, numResults?: number, filters?: MetadataFilter[]): Promise<SearchResult[]>;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|