@opensaas/stack-rag 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -0
- package/CHANGELOG.md +10 -0
- package/CLAUDE.md +565 -0
- package/LICENSE +21 -0
- package/README.md +406 -0
- package/dist/config/index.d.ts +63 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +94 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/plugin.d.ts +38 -0
- package/dist/config/plugin.d.ts.map +1 -0
- package/dist/config/plugin.js +215 -0
- package/dist/config/plugin.js.map +1 -0
- package/dist/config/plugin.test.d.ts +2 -0
- package/dist/config/plugin.test.d.ts.map +1 -0
- package/dist/config/plugin.test.js +554 -0
- package/dist/config/plugin.test.js.map +1 -0
- package/dist/config/types.d.ts +249 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +5 -0
- package/dist/config/types.js.map +1 -0
- package/dist/fields/embedding.d.ts +85 -0
- package/dist/fields/embedding.d.ts.map +1 -0
- package/dist/fields/embedding.js +81 -0
- package/dist/fields/embedding.js.map +1 -0
- package/dist/fields/embedding.test.d.ts +2 -0
- package/dist/fields/embedding.test.d.ts.map +1 -0
- package/dist/fields/embedding.test.js +323 -0
- package/dist/fields/embedding.test.js.map +1 -0
- package/dist/fields/index.d.ts +6 -0
- package/dist/fields/index.d.ts.map +1 -0
- package/dist/fields/index.js +5 -0
- package/dist/fields/index.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +9 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp/index.d.ts +19 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +18 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/providers/index.d.ts +38 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +68 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/ollama.d.ts +49 -0
- package/dist/providers/ollama.d.ts.map +1 -0
- package/dist/providers/ollama.js +151 -0
- package/dist/providers/ollama.js.map +1 -0
- package/dist/providers/openai.d.ts +41 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +126 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/providers.test.d.ts +2 -0
- package/dist/providers/providers.test.d.ts.map +1 -0
- package/dist/providers/providers.test.js +224 -0
- package/dist/providers/providers.test.js.map +1 -0
- package/dist/providers/types.d.ts +88 -0
- package/dist/providers/types.d.ts.map +1 -0
- package/dist/providers/types.js +2 -0
- package/dist/providers/types.js.map +1 -0
- package/dist/runtime/batch.d.ts +183 -0
- package/dist/runtime/batch.d.ts.map +1 -0
- package/dist/runtime/batch.js +240 -0
- package/dist/runtime/batch.js.map +1 -0
- package/dist/runtime/batch.test.d.ts +2 -0
- package/dist/runtime/batch.test.d.ts.map +1 -0
- package/dist/runtime/batch.test.js +251 -0
- package/dist/runtime/batch.test.js.map +1 -0
- package/dist/runtime/chunking.d.ts +42 -0
- package/dist/runtime/chunking.d.ts.map +1 -0
- package/dist/runtime/chunking.js +264 -0
- package/dist/runtime/chunking.js.map +1 -0
- package/dist/runtime/chunking.test.d.ts +2 -0
- package/dist/runtime/chunking.test.d.ts.map +1 -0
- package/dist/runtime/chunking.test.js +212 -0
- package/dist/runtime/chunking.test.js.map +1 -0
- package/dist/runtime/embeddings.d.ts +147 -0
- package/dist/runtime/embeddings.d.ts.map +1 -0
- package/dist/runtime/embeddings.js +201 -0
- package/dist/runtime/embeddings.js.map +1 -0
- package/dist/runtime/embeddings.test.d.ts +2 -0
- package/dist/runtime/embeddings.test.d.ts.map +1 -0
- package/dist/runtime/embeddings.test.js +366 -0
- package/dist/runtime/embeddings.test.js.map +1 -0
- package/dist/runtime/index.d.ts +14 -0
- package/dist/runtime/index.d.ts.map +1 -0
- package/dist/runtime/index.js +18 -0
- package/dist/runtime/index.js.map +1 -0
- package/dist/runtime/search.d.ts +135 -0
- package/dist/runtime/search.d.ts.map +1 -0
- package/dist/runtime/search.js +101 -0
- package/dist/runtime/search.js.map +1 -0
- package/dist/storage/index.d.ts +41 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +73 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/json.d.ts +34 -0
- package/dist/storage/json.d.ts.map +1 -0
- package/dist/storage/json.js +82 -0
- package/dist/storage/json.js.map +1 -0
- package/dist/storage/pgvector.d.ts +53 -0
- package/dist/storage/pgvector.d.ts.map +1 -0
- package/dist/storage/pgvector.js +168 -0
- package/dist/storage/pgvector.js.map +1 -0
- package/dist/storage/sqlite-vss.d.ts +49 -0
- package/dist/storage/sqlite-vss.d.ts.map +1 -0
- package/dist/storage/sqlite-vss.js +148 -0
- package/dist/storage/sqlite-vss.js.map +1 -0
- package/dist/storage/storage.test.d.ts +2 -0
- package/dist/storage/storage.test.d.ts.map +1 -0
- package/dist/storage/storage.test.js +440 -0
- package/dist/storage/storage.test.js.map +1 -0
- package/dist/storage/types.d.ts +79 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +49 -0
- package/dist/storage/types.js.map +1 -0
- package/package.json +82 -0
- package/src/config/index.ts +116 -0
- package/src/config/plugin.test.ts +664 -0
- package/src/config/plugin.ts +257 -0
- package/src/config/types.ts +283 -0
- package/src/fields/embedding.test.ts +408 -0
- package/src/fields/embedding.ts +150 -0
- package/src/fields/index.ts +6 -0
- package/src/index.ts +33 -0
- package/src/mcp/index.ts +21 -0
- package/src/providers/index.ts +81 -0
- package/src/providers/ollama.ts +186 -0
- package/src/providers/openai.ts +161 -0
- package/src/providers/providers.test.ts +275 -0
- package/src/providers/types.ts +100 -0
- package/src/runtime/batch.test.ts +332 -0
- package/src/runtime/batch.ts +424 -0
- package/src/runtime/chunking.test.ts +258 -0
- package/src/runtime/chunking.ts +334 -0
- package/src/runtime/embeddings.test.ts +441 -0
- package/src/runtime/embeddings.ts +380 -0
- package/src/runtime/index.ts +51 -0
- package/src/runtime/search.ts +243 -0
- package/src/storage/index.ts +86 -0
- package/src/storage/json.ts +106 -0
- package/src/storage/pgvector.ts +206 -0
- package/src/storage/sqlite-vss.ts +193 -0
- package/src/storage/storage.test.ts +521 -0
- package/src/storage/types.ts +126 -0
- package/tsconfig.json +13 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/vitest.config.ts +18 -0
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
+
import { batchProcess, RateLimiter, ProcessingQueue, } from './batch.js';
|
|
3
|
+
// Mock embedding provider
|
|
4
|
+
function createMockProvider(delayMs = 0) {
|
|
5
|
+
return {
|
|
6
|
+
type: 'mock',
|
|
7
|
+
model: 'mock-model',
|
|
8
|
+
dimensions: 3,
|
|
9
|
+
embed: vi.fn(async (text) => {
|
|
10
|
+
if (delayMs > 0)
|
|
11
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
12
|
+
return [text.length / 10, text.length / 20, text.length / 30];
|
|
13
|
+
}),
|
|
14
|
+
embedBatch: vi.fn(async (texts) => {
|
|
15
|
+
if (delayMs > 0)
|
|
16
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
17
|
+
return texts.map((text) => [text.length / 10, text.length / 20, text.length / 30]);
|
|
18
|
+
}),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
describe('batchProcess', () => {
|
|
22
|
+
beforeEach(() => {
|
|
23
|
+
vi.clearAllMocks();
|
|
24
|
+
});
|
|
25
|
+
it('should process all texts successfully', async () => {
|
|
26
|
+
const provider = createMockProvider();
|
|
27
|
+
const texts = ['Text 1', 'Text 2', 'Text 3'];
|
|
28
|
+
const result = await batchProcess({
|
|
29
|
+
provider,
|
|
30
|
+
texts,
|
|
31
|
+
batchSize: 2,
|
|
32
|
+
});
|
|
33
|
+
expect(result.embeddings).toHaveLength(3);
|
|
34
|
+
expect(result.failed).toHaveLength(0);
|
|
35
|
+
expect(result.stats.successful).toBe(3);
|
|
36
|
+
expect(result.stats.failed).toBe(0);
|
|
37
|
+
});
|
|
38
|
+
it('should respect batch size', async () => {
|
|
39
|
+
const provider = createMockProvider();
|
|
40
|
+
const texts = Array(10).fill('Test');
|
|
41
|
+
await batchProcess({
|
|
42
|
+
provider,
|
|
43
|
+
texts,
|
|
44
|
+
batchSize: 3,
|
|
45
|
+
});
|
|
46
|
+
// Should be called 4 times: 3 + 3 + 3 + 1
|
|
47
|
+
expect(provider.embedBatch).toHaveBeenCalledTimes(4);
|
|
48
|
+
});
|
|
49
|
+
it('should report progress', async () => {
|
|
50
|
+
const provider = createMockProvider();
|
|
51
|
+
const texts = Array(10).fill('Test');
|
|
52
|
+
const progressCalls = [];
|
|
53
|
+
await batchProcess({
|
|
54
|
+
provider,
|
|
55
|
+
texts,
|
|
56
|
+
batchSize: 3,
|
|
57
|
+
onProgress: (progress) => {
|
|
58
|
+
progressCalls.push(progress);
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
expect(progressCalls.length).toBeGreaterThan(0);
|
|
62
|
+
const lastProgress = progressCalls[progressCalls.length - 1];
|
|
63
|
+
expect(lastProgress.processed).toBe(10);
|
|
64
|
+
expect(lastProgress.total).toBe(10);
|
|
65
|
+
expect(lastProgress.percentage).toBe(100);
|
|
66
|
+
});
|
|
67
|
+
it('should handle errors with error callback', async () => {
|
|
68
|
+
const provider = createMockProvider();
|
|
69
|
+
provider.embedBatch = vi.fn().mockRejectedValue(new Error('API Error'));
|
|
70
|
+
const texts = ['Text 1', 'Text 2'];
|
|
71
|
+
const errors = [];
|
|
72
|
+
const result = await batchProcess({
|
|
73
|
+
provider,
|
|
74
|
+
texts,
|
|
75
|
+
batchSize: 1,
|
|
76
|
+
maxRetries: 0,
|
|
77
|
+
onError: (error) => {
|
|
78
|
+
errors.push(error);
|
|
79
|
+
},
|
|
80
|
+
});
|
|
81
|
+
expect(errors.length).toBeGreaterThan(0);
|
|
82
|
+
expect(result.failed.length).toBe(2);
|
|
83
|
+
expect(result.stats.failed).toBe(2);
|
|
84
|
+
});
|
|
85
|
+
it('should throw error without error callback', async () => {
|
|
86
|
+
const provider = createMockProvider();
|
|
87
|
+
provider.embedBatch = vi.fn().mockRejectedValue(new Error('API Error'));
|
|
88
|
+
const texts = ['Text 1'];
|
|
89
|
+
await expect(batchProcess({
|
|
90
|
+
provider,
|
|
91
|
+
texts,
|
|
92
|
+
batchSize: 1,
|
|
93
|
+
maxRetries: 0,
|
|
94
|
+
})).rejects.toThrow('API Error');
|
|
95
|
+
});
|
|
96
|
+
it('should retry failed batches', async () => {
|
|
97
|
+
const provider = createMockProvider();
|
|
98
|
+
let callCount = 0;
|
|
99
|
+
provider.embedBatch = vi.fn().mockImplementation(async () => {
|
|
100
|
+
callCount++;
|
|
101
|
+
if (callCount < 3) {
|
|
102
|
+
throw new Error('Temporary error');
|
|
103
|
+
}
|
|
104
|
+
return [[1, 2, 3]];
|
|
105
|
+
});
|
|
106
|
+
const result = await batchProcess({
|
|
107
|
+
provider,
|
|
108
|
+
texts: ['Test'],
|
|
109
|
+
batchSize: 1,
|
|
110
|
+
maxRetries: 3,
|
|
111
|
+
retryDelay: 10,
|
|
112
|
+
});
|
|
113
|
+
expect(result.embeddings).toHaveLength(1);
|
|
114
|
+
expect(provider.embedBatch).toHaveBeenCalledTimes(3); // Initial + 2 retries
|
|
115
|
+
});
|
|
116
|
+
it('should track duration', async () => {
|
|
117
|
+
const provider = createMockProvider(10); // 10ms delay
|
|
118
|
+
const texts = ['Text 1', 'Text 2'];
|
|
119
|
+
const result = await batchProcess({
|
|
120
|
+
provider,
|
|
121
|
+
texts,
|
|
122
|
+
batchSize: 1,
|
|
123
|
+
rateLimit: 1000, // High rate limit to minimize delay
|
|
124
|
+
});
|
|
125
|
+
expect(result.stats.duration).toBeGreaterThan(0);
|
|
126
|
+
});
|
|
127
|
+
it('should include source hash in embeddings', async () => {
|
|
128
|
+
const provider = createMockProvider();
|
|
129
|
+
const texts = ['Text 1'];
|
|
130
|
+
const result = await batchProcess({
|
|
131
|
+
provider,
|
|
132
|
+
texts,
|
|
133
|
+
includeSourceHash: true,
|
|
134
|
+
});
|
|
135
|
+
expect(result.embeddings[0].metadata.sourceHash).toBeDefined();
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
describe('RateLimiter', () => {
|
|
139
|
+
it('should allow requests under rate limit', async () => {
|
|
140
|
+
const limiter = new RateLimiter(100); // 100 requests per minute
|
|
141
|
+
const start = Date.now();
|
|
142
|
+
// Should process quickly since we're under limit
|
|
143
|
+
await limiter.waitForSlot();
|
|
144
|
+
await limiter.waitForSlot();
|
|
145
|
+
await limiter.waitForSlot();
|
|
146
|
+
const duration = Date.now() - start;
|
|
147
|
+
// Should be nearly instant
|
|
148
|
+
expect(duration).toBeLessThan(100);
|
|
149
|
+
});
|
|
150
|
+
it('should throttle requests exceeding rate limit', { timeout: 70000 }, async () => {
|
|
151
|
+
const limiter = new RateLimiter(2); // Only 2 requests per minute
|
|
152
|
+
const results = [];
|
|
153
|
+
// Try to make 3 requests
|
|
154
|
+
for (let i = 0; i < 3; i++) {
|
|
155
|
+
await limiter.waitForSlot();
|
|
156
|
+
results.push(Date.now());
|
|
157
|
+
}
|
|
158
|
+
// Third request should be delayed
|
|
159
|
+
const delay1 = results[1] - results[0];
|
|
160
|
+
const delay2 = results[2] - results[1];
|
|
161
|
+
expect(delay1).toBeLessThan(1000); // First two are quick
|
|
162
|
+
expect(delay2).toBeGreaterThan(100); // Third is delayed
|
|
163
|
+
}); // 70 second timeout for rate limiting test
|
|
164
|
+
it('should execute function with rate limiting', async () => {
|
|
165
|
+
const limiter = new RateLimiter(100);
|
|
166
|
+
const result = await limiter.execute(async () => {
|
|
167
|
+
return 'success';
|
|
168
|
+
});
|
|
169
|
+
expect(result).toBe('success');
|
|
170
|
+
});
|
|
171
|
+
it('should handle errors in executed function', async () => {
|
|
172
|
+
const limiter = new RateLimiter(100);
|
|
173
|
+
await expect(limiter.execute(async () => {
|
|
174
|
+
throw new Error('Test error');
|
|
175
|
+
})).rejects.toThrow('Test error');
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
describe('ProcessingQueue', () => {
|
|
179
|
+
it('should process items sequentially with concurrency 1', async () => {
|
|
180
|
+
const processed = [];
|
|
181
|
+
const queue = new ProcessingQueue(async (item) => {
|
|
182
|
+
processed.push(item);
|
|
183
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
184
|
+
return item * 2;
|
|
185
|
+
}, 1);
|
|
186
|
+
const promises = [queue.add(1), queue.add(2), queue.add(3)];
|
|
187
|
+
const results = await Promise.all(promises);
|
|
188
|
+
expect(results).toEqual([2, 4, 6]);
|
|
189
|
+
expect(processed).toEqual([1, 2, 3]);
|
|
190
|
+
});
|
|
191
|
+
it('should process items concurrently with concurrency > 1', async () => {
|
|
192
|
+
const queue = new ProcessingQueue(async (item) => {
|
|
193
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
194
|
+
return item * 2;
|
|
195
|
+
}, 3);
|
|
196
|
+
const start = Date.now();
|
|
197
|
+
const results = await queue.addBatch([1, 2, 3, 4, 5]);
|
|
198
|
+
const duration = Date.now() - start;
|
|
199
|
+
expect(results).toEqual([2, 4, 6, 8, 10]);
|
|
200
|
+
// With concurrency 3, should be faster than sequential
|
|
201
|
+
// 5 items with 10ms each sequentially = 50ms
|
|
202
|
+
// With concurrency 3: ceil(5/3) * 10ms = 20ms
|
|
203
|
+
expect(duration).toBeLessThan(40);
|
|
204
|
+
});
|
|
205
|
+
it('should track queue size', async () => {
|
|
206
|
+
const queue = new ProcessingQueue(async (item) => {
|
|
207
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
208
|
+
return item;
|
|
209
|
+
}, 1);
|
|
210
|
+
// Add items quickly
|
|
211
|
+
queue.add(1);
|
|
212
|
+
queue.add(2);
|
|
213
|
+
queue.add(3);
|
|
214
|
+
// Queue should have items waiting
|
|
215
|
+
expect(queue.size).toBeGreaterThan(0);
|
|
216
|
+
});
|
|
217
|
+
it('should track active count', async () => {
|
|
218
|
+
const queue = new ProcessingQueue(async (item) => {
|
|
219
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
220
|
+
return item;
|
|
221
|
+
}, 2);
|
|
222
|
+
// Start processing
|
|
223
|
+
const promise1 = queue.add(1);
|
|
224
|
+
const promise2 = queue.add(2);
|
|
225
|
+
const promise3 = queue.add(3);
|
|
226
|
+
// Give time for processing to start
|
|
227
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
228
|
+
// Should have 2 active (concurrency limit)
|
|
229
|
+
expect(queue.activeCount).toBeLessThanOrEqual(2);
|
|
230
|
+
await Promise.all([promise1, promise2, promise3]);
|
|
231
|
+
});
|
|
232
|
+
it('should handle errors in processor', async () => {
|
|
233
|
+
const queue = new ProcessingQueue(async (item) => {
|
|
234
|
+
if (item === 2) {
|
|
235
|
+
throw new Error('Processing error');
|
|
236
|
+
}
|
|
237
|
+
return item * 2;
|
|
238
|
+
}, 1);
|
|
239
|
+
const result1 = await queue.add(1);
|
|
240
|
+
expect(result1).toBe(2);
|
|
241
|
+
await expect(queue.add(2)).rejects.toThrow('Processing error');
|
|
242
|
+
const result3 = await queue.add(3);
|
|
243
|
+
expect(result3).toBe(6);
|
|
244
|
+
});
|
|
245
|
+
it('should process empty batch', async () => {
|
|
246
|
+
const queue = new ProcessingQueue(async (item) => item * 2, 1);
|
|
247
|
+
const results = await queue.addBatch([]);
|
|
248
|
+
expect(results).toEqual([]);
|
|
249
|
+
});
|
|
250
|
+
});
|
|
251
|
+
//# sourceMappingURL=batch.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch.test.js","sourceRoot":"","sources":["../../src/runtime/batch.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AAC7D,OAAO,EACL,YAAY,EACZ,WAAW,EACX,eAAe,GAGhB,MAAM,YAAY,CAAA;AAGnB,0BAA0B;AAC1B,SAAS,kBAAkB,CAAC,UAAkB,CAAC;IAC7C,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,YAAY;QACnB,UAAU,EAAE,CAAC;QACb,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YAClC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAA;YAC7E,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,CAAA;QAC/D,CAAC,CAAC;QACF,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,KAAe,EAAE,EAAE;YAC1C,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAA;YAC7E,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAA;QACpF,CAAC,CAAC;KACH,CAAA;AACH,CAAC;AAED,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAA;IACpB,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,QAAQ,GAAG,kBAAkB,EAAE,CAAA;QACrC,MAAM,KAAK,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAE5C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,QAAQ;YACR,KAAK;YACL,SAAS,EAAE,CAAC;SACb,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACrC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACvC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACrC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;QACzC,MAAM,QAAQ,GAAG,kBAAkB,EAAE,CAAA;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAEpC,MAAM,YAAY,CAAC;YACjB,QAAQ;YACR,KAAK;YACL,SAAS,EAAE,CAAC;SACb,CAAC,CAAA;QAEF,0CAA0C;QAC1C,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAA;IACtD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;QACtC,MAAM,QAAQ,GAAG,kBAAkB,EAAE,CAAA;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACpC,MAAM,aAAa,GAAoB,EAAE,CAAA;QAEzC,MAAM,YAAY,CAAC;YACjB,QAAQ;YACR,KAAK;YACL,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC,QAAQ,EAAE,EAAE;gBACvB,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAC9B,CAAC;SACF,CAAC,CAAA;QAEF,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QAE/C,MAAM,YAAY,GAAG,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAC5D,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACvC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACnC,MAAM,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,QAAQ,GAAG,kBAAkB,EAAE,CAAA;QACrC,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,CAAA;QAEvE,MAAM,KAAK,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAClC,MAAM,MAAM,GAAiB,EAAE,CAAA;QAE/B,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,QAAQ;YACR,KAAK;YACL,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACjB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACpB,CAAC;SACF,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACrC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,QAAQ,GAAG,kBAAkB,EAAE,CAAA;QACrC,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,CAAA;QAEvE,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAA;QAExB,MAAM,MAAM,CACV,YAAY,CAAC;YACX,QAAQ;YACR,KAAK;YACL,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC;SACd,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAChC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,QAAQ,GAAG,kBAAkB,EAAE,CAAA;QACrC,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YAC1D,SAAS,EAAE,CAAA;YACX,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAA;YACpC,CAAC;YACD,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACpB,CAAC,CAAC,CAAA;QAEF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,QAAQ;YACR,KAAK,EAAE,CAAC,MAAM,CAAC;YACf,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,EAAE;SACf,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACzC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAA,CAAC,sBAAsB;IAC7E,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;QACrC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,EAAE,CAAC,CAAA,CAAC,aAAa;QACrD,MAAM,KAAK,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAElC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,QAAQ;YACR,KAAK;YACL,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,IAAI,EAAE,oCAAoC;SACtD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAClD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,QAAQ,GAAG,kBAAkB,EAAE,CAAA;QACrC,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAA;QAExB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,QAAQ;YACR,KAAK;YACL,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE,CAAA;IAChE,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,CAAA,CAAC,0BAA0B;QAE/D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAExB,iDAAiD;QACjD,MAAM,OAAO,CAAC,WAAW,EAAE,CAAA;QAC3B,MAAM,OAAO,CAAC,WAAW,EAAE,CAAA;QAC3B,MAAM,OAAO,CAAC,WAAW,EAAE,CAAA;QAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;QAEnC,2BAA2B;QAC3B,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IACpC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+CAA+C,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,IAAI,EAAE;QACjF,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,CAAA,CAAC,6BAA6B;QAEhE,MAAM,OAAO,GAAa,EAAE,CAAA;QAE5B,yBAAyB;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,OAAO,CAAC,WAAW,EAAE,CAAA;YAC3B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QAC1B,CAAC;QAED,kCAAkC;QAClC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QACtC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QAEtC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA,CAAC,sBAAsB;QACxD,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAA,CAAC,mBAAmB;IACzD,CAAC,CAAC,CAAA,CAAC,2CAA2C;IAE9C,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,CAAA;QAEpC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YAC9C,OAAO,SAAS,CAAA;QAClB,CAAC,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAChC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,CAAA;QAEpC,MAAM,MAAM,CACV,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAA;QAC/B,CAAC,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;IACjC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,SAAS,GAAa,EAAE,CAAA;QAC9B,MAAM,KAAK,GAAG,IAAI,eAAe,CAC/B,KAAK,EAAE,IAAY,EAAE,EAAE;YACrB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACpB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;YACvD,OAAO,IAAI,GAAG,CAAC,CAAA;QACjB,CAAC,EACD,CAAC,CACF,CAAA;QAED,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;QAE3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAE3C,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IACtC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,KAAK,GAAG,IAAI,eAAe,CAC/B,KAAK,EAAE,IAAY,EAAE,EAAE;YACrB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;YACvD,OAAO,IAAI,GAAG,CAAC,CAAA;QACjB,CAAC,EACD,CAAC,CACF,CAAA;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;QAEnC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;QAEzC,uDAAuD;QACvD,6CAA6C;QAC7C,8CAA8C;QAC9C,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAA;IACnC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;QACvC,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;YACvD,OAAO,IAAI,CAAA;QACb,CAAC,EAAE,CAAC,CAAC,CAAA;QAEL,oBAAoB;QACpB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QACZ,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QACZ,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QAEZ,kCAAkC;QAClC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IACvC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;QACzC,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;YACvD,OAAO,IAAI,CAAA;QACb,CAAC,EAAE,CAAC,CAAC,CAAA;QAEL,mBAAmB;QACnB,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QAE7B,oCAAoC;QACpC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;QAEvD,2CAA2C;QAC3C,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAA;QAEhD,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAA;IACnD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YACvD,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAA;YACrC,CAAC;YACD,OAAO,IAAI,GAAG,CAAC,CAAA;QACjB,CAAC,EAAE,CAAC,CAAC,CAAA;QAEL,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAEvB,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAA;QAE9D,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACzB,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4BAA4B,EAAE,KAAK,IAAI,EAAE;QAC1C,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAEtE,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;QAExC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text chunking utilities for splitting documents into smaller segments
|
|
3
|
+
* suitable for embedding generation.
|
|
4
|
+
*/
|
|
5
|
+
export type ChunkingStrategy = 'recursive' | 'sentence' | 'sliding-window' | 'token-aware';
|
|
6
|
+
export interface ChunkingOptions {
|
|
7
|
+
/** Target chunk size in characters */
|
|
8
|
+
chunkSize?: number;
|
|
9
|
+
/** Overlap between chunks in characters */
|
|
10
|
+
chunkOverlap?: number;
|
|
11
|
+
/** Strategy for chunking text */
|
|
12
|
+
strategy?: ChunkingStrategy;
|
|
13
|
+
/** Separators for recursive strategy (in priority order) */
|
|
14
|
+
separators?: string[];
|
|
15
|
+
/** Token limit for token-aware strategy */
|
|
16
|
+
tokenLimit?: number;
|
|
17
|
+
}
|
|
18
|
+
export interface TextChunk {
|
|
19
|
+
/** The chunked text content */
|
|
20
|
+
text: string;
|
|
21
|
+
/** Start position in original text */
|
|
22
|
+
start: number;
|
|
23
|
+
/** End position in original text */
|
|
24
|
+
end: number;
|
|
25
|
+
/** Chunk index */
|
|
26
|
+
index: number;
|
|
27
|
+
/** Metadata about the chunk */
|
|
28
|
+
metadata?: Record<string, unknown>;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Split text into chunks using specified strategy
|
|
32
|
+
*/
|
|
33
|
+
export declare function chunkText(text: string, options?: ChunkingOptions): TextChunk[];
|
|
34
|
+
/**
|
|
35
|
+
* Estimate token count for text (rough approximation)
|
|
36
|
+
*/
|
|
37
|
+
export declare function estimateTokenCount(text: string): number;
|
|
38
|
+
/**
|
|
39
|
+
* Merge small chunks to improve efficiency
|
|
40
|
+
*/
|
|
41
|
+
export declare function mergeSmallChunks(chunks: TextChunk[], minSize: number): TextChunk[];
|
|
42
|
+
//# sourceMappingURL=chunking.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../../src/runtime/chunking.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,MAAM,gBAAgB,GAAG,WAAW,GAAG,UAAU,GAAG,gBAAgB,GAAG,aAAa,CAAA;AAE1F,MAAM,WAAW,eAAe;IAC9B,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,2CAA2C;IAC3C,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,iCAAiC;IACjC,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IAC3B,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;IACrB,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAA;IACb,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAA;IACX,kBAAkB;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAID;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,SAAS,EAAE,CA8BlF;AAoOD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGvD;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,CA0BlF"}
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text chunking utilities for splitting documents into smaller segments
|
|
3
|
+
* suitable for embedding generation.
|
|
4
|
+
*/
|
|
5
|
+
const DEFAULT_SEPARATORS = ['\n\n', '\n', '. ', ' ', ''];
|
|
6
|
+
/**
|
|
7
|
+
* Split text into chunks using specified strategy
|
|
8
|
+
*/
|
|
9
|
+
export function chunkText(text, options = {}) {
|
|
10
|
+
const { chunkSize = 1000, chunkOverlap = 200, strategy = 'recursive', separators = DEFAULT_SEPARATORS, tokenLimit, } = options;
|
|
11
|
+
// Handle empty text early
|
|
12
|
+
if (!text || text.trim().length === 0) {
|
|
13
|
+
return [];
|
|
14
|
+
}
|
|
15
|
+
if (chunkOverlap >= chunkSize) {
|
|
16
|
+
throw new Error('chunkOverlap must be less than chunkSize');
|
|
17
|
+
}
|
|
18
|
+
switch (strategy) {
|
|
19
|
+
case 'recursive':
|
|
20
|
+
return recursiveChunk(text, chunkSize, chunkOverlap, separators);
|
|
21
|
+
case 'sentence':
|
|
22
|
+
return sentenceChunk(text, chunkSize, chunkOverlap);
|
|
23
|
+
case 'sliding-window':
|
|
24
|
+
return slidingWindowChunk(text, chunkSize, chunkOverlap);
|
|
25
|
+
case 'token-aware':
|
|
26
|
+
return tokenAwareChunk(text, tokenLimit || chunkSize, chunkOverlap);
|
|
27
|
+
default:
|
|
28
|
+
throw new Error(`Unknown chunking strategy: ${strategy}`);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Recursive text splitting - tries to split by paragraphs, then sentences, then words
|
|
33
|
+
*/
|
|
34
|
+
function recursiveChunk(text, chunkSize, overlap, separators) {
|
|
35
|
+
const chunks = [];
|
|
36
|
+
function splitRecursive(content, startPos, sepIndex) {
|
|
37
|
+
if (content.length <= chunkSize) {
|
|
38
|
+
if (content.trim()) {
|
|
39
|
+
chunks.push({
|
|
40
|
+
text: content,
|
|
41
|
+
start: startPos,
|
|
42
|
+
end: startPos + content.length,
|
|
43
|
+
index: chunks.length,
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
if (sepIndex >= separators.length) {
|
|
49
|
+
// No more separators, force split at chunkSize
|
|
50
|
+
let pos = 0;
|
|
51
|
+
while (pos < content.length) {
|
|
52
|
+
const end = Math.min(pos + chunkSize, content.length);
|
|
53
|
+
const chunk = content.slice(pos, end);
|
|
54
|
+
if (chunk.trim()) {
|
|
55
|
+
chunks.push({
|
|
56
|
+
text: chunk,
|
|
57
|
+
start: startPos + pos,
|
|
58
|
+
end: startPos + end,
|
|
59
|
+
index: chunks.length,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
pos += chunkSize - overlap;
|
|
63
|
+
}
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
const separator = separators[sepIndex];
|
|
67
|
+
const parts = content.split(separator);
|
|
68
|
+
let currentChunk = '';
|
|
69
|
+
let chunkStart = startPos;
|
|
70
|
+
for (let i = 0; i < parts.length; i++) {
|
|
71
|
+
const part = parts[i] + (i < parts.length - 1 ? separator : '');
|
|
72
|
+
if (currentChunk.length + part.length <= chunkSize) {
|
|
73
|
+
currentChunk += part;
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
if (currentChunk.trim()) {
|
|
77
|
+
// Try to split current chunk with next separator
|
|
78
|
+
if (currentChunk.length > chunkSize) {
|
|
79
|
+
splitRecursive(currentChunk, chunkStart, sepIndex + 1);
|
|
80
|
+
chunkStart += currentChunk.length;
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
chunks.push({
|
|
84
|
+
text: currentChunk,
|
|
85
|
+
start: chunkStart,
|
|
86
|
+
end: chunkStart + currentChunk.length,
|
|
87
|
+
index: chunks.length,
|
|
88
|
+
});
|
|
89
|
+
chunkStart += currentChunk.length;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
// Handle overlap
|
|
93
|
+
if (overlap > 0 && currentChunk.length >= overlap) {
|
|
94
|
+
currentChunk = currentChunk.slice(-overlap) + part;
|
|
95
|
+
chunkStart -= overlap;
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
currentChunk = part;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (currentChunk.trim()) {
|
|
103
|
+
if (currentChunk.length > chunkSize) {
|
|
104
|
+
splitRecursive(currentChunk, chunkStart, sepIndex + 1);
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
chunks.push({
|
|
108
|
+
text: currentChunk,
|
|
109
|
+
start: chunkStart,
|
|
110
|
+
end: chunkStart + currentChunk.length,
|
|
111
|
+
index: chunks.length,
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
splitRecursive(text, 0, 0);
|
|
117
|
+
return chunks;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Sentence-based chunking - preserves sentence boundaries
|
|
121
|
+
*/
|
|
122
|
+
function sentenceChunk(text, chunkSize, overlap) {
|
|
123
|
+
const chunks = [];
|
|
124
|
+
// Split into sentences (simple regex, can be improved)
|
|
125
|
+
const sentenceRegex = /[^.!?]+[.!?]+/g;
|
|
126
|
+
const sentences = [];
|
|
127
|
+
let match;
|
|
128
|
+
while ((match = sentenceRegex.exec(text)) !== null) {
|
|
129
|
+
sentences.push({
|
|
130
|
+
text: match[0],
|
|
131
|
+
start: match.index,
|
|
132
|
+
end: match.index + match[0].length,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
if (sentences.length === 0) {
|
|
136
|
+
// No sentences found, return whole text as one chunk
|
|
137
|
+
return [
|
|
138
|
+
{
|
|
139
|
+
text: text,
|
|
140
|
+
start: 0,
|
|
141
|
+
end: text.length,
|
|
142
|
+
index: 0,
|
|
143
|
+
},
|
|
144
|
+
];
|
|
145
|
+
}
|
|
146
|
+
let currentChunk = [];
|
|
147
|
+
let currentLength = 0;
|
|
148
|
+
for (let i = 0; i < sentences.length; i++) {
|
|
149
|
+
const sentence = sentences[i];
|
|
150
|
+
if (currentLength + sentence.text.length > chunkSize && currentChunk.length > 0) {
|
|
151
|
+
// Save current chunk
|
|
152
|
+
const chunkText = currentChunk.map((s) => s.text).join('');
|
|
153
|
+
chunks.push({
|
|
154
|
+
text: chunkText,
|
|
155
|
+
start: currentChunk[0].start,
|
|
156
|
+
end: currentChunk[currentChunk.length - 1].end,
|
|
157
|
+
index: chunks.length,
|
|
158
|
+
});
|
|
159
|
+
// Calculate overlap
|
|
160
|
+
if (overlap > 0) {
|
|
161
|
+
let overlapLength = 0;
|
|
162
|
+
const overlapSentences = [];
|
|
163
|
+
for (let j = currentChunk.length - 1; j >= 0; j--) {
|
|
164
|
+
if (overlapLength + currentChunk[j].text.length <= overlap) {
|
|
165
|
+
overlapSentences.unshift(currentChunk[j]);
|
|
166
|
+
overlapLength += currentChunk[j].text.length;
|
|
167
|
+
}
|
|
168
|
+
else {
|
|
169
|
+
break;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
currentChunk = overlapSentences;
|
|
173
|
+
currentLength = overlapLength;
|
|
174
|
+
}
|
|
175
|
+
else {
|
|
176
|
+
currentChunk = [];
|
|
177
|
+
currentLength = 0;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
currentChunk.push(sentence);
|
|
181
|
+
currentLength += sentence.text.length;
|
|
182
|
+
}
|
|
183
|
+
// Add final chunk
|
|
184
|
+
if (currentChunk.length > 0) {
|
|
185
|
+
const chunkText = currentChunk.map((s) => s.text).join('');
|
|
186
|
+
chunks.push({
|
|
187
|
+
text: chunkText,
|
|
188
|
+
start: currentChunk[0].start,
|
|
189
|
+
end: currentChunk[currentChunk.length - 1].end,
|
|
190
|
+
index: chunks.length,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
return chunks;
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Sliding window chunking - fixed-size chunks with overlap
|
|
197
|
+
*/
|
|
198
|
+
function slidingWindowChunk(text, chunkSize, overlap) {
|
|
199
|
+
const chunks = [];
|
|
200
|
+
const step = chunkSize - overlap;
|
|
201
|
+
for (let i = 0; i < text.length; i += step) {
|
|
202
|
+
const end = Math.min(i + chunkSize, text.length);
|
|
203
|
+
const chunk = text.slice(i, end);
|
|
204
|
+
if (chunk.trim()) {
|
|
205
|
+
chunks.push({
|
|
206
|
+
text: chunk,
|
|
207
|
+
start: i,
|
|
208
|
+
end: end,
|
|
209
|
+
index: chunks.length,
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
// Stop if we've reached the end
|
|
213
|
+
if (end === text.length)
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
return chunks;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Token-aware chunking - estimates token count and splits accordingly
|
|
220
|
+
* Uses a rough estimate of ~4 characters per token (actual depends on tokenizer)
|
|
221
|
+
*/
|
|
222
|
+
function tokenAwareChunk(text, tokenLimit, overlap) {
|
|
223
|
+
const CHARS_PER_TOKEN = 4; // Rough estimate
|
|
224
|
+
const chunkSize = tokenLimit * CHARS_PER_TOKEN;
|
|
225
|
+
const overlapChars = overlap * CHARS_PER_TOKEN;
|
|
226
|
+
// Use recursive strategy with token-aware chunk size
|
|
227
|
+
return recursiveChunk(text, chunkSize, overlapChars, DEFAULT_SEPARATORS);
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Estimate token count for text (rough approximation)
|
|
231
|
+
*/
|
|
232
|
+
export function estimateTokenCount(text) {
|
|
233
|
+
const CHARS_PER_TOKEN = 4;
|
|
234
|
+
return Math.ceil(text.length / CHARS_PER_TOKEN);
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Merge small chunks to improve efficiency
|
|
238
|
+
*/
|
|
239
|
+
export function mergeSmallChunks(chunks, minSize) {
|
|
240
|
+
if (chunks.length === 0)
|
|
241
|
+
return [];
|
|
242
|
+
const merged = [];
|
|
243
|
+
let current = chunks[0];
|
|
244
|
+
for (let i = 1; i < chunks.length; i++) {
|
|
245
|
+
const next = chunks[i];
|
|
246
|
+
if (current.text.length < minSize) {
|
|
247
|
+
// Merge with next chunk
|
|
248
|
+
current = {
|
|
249
|
+
text: current.text + next.text,
|
|
250
|
+
start: current.start,
|
|
251
|
+
end: next.end,
|
|
252
|
+
index: merged.length,
|
|
253
|
+
metadata: { ...current.metadata, ...next.metadata },
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
else {
|
|
257
|
+
merged.push(current);
|
|
258
|
+
current = { ...next, index: merged.length };
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
merged.push(current);
|
|
262
|
+
return merged;
|
|
263
|
+
}
|
|
264
|
+
//# sourceMappingURL=chunking.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../../src/runtime/chunking.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA8BH,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAA;AAExD;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,UAA2B,EAAE;IACnE,MAAM,EACJ,SAAS,GAAG,IAAI,EAChB,YAAY,GAAG,GAAG,EAClB,QAAQ,GAAG,WAAW,EACtB,UAAU,GAAG,kBAAkB,EAC/B,UAAU,GACX,GAAG,OAAO,CAAA;IAEX,0BAA0B;IAC1B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,CAAA;IACX,CAAC;IAED,IAAI,YAAY,IAAI,SAAS,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;IAC7D,CAAC;IAED,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,CAAC,CAAA;QAClE,KAAK,UAAU;YACb,OAAO,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,YAAY,CAAC,CAAA;QACrD,KAAK,gBAAgB;YACnB,OAAO,kBAAkB,CAAC,IAAI,EAAE,SAAS,EAAE,YAAY,CAAC,CAAA;QAC1D,KAAK,aAAa;YAChB,OAAO,eAAe,CAAC,IAAI,EAAE,UAAU,IAAI,SAAS,EAAE,YAAY,CAAC,CAAA;QACrE;YACE,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,EAAE,CAAC,CAAA;IAC7D,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CACrB,IAAY,EACZ,SAAiB,EACjB,OAAe,EACf,UAAoB;IAEpB,MAAM,MAAM,GAAgB,EAAE,CAAA;IAE9B,SAAS,cAAc,CAAC,OAAe,EAAE,QAAgB,EAAE,QAAgB;QACzE,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAChC,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,QAAQ;oBACf,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM;oBAC9B,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAA;YACJ,CAAC;YACD,OAAM;QACR,CAAC;QAED,IAAI,QAAQ,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YAClC,+CAA+C;YAC/C,IAAI,GAAG,GAAG,CAAC,CAAA;YACX,OAAO,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;gBACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;gBACrC,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;oBACjB,MAAM,CAAC,IAAI,CAAC;wBACV,IAAI,EAAE,KAAK;wBACX,KAAK,EAAE,QAAQ,GAAG,GAAG;wBACrB,GAAG,EAAE,QAAQ,GAAG,GAAG;wBACnB,KAAK,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAA;gBACJ,CAAC;gBACD,GAAG,IAAI,SAAS,GAAG,OAAO,CAAA;YAC5B,CAAC;YACD,OAAM;QACR,CAAC;QAED,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;QACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QAEtC,IAAI,YAAY,GAAG,EAAE,CAAA;QACrB,IAAI,UAAU,GAAG,QAAQ,CAAA;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAE/D,IAAI,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;gBACnD,YAAY,IAAI,IAAI,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACN,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;oBACxB,iDAAiD;oBACjD,IAAI,YAAY,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;wBACpC,cAAc,CAAC,YAAY,EAAE,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAA;wBACtD,UAAU,IAAI,YAAY,CAAC,MAAM,CAAA;oBACnC,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,CAAC;4BACV,IAAI,EAAE,YAAY;4BAClB,KAAK,EAAE,UAAU;4BACjB,GAAG,EAAE,UAAU,GAAG,YAAY,CAAC,MAAM;4BACrC,KAAK,EAAE,MAAM,CAAC,MAAM;yBACrB,CAAC,CAAA;wBACF,UAAU,IAAI,YAAY,CAAC,MAAM,CAAA;oBACnC,CAAC;gBACH,CAAC;gBAED,iBAAiB;gBACjB,IAAI,OAAO,GAAG,CAAC,IAAI,YAAY,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;oBAClD,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,IAAI,CAAA;oBAClD,UAAU,IAAI,OAAO,CAAA;gBACvB,CAAC;qBAAM,CAAC;oBACN,YAAY,GAAG,IAAI,CAAA;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;YACxB,IAAI,YAAY,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;gBACpC,cAAc,CAAC,YAAY,EAAE,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAA;YACxD,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,YAAY;oBAClB,KAAK,EAAE,UAAU;oBACjB,GAAG,EAAE,UAAU,GAAG,YAAY,CAAC,MAAM;oBACrC,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,cAAc,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAC1B,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,IAAY,EAAE,SAAiB,EAAE,OAAe;IACrE,MAAM,MAAM,GAAgB,EAAE,CAAA;IAE9B,uDAAuD;IACvD,MAAM,aAAa,GAAG,gBAAgB,CAAA;IACtC,MAAM,SAAS,GAAmD,EAAE,CAAA;IAEpE,IAAI,KAA6B,CAAA;IACjC,OAAO,CAAC,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACnD,SAAS,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;YACd,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;SACnC,CAAC,CAAA;IACJ,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,qDAAqD;QACrD,OAAO;YACL;gBACE,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,GAAG,EAAE,IAAI,CAAC,MAAM;gBAChB,KAAK,EAAE,CAAC;aACT;SACF,CAAA;IACH,CAAC;IAED,IAAI,YAAY,GAAqB,EAAE,CAAA;IACvC,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;QAE7B,IAAI,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChF,qBAAqB;YACrB,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAC1D,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK;gBAC5B,GAAG,EAAE,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG;gBAC9C,KAAK,EAAE,MAAM,CAAC,MAAM;aACrB,CAAC,CAAA;YAEF,oBAAoB;YACpB,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,IAAI,aAAa,GAAG,CAAC,CAAA;gBACrB,MAAM,gBAAgB,GAAqB,EAAE,CAAA;gBAE7C,KAAK,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAClD,IAAI,aAAa,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC3D,gBAAgB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAA;wBACzC,aAAa,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAA;oBAC9C,CAAC;yBAAM,CAAC;wBACN,MAAK;oBACP,CAAC;gBACH,CAAC;gBAED,YAAY,GAAG,gBAAgB,CAAA;gBAC/B,aAAa,GAAG,aAAa,CAAA;YAC/B,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,EAAE,CAAA;gBACjB,aAAa,GAAG,CAAC,CAAA;YACnB,CAAC;QACH,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC3B,aAAa,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAA;IACvC,CAAC;IAED,kBAAkB;IAClB,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC1D,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK;YAC5B,GAAG,EAAE,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG;YAC9C,KAAK,EAAE,MAAM,CAAC,MAAM;SACrB,CAAC,CAAA;IACJ,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,IAAY,EAAE,SAAiB,EAAE,OAAe;IAC1E,MAAM,MAAM,GAAgB,EAAE,CAAA;IAC9B,MAAM,IAAI,GAAG,SAAS,GAAG,OAAO,CAAA;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;QAEhC,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,GAAG,EAAE,GAAG;gBACR,KAAK,EAAE,MAAM,CAAC,MAAM;aACrB,CAAC,CAAA;QACJ,CAAC;QAED,gCAAgC;QAChC,IAAI,GAAG,KAAK,IAAI,CAAC,MAAM;YAAE,MAAK;IAChC,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,IAAY,EAAE,UAAkB,EAAE,OAAe;IACxE,MAAM,eAAe,GAAG,CAAC,CAAA,CAAC,iBAAiB;IAC3C,MAAM,SAAS,GAAG,UAAU,GAAG,eAAe,CAAA;IAC9C,MAAM,YAAY,GAAG,OAAO,GAAG,eAAe,CAAA;IAE9C,qDAAqD;IACrD,OAAO,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,YAAY,EAAE,kBAAkB,CAAC,CAAA;AAC1E,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,MAAM,eAAe,GAAG,CAAC,CAAA;IACzB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,CAAA;AACjD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAe;IACnE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAElC,MAAM,MAAM,GAAgB,EAAE,CAAA;IAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;QAEtB,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;YAClC,wBAAwB;YACxB,OAAO,GAAG;gBACR,IAAI,EAAE,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI;gBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,QAAQ,EAAE,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE;aACpD,CAAA;QACH,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACpB,OAAO,GAAG,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAA;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACpB,OAAO,MAAM,CAAA;AACf,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunking.test.d.ts","sourceRoot":"","sources":["../../src/runtime/chunking.test.ts"],"names":[],"mappings":""}
|