@opensaas/stack-rag 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/.turbo/turbo-build.log +4 -0
  2. package/CHANGELOG.md +10 -0
  3. package/CLAUDE.md +565 -0
  4. package/LICENSE +21 -0
  5. package/README.md +406 -0
  6. package/dist/config/index.d.ts +63 -0
  7. package/dist/config/index.d.ts.map +1 -0
  8. package/dist/config/index.js +94 -0
  9. package/dist/config/index.js.map +1 -0
  10. package/dist/config/plugin.d.ts +38 -0
  11. package/dist/config/plugin.d.ts.map +1 -0
  12. package/dist/config/plugin.js +215 -0
  13. package/dist/config/plugin.js.map +1 -0
  14. package/dist/config/plugin.test.d.ts +2 -0
  15. package/dist/config/plugin.test.d.ts.map +1 -0
  16. package/dist/config/plugin.test.js +554 -0
  17. package/dist/config/plugin.test.js.map +1 -0
  18. package/dist/config/types.d.ts +249 -0
  19. package/dist/config/types.d.ts.map +1 -0
  20. package/dist/config/types.js +5 -0
  21. package/dist/config/types.js.map +1 -0
  22. package/dist/fields/embedding.d.ts +85 -0
  23. package/dist/fields/embedding.d.ts.map +1 -0
  24. package/dist/fields/embedding.js +81 -0
  25. package/dist/fields/embedding.js.map +1 -0
  26. package/dist/fields/embedding.test.d.ts +2 -0
  27. package/dist/fields/embedding.test.d.ts.map +1 -0
  28. package/dist/fields/embedding.test.js +323 -0
  29. package/dist/fields/embedding.test.js.map +1 -0
  30. package/dist/fields/index.d.ts +6 -0
  31. package/dist/fields/index.d.ts.map +1 -0
  32. package/dist/fields/index.js +5 -0
  33. package/dist/fields/index.js.map +1 -0
  34. package/dist/index.d.ts +8 -0
  35. package/dist/index.d.ts.map +1 -0
  36. package/dist/index.js +9 -0
  37. package/dist/index.js.map +1 -0
  38. package/dist/mcp/index.d.ts +19 -0
  39. package/dist/mcp/index.d.ts.map +1 -0
  40. package/dist/mcp/index.js +18 -0
  41. package/dist/mcp/index.js.map +1 -0
  42. package/dist/providers/index.d.ts +38 -0
  43. package/dist/providers/index.d.ts.map +1 -0
  44. package/dist/providers/index.js +68 -0
  45. package/dist/providers/index.js.map +1 -0
  46. package/dist/providers/ollama.d.ts +49 -0
  47. package/dist/providers/ollama.d.ts.map +1 -0
  48. package/dist/providers/ollama.js +151 -0
  49. package/dist/providers/ollama.js.map +1 -0
  50. package/dist/providers/openai.d.ts +41 -0
  51. package/dist/providers/openai.d.ts.map +1 -0
  52. package/dist/providers/openai.js +126 -0
  53. package/dist/providers/openai.js.map +1 -0
  54. package/dist/providers/providers.test.d.ts +2 -0
  55. package/dist/providers/providers.test.d.ts.map +1 -0
  56. package/dist/providers/providers.test.js +224 -0
  57. package/dist/providers/providers.test.js.map +1 -0
  58. package/dist/providers/types.d.ts +88 -0
  59. package/dist/providers/types.d.ts.map +1 -0
  60. package/dist/providers/types.js +2 -0
  61. package/dist/providers/types.js.map +1 -0
  62. package/dist/runtime/batch.d.ts +183 -0
  63. package/dist/runtime/batch.d.ts.map +1 -0
  64. package/dist/runtime/batch.js +240 -0
  65. package/dist/runtime/batch.js.map +1 -0
  66. package/dist/runtime/batch.test.d.ts +2 -0
  67. package/dist/runtime/batch.test.d.ts.map +1 -0
  68. package/dist/runtime/batch.test.js +251 -0
  69. package/dist/runtime/batch.test.js.map +1 -0
  70. package/dist/runtime/chunking.d.ts +42 -0
  71. package/dist/runtime/chunking.d.ts.map +1 -0
  72. package/dist/runtime/chunking.js +264 -0
  73. package/dist/runtime/chunking.js.map +1 -0
  74. package/dist/runtime/chunking.test.d.ts +2 -0
  75. package/dist/runtime/chunking.test.d.ts.map +1 -0
  76. package/dist/runtime/chunking.test.js +212 -0
  77. package/dist/runtime/chunking.test.js.map +1 -0
  78. package/dist/runtime/embeddings.d.ts +147 -0
  79. package/dist/runtime/embeddings.d.ts.map +1 -0
  80. package/dist/runtime/embeddings.js +201 -0
  81. package/dist/runtime/embeddings.js.map +1 -0
  82. package/dist/runtime/embeddings.test.d.ts +2 -0
  83. package/dist/runtime/embeddings.test.d.ts.map +1 -0
  84. package/dist/runtime/embeddings.test.js +366 -0
  85. package/dist/runtime/embeddings.test.js.map +1 -0
  86. package/dist/runtime/index.d.ts +14 -0
  87. package/dist/runtime/index.d.ts.map +1 -0
  88. package/dist/runtime/index.js +18 -0
  89. package/dist/runtime/index.js.map +1 -0
  90. package/dist/runtime/search.d.ts +135 -0
  91. package/dist/runtime/search.d.ts.map +1 -0
  92. package/dist/runtime/search.js +101 -0
  93. package/dist/runtime/search.js.map +1 -0
  94. package/dist/storage/index.d.ts +41 -0
  95. package/dist/storage/index.d.ts.map +1 -0
  96. package/dist/storage/index.js +73 -0
  97. package/dist/storage/index.js.map +1 -0
  98. package/dist/storage/json.d.ts +34 -0
  99. package/dist/storage/json.d.ts.map +1 -0
  100. package/dist/storage/json.js +82 -0
  101. package/dist/storage/json.js.map +1 -0
  102. package/dist/storage/pgvector.d.ts +53 -0
  103. package/dist/storage/pgvector.d.ts.map +1 -0
  104. package/dist/storage/pgvector.js +168 -0
  105. package/dist/storage/pgvector.js.map +1 -0
  106. package/dist/storage/sqlite-vss.d.ts +49 -0
  107. package/dist/storage/sqlite-vss.d.ts.map +1 -0
  108. package/dist/storage/sqlite-vss.js +148 -0
  109. package/dist/storage/sqlite-vss.js.map +1 -0
  110. package/dist/storage/storage.test.d.ts +2 -0
  111. package/dist/storage/storage.test.d.ts.map +1 -0
  112. package/dist/storage/storage.test.js +440 -0
  113. package/dist/storage/storage.test.js.map +1 -0
  114. package/dist/storage/types.d.ts +79 -0
  115. package/dist/storage/types.d.ts.map +1 -0
  116. package/dist/storage/types.js +49 -0
  117. package/dist/storage/types.js.map +1 -0
  118. package/package.json +82 -0
  119. package/src/config/index.ts +116 -0
  120. package/src/config/plugin.test.ts +664 -0
  121. package/src/config/plugin.ts +257 -0
  122. package/src/config/types.ts +283 -0
  123. package/src/fields/embedding.test.ts +408 -0
  124. package/src/fields/embedding.ts +150 -0
  125. package/src/fields/index.ts +6 -0
  126. package/src/index.ts +33 -0
  127. package/src/mcp/index.ts +21 -0
  128. package/src/providers/index.ts +81 -0
  129. package/src/providers/ollama.ts +186 -0
  130. package/src/providers/openai.ts +161 -0
  131. package/src/providers/providers.test.ts +275 -0
  132. package/src/providers/types.ts +100 -0
  133. package/src/runtime/batch.test.ts +332 -0
  134. package/src/runtime/batch.ts +424 -0
  135. package/src/runtime/chunking.test.ts +258 -0
  136. package/src/runtime/chunking.ts +334 -0
  137. package/src/runtime/embeddings.test.ts +441 -0
  138. package/src/runtime/embeddings.ts +380 -0
  139. package/src/runtime/index.ts +51 -0
  140. package/src/runtime/search.ts +243 -0
  141. package/src/storage/index.ts +86 -0
  142. package/src/storage/json.ts +106 -0
  143. package/src/storage/pgvector.ts +206 -0
  144. package/src/storage/sqlite-vss.ts +193 -0
  145. package/src/storage/storage.test.ts +521 -0
  146. package/src/storage/types.ts +126 -0
  147. package/tsconfig.json +13 -0
  148. package/tsconfig.tsbuildinfo +1 -0
  149. package/vitest.config.ts +18 -0
@@ -0,0 +1,88 @@
1
+ /**
2
+ * Embedding provider interface
3
+ * All embedding providers must implement this interface
4
+ */
5
+ export interface EmbeddingProvider {
6
+ /**
7
+ * Provider type identifier
8
+ */
9
+ readonly type: string;
10
+ /**
11
+ * Model name being used
12
+ */
13
+ readonly model: string;
14
+ /**
15
+ * Number of dimensions in the embedding vector
16
+ */
17
+ readonly dimensions: number;
18
+ /**
19
+ * Generate embedding for a single text
20
+ *
21
+ * @param text - The text to embed
22
+ * @returns The embedding vector
23
+ */
24
+ embed(text: string): Promise<number[]>;
25
+ /**
26
+ * Generate embeddings for multiple texts in a batch
27
+ * More efficient than calling embed() multiple times
28
+ *
29
+ * @param texts - Array of texts to embed
30
+ * @returns Array of embedding vectors in the same order as inputs
31
+ */
32
+ embedBatch(texts: string[]): Promise<number[][]>;
33
+ }
34
+ /**
35
+ * Result from embedding generation
36
+ */
37
+ export type EmbeddingResult = {
38
+ /**
39
+ * The embedding vector
40
+ */
41
+ vector: number[];
42
+ /**
43
+ * Number of tokens in the input text
44
+ */
45
+ tokens?: number;
46
+ /**
47
+ * Model used for generation
48
+ */
49
+ model: string;
50
+ };
51
+ /**
52
+ * Batch embedding result
53
+ */
54
+ export type BatchEmbeddingResult = {
55
+ /**
56
+ * Array of embedding vectors
57
+ */
58
+ vectors: number[][];
59
+ /**
60
+ * Total tokens processed
61
+ */
62
+ totalTokens?: number;
63
+ /**
64
+ * Model used for generation
65
+ */
66
+ model: string;
67
+ };
68
+ /**
69
+ * Options for embedding generation
70
+ */
71
+ export type EmbedOptions = {
72
+ /**
73
+ * Maximum number of texts to embed in a single batch
74
+ * @default 10
75
+ */
76
+ batchSize?: number;
77
+ /**
78
+ * Rate limit in requests per minute
79
+ * @default 100
80
+ */
81
+ rateLimit?: number;
82
+ /**
83
+ * Whether to show progress for large batches
84
+ * @default false
85
+ */
86
+ showProgress?: boolean;
87
+ };
88
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/providers/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IAEtB;;OAEG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAE3B;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAEtC;;;;;;OAMG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;CACjD;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B;;OAEG;IACH,MAAM,EAAE,MAAM,EAAE,CAAA;IAEhB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;CACd,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC;;OAEG;IACH,OAAO,EAAE,MAAM,EAAE,EAAE,CAAA;IAEnB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;CACd,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB,CAAA"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/providers/types.ts"],"names":[],"mappings":""}
@@ -0,0 +1,183 @@
1
+ /**
2
+ * Batch processing utilities with rate limiting and progress tracking
3
+ */
4
+ import type { EmbeddingProvider } from '../providers/types.js';
5
+ import type { StoredEmbedding } from '../config/types.js';
6
+ export interface BatchProcessOptions {
7
+ /**
8
+ * Embedding provider to use
9
+ */
10
+ provider: EmbeddingProvider;
11
+ /**
12
+ * Array of texts to process
13
+ */
14
+ texts: string[];
15
+ /**
16
+ * Batch size for processing
17
+ * @default 10
18
+ */
19
+ batchSize?: number;
20
+ /**
21
+ * Rate limit in requests per minute
22
+ * @default 100
23
+ */
24
+ rateLimit?: number;
25
+ /**
26
+ * Progress callback called after each batch
27
+ */
28
+ onProgress?: (progress: BatchProgress) => void;
29
+ /**
30
+ * Error callback called when a batch fails
31
+ * If not provided, errors will be thrown
32
+ */
33
+ onError?: (error: BatchError) => void;
34
+ /**
35
+ * Number of retries for failed batches
36
+ * @default 3
37
+ */
38
+ maxRetries?: number;
39
+ /**
40
+ * Initial retry delay in milliseconds
41
+ * @default 1000
42
+ */
43
+ retryDelay?: number;
44
+ /**
45
+ * Whether to include source hash in metadata
46
+ * @default true
47
+ */
48
+ includeSourceHash?: boolean;
49
+ }
50
+ export interface BatchProgress {
51
+ /**
52
+ * Number of items processed so far
53
+ */
54
+ processed: number;
55
+ /**
56
+ * Total number of items to process
57
+ */
58
+ total: number;
59
+ /**
60
+ * Number of items that failed
61
+ */
62
+ failed: number;
63
+ /**
64
+ * Percentage completed (0-100)
65
+ */
66
+ percentage: number;
67
+ /**
68
+ * Current batch number (1-indexed)
69
+ */
70
+ currentBatch: number;
71
+ /**
72
+ * Total number of batches
73
+ */
74
+ totalBatches: number;
75
+ }
76
+ export interface BatchError {
77
+ /**
78
+ * Batch number that failed
79
+ */
80
+ batchNumber: number;
81
+ /**
82
+ * Items in the failed batch
83
+ */
84
+ items: string[];
85
+ /**
86
+ * Error that occurred
87
+ */
88
+ error: Error;
89
+ /**
90
+ * Number of retry attempts made
91
+ */
92
+ retries: number;
93
+ }
94
+ export interface BatchProcessResult {
95
+ /**
96
+ * Successfully generated embeddings
97
+ */
98
+ embeddings: StoredEmbedding[];
99
+ /**
100
+ * Texts that failed to process
101
+ */
102
+ failed: Array<{
103
+ text: string;
104
+ error: Error;
105
+ }>;
106
+ /**
107
+ * Total processing statistics
108
+ */
109
+ stats: {
110
+ total: number;
111
+ successful: number;
112
+ failed: number;
113
+ duration: number;
114
+ };
115
+ }
116
+ /**
117
+ * Process embeddings in batches with rate limiting and retry logic
118
+ *
119
+ * @example
120
+ * ```typescript
121
+ * const result = await batchProcess({
122
+ * provider: createEmbeddingProvider({ type: 'openai', apiKey: '...' }),
123
+ * texts: largeTextArray,
124
+ * batchSize: 10,
125
+ * rateLimit: 60, // 60 requests per minute
126
+ * onProgress: (progress) => {
127
+ * console.log(`Progress: ${progress.percentage}%`)
128
+ * },
129
+ * })
130
+ *
131
+ * console.log(`Successfully processed: ${result.stats.successful}`)
132
+ * console.log(`Failed: ${result.stats.failed}`)
133
+ * ```
134
+ */
135
+ export declare function batchProcess(options: BatchProcessOptions): Promise<BatchProcessResult>;
136
+ /**
137
+ * Rate limiter for controlling API request rate
138
+ */
139
+ export declare class RateLimiter {
140
+ private queue;
141
+ private requestTimestamps;
142
+ private readonly requestsPerMinute;
143
+ constructor(requestsPerMinute: number);
144
+ /**
145
+ * Wait until rate limit allows next request
146
+ */
147
+ waitForSlot(): Promise<void>;
148
+ /**
149
+ * Execute a function with rate limiting
150
+ */
151
+ execute<T>(fn: () => Promise<T>): Promise<T>;
152
+ }
153
+ /**
154
+ * Queue for processing items with concurrency control
155
+ */
156
+ export declare class ProcessingQueue<T, R> {
157
+ private queue;
158
+ private processing;
159
+ private readonly concurrency;
160
+ private readonly processor;
161
+ constructor(processor: (item: T) => Promise<R>, concurrency?: number);
162
+ /**
163
+ * Add an item to the queue
164
+ */
165
+ add(item: T): Promise<R>;
166
+ /**
167
+ * Add multiple items to the queue
168
+ */
169
+ addBatch(items: T[]): Promise<R[]>;
170
+ /**
171
+ * Process next item in queue
172
+ */
173
+ private processNext;
174
+ /**
175
+ * Get current queue size
176
+ */
177
+ get size(): number;
178
+ /**
179
+ * Get number of items currently being processed
180
+ */
181
+ get activeCount(): number;
182
+ }
183
+ //# sourceMappingURL=batch.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../../src/runtime/batch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAC9D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AAGzD,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,QAAQ,EAAE,iBAAiB,CAAA;IAE3B;;OAEG;IACH,KAAK,EAAE,MAAM,EAAE,CAAA;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAA;IAE9C;;;OAGG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAA;IAErC;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAA;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IAEb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAElB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;IAEpB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,UAAU;IACzB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IAEnB;;OAEG;IACH,KAAK,EAAE,MAAM,EAAE,CAAA;IAEf;;OAEG;IACH,KAAK,EAAE,KAAK,CAAA;IAEZ;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,UAAU,EAAE,eAAe,EAAE,CAAA;IAE7B;;OAEG;IACH,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAA;KAAE,CAAC,CAAA;IAE7C;;OAEG;IACH,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAA;QACb,UAAU,EAAE,MAAM,CAAA;QAClB,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;KACjB,CAAA;CACF;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA+F5F;AAED;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,KAAK,CAAwB;IACrC,OAAO,CAAC,iBAAiB,CAAe;IACxC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAQ;gBAE9B,iBAAiB,EAAE,MAAM;IAIrC;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IA4BlC;;OAEG;IACG,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAInD;AAmDD;;GAEG;AACH,qBAAa,eAAe,CAAC,CAAC,EAAE,CAAC;IAC/B,OAAO,CAAC,KAAK,CACT;IACJ,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyB;gBAEvC,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,WAAW,GAAE,MAAU;IAKvE;;OAEG;IACG,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAO9B;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAIxC;;OAEG;YACW,WAAW;IAqBzB;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;OAEG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;CACF"}
@@ -0,0 +1,240 @@
1
+ /**
2
+ * Batch processing utilities with rate limiting and progress tracking
3
+ */
4
+ import { generateEmbeddings } from './embeddings.js';
5
+ /**
6
+ * Process embeddings in batches with rate limiting and retry logic
7
+ *
8
+ * @example
9
+ * ```typescript
10
+ * const result = await batchProcess({
11
+ * provider: createEmbeddingProvider({ type: 'openai', apiKey: '...' }),
12
+ * texts: largeTextArray,
13
+ * batchSize: 10,
14
+ * rateLimit: 60, // 60 requests per minute
15
+ * onProgress: (progress) => {
16
+ * console.log(`Progress: ${progress.percentage}%`)
17
+ * },
18
+ * })
19
+ *
20
+ * console.log(`Successfully processed: ${result.stats.successful}`)
21
+ * console.log(`Failed: ${result.stats.failed}`)
22
+ * ```
23
+ */
24
+ export async function batchProcess(options) {
25
+ const { provider, texts, batchSize = 10, rateLimit = 100, onProgress, onError, maxRetries = 3, retryDelay = 1000, includeSourceHash = true, } = options;
26
+ const startTime = Date.now();
27
+ const totalBatches = Math.ceil(texts.length / batchSize);
28
+ const embeddings = [];
29
+ const failed = [];
30
+ // Calculate delay between batches to respect rate limit
31
+ const delayBetweenBatches = calculateBatchDelay(rateLimit);
32
+ for (let i = 0; i < texts.length; i += batchSize) {
33
+ const batchNumber = Math.floor(i / batchSize) + 1;
34
+ const batch = texts.slice(i, i + batchSize);
35
+ try {
36
+ // Process batch with retry logic
37
+ const batchEmbeddings = await retryWithBackoff(async () => generateEmbeddings({
38
+ provider,
39
+ texts: batch,
40
+ includeSourceHash,
41
+ batchSize: batch.length,
42
+ }), maxRetries, retryDelay);
43
+ embeddings.push(...batchEmbeddings);
44
+ // Report progress
45
+ if (onProgress) {
46
+ const processed = Math.min(i + batchSize, texts.length);
47
+ onProgress({
48
+ processed,
49
+ total: texts.length,
50
+ failed: failed.length,
51
+ percentage: Math.round((processed / texts.length) * 100),
52
+ currentBatch: batchNumber,
53
+ totalBatches,
54
+ });
55
+ }
56
+ // Rate limiting: wait before next batch (except for last batch)
57
+ if (batchNumber < totalBatches && delayBetweenBatches > 0) {
58
+ await sleep(delayBetweenBatches);
59
+ }
60
+ }
61
+ catch (error) {
62
+ const batchError = {
63
+ batchNumber,
64
+ items: batch,
65
+ error: error instanceof Error ? error : new Error(String(error)),
66
+ retries: maxRetries,
67
+ };
68
+ if (onError) {
69
+ onError(batchError);
70
+ }
71
+ else {
72
+ // If no error handler, throw the error
73
+ throw error;
74
+ }
75
+ // Add all items in batch to failed list
76
+ for (const text of batch) {
77
+ failed.push({
78
+ text,
79
+ error: batchError.error,
80
+ });
81
+ }
82
+ }
83
+ }
84
+ const duration = Date.now() - startTime;
85
+ return {
86
+ embeddings,
87
+ failed,
88
+ stats: {
89
+ total: texts.length,
90
+ successful: embeddings.length,
91
+ failed: failed.length,
92
+ duration,
93
+ },
94
+ };
95
+ }
96
+ /**
97
+ * Rate limiter for controlling API request rate
98
+ */
99
+ export class RateLimiter {
100
+ queue = [];
101
+ requestTimestamps = [];
102
+ requestsPerMinute;
103
+ constructor(requestsPerMinute) {
104
+ this.requestsPerMinute = requestsPerMinute;
105
+ }
106
+ /**
107
+ * Wait until rate limit allows next request
108
+ */
109
+ async waitForSlot() {
110
+ return new Promise((resolve) => {
111
+ const tryAcquire = () => {
112
+ const now = Date.now();
113
+ const oneMinuteAgo = now - 60_000;
114
+ // Remove timestamps older than 1 minute
115
+ this.requestTimestamps = this.requestTimestamps.filter((t) => t > oneMinuteAgo);
116
+ // Check if we can make a request
117
+ if (this.requestTimestamps.length < this.requestsPerMinute) {
118
+ this.requestTimestamps.push(now);
119
+ resolve();
120
+ return;
121
+ }
122
+ // Calculate wait time until oldest request expires
123
+ const oldestTimestamp = this.requestTimestamps[0];
124
+ const waitTime = oldestTimestamp + 60_000 - now;
125
+ // Try again after wait time
126
+ setTimeout(tryAcquire, Math.max(waitTime, 100));
127
+ };
128
+ tryAcquire();
129
+ });
130
+ }
131
+ /**
132
+ * Execute a function with rate limiting
133
+ */
134
+ async execute(fn) {
135
+ await this.waitForSlot();
136
+ return fn();
137
+ }
138
+ }
139
+ /**
140
+ * Calculate delay between batches to respect rate limit
141
+ */
142
+ function calculateBatchDelay(requestsPerMinute) {
143
+ // Each batch counts as one request
144
+ const requestsPerSecond = requestsPerMinute / 60;
145
+ const delayPerRequest = 1000 / requestsPerSecond;
146
+ // Return delay in milliseconds
147
+ return Math.ceil(delayPerRequest);
148
+ }
149
+ /**
150
+ * Sleep for specified milliseconds
151
+ */
152
+ function sleep(ms) {
153
+ return new Promise((resolve) => setTimeout(resolve, ms));
154
+ }
155
+ /**
156
+ * Retry a function with exponential backoff
157
+ */
158
+ async function retryWithBackoff(fn, maxRetries, initialDelay) {
159
+ let lastError;
160
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
161
+ try {
162
+ return await fn();
163
+ }
164
+ catch (error) {
165
+ lastError = error instanceof Error ? error : new Error(String(error));
166
+ // Don't retry on last attempt
167
+ if (attempt === maxRetries) {
168
+ break;
169
+ }
170
+ // Exponential backoff: delay * 2^attempt
171
+ const delay = initialDelay * Math.pow(2, attempt);
172
+ await sleep(delay);
173
+ }
174
+ }
175
+ throw lastError;
176
+ }
177
+ /**
178
+ * Queue for processing items with concurrency control
179
+ */
180
+ export class ProcessingQueue {
181
+ queue = [];
182
+ processing = 0;
183
+ concurrency;
184
+ processor;
185
+ constructor(processor, concurrency = 1) {
186
+ this.processor = processor;
187
+ this.concurrency = concurrency;
188
+ }
189
+ /**
190
+ * Add an item to the queue
191
+ */
192
+ async add(item) {
193
+ return new Promise((resolve, reject) => {
194
+ this.queue.push({ item, resolve, reject });
195
+ this.processNext();
196
+ });
197
+ }
198
+ /**
199
+ * Add multiple items to the queue
200
+ */
201
+ async addBatch(items) {
202
+ return Promise.all(items.map((item) => this.add(item)));
203
+ }
204
+ /**
205
+ * Process next item in queue
206
+ */
207
+ async processNext() {
208
+ if (this.processing >= this.concurrency || this.queue.length === 0) {
209
+ return;
210
+ }
211
+ this.processing++;
212
+ const next = this.queue.shift();
213
+ if (next) {
214
+ try {
215
+ const result = await this.processor(next.item);
216
+ next.resolve(result);
217
+ }
218
+ catch (error) {
219
+ next.reject(error instanceof Error ? error : new Error(String(error)));
220
+ }
221
+ finally {
222
+ this.processing--;
223
+ this.processNext();
224
+ }
225
+ }
226
+ }
227
+ /**
228
+ * Get current queue size
229
+ */
230
+ get size() {
231
+ return this.queue.length;
232
+ }
233
+ /**
234
+ * Get number of items currently being processed
235
+ */
236
+ get activeCount() {
237
+ return this.processing;
238
+ }
239
+ }
240
+ //# sourceMappingURL=batch.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"batch.js","sourceRoot":"","sources":["../../src/runtime/batch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAmIpD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,EACJ,QAAQ,EACR,KAAK,EACL,SAAS,GAAG,EAAE,EACd,SAAS,GAAG,GAAG,EACf,UAAU,EACV,OAAO,EACP,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,IAAI,EACjB,iBAAiB,GAAG,IAAI,GACzB,GAAG,OAAO,CAAA;IAEX,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,CAAA;IACxD,MAAM,UAAU,GAAsB,EAAE,CAAA;IACxC,MAAM,MAAM,GAA0C,EAAE,CAAA;IAExD,wDAAwD;IACxD,MAAM,mBAAmB,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAA;IAE1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAA;QACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAA;QAE3C,IAAI,CAAC;YACH,iCAAiC;YACjC,MAAM,eAAe,GAAG,MAAM,gBAAgB,CAC5C,KAAK,IAAI,EAAE,CACT,kBAAkB,CAAC;gBACjB,QAAQ;gBACR,KAAK,EAAE,KAAK;gBACZ,iBAAiB;gBACjB,SAAS,EAAE,KAAK,CAAC,MAAM;aACxB,CAAC,EACJ,UAAU,EACV,UAAU,CACX,CAAA;YAED,UAAU,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAA;YAEnC,kBAAkB;YAClB,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;gBACvD,UAAU,CAAC;oBACT,SAAS;oBACT,KAAK,EAAE,KAAK,CAAC,MAAM;oBACnB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;oBACxD,YAAY,EAAE,WAAW;oBACzB,YAAY;iBACb,CAAC,CAAA;YACJ,CAAC;YAED,gEAAgE;YAChE,IAAI,WAAW,GAAG,YAAY,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;gBAC1D,MAAM,KAAK,CAAC,mBAAmB,CAAC,CAAA;YAClC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,UAAU,GAAe;gBAC7B,WAAW;gBACX,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChE,OAAO,EAAE,UAAU;aACpB,CAAA;YAED,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,UAAU,CAAC,CAAA;YACrB,CAAC;iBAAM,CAAC;gBACN,uCAAuC;gBACvC,MAAM,KAAK,CAAA;YACb,CAAC;YAED,wCAAwC;YACxC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI;oBACJ,KAAK,EAAE,UAAU,CAAC,KAAK;iBACxB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;IAEvC,OAAO;QACL,UAAU;QACV,MAAM;QACN,KAAK,EAAE;YACL,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,UAAU,EAAE,UAAU,CAAC,MAAM;YAC7B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ;SACT;KACF,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,WAAW;IACd,KAAK,GAAsB,EAAE,CAAA;IAC7B,iBAAiB,GAAa,EAAE,CAAA;IACvB,iBAAiB,CAAQ;IAE1C,YAAY,iBAAyB;QACnC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAA;IAC5C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,UAAU,GAAG,GAAG,EAAE;gBACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;gBACtB,MAAM,YAAY,GAAG,GAAG,GAAG,MAAM,CAAA;gBAEjC,wCAAwC;gBACxC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,CAAA;gBAE/E,iCAAiC;gBACjC,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAC3D,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;oBAChC,OAAO,EAAE,CAAA;oBACT,OAAM;gBACR,CAAC;gBAED,mDAAmD;gBACnD,MAAM,eAAe,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAA;gBACjD,MAAM,QAAQ,GAAG,eAAe,GAAG,MAAM,GAAG,GAAG,CAAA;gBAE/C,4BAA4B;gBAC5B,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAA;YACjD,CAAC,CAAA;YAED,UAAU,EAAE,CAAA;QACd,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAI,EAAoB;QACnC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAA;QACxB,OAAO,EAAE,EAAE,CAAA;IACb,CAAC;CACF;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,iBAAyB;IACpD,mCAAmC;IACnC,MAAM,iBAAiB,GAAG,iBAAiB,GAAG,EAAE,CAAA;IAChD,MAAM,eAAe,GAAG,IAAI,GAAG,iBAAiB,CAAA;IAEhD,+BAA+B;IAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;AACnC,CAAC;AAED;;GAEG;AACH,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,EAAoB,EACpB,UAAkB,EAClB,YAAoB;IAEpB,IAAI,SAA4B,CAAA;IAEhC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAA;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;YAErE,8BAA8B;YAC9B,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;gBAC3B,MAAK;YACP,CAAC;YAED,yCAAyC;YACzC,MAAM,KAAK,GAAG,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;YACjD,MAAM,KAAK,CAAC,KAAK,CAAC,CAAA;QACpB,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAA;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,eAAe;IAClB,KAAK,GACX,EAAE,CAAA;IACI,UAAU,GAAG,CAAC,CAAA;IACL,WAAW,CAAQ;IACnB,SAAS,CAAyB;IAEnD,YAAY,SAAkC,EAAE,cAAsB,CAAC;QACrE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAA;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,IAAO;QACf,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;YAC1C,IAAI,CAAC,WAAW,EAAE,CAAA;QACpB,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAU;QACvB,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACzD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW;QACvB,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnE,OAAM;QACR,CAAC;QAED,IAAI,CAAC,UAAU,EAAE,CAAA;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QAE/B,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAC9C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACtB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YACxE,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,UAAU,EAAE,CAAA;gBACjB,IAAI,CAAC,WAAW,EAAE,CAAA;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;IAC1B,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,UAAU,CAAA;IACxB,CAAC;CACF"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=batch.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"batch.test.d.ts","sourceRoot":"","sources":["../../src/runtime/batch.test.ts"],"names":[],"mappings":""}