@mcp-monorepo/notion-query 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/dist/index.js +56 -7
  2. package/dist/index.js.map +1 -1
  3. package/dist/lib/client.d.ts +9 -0
  4. package/dist/lib/client.d.ts.map +1 -0
  5. package/dist/lib/client.js +19 -0
  6. package/dist/lib/client.js.map +1 -0
  7. package/dist/lib/config.d.ts +44 -0
  8. package/dist/lib/config.d.ts.map +1 -0
  9. package/dist/lib/config.js +49 -0
  10. package/dist/lib/config.js.map +1 -0
  11. package/dist/lib/id-utils.d.ts +8 -0
  12. package/dist/lib/id-utils.d.ts.map +1 -0
  13. package/dist/lib/id-utils.js +20 -0
  14. package/dist/lib/id-utils.js.map +1 -0
  15. package/dist/lib/markdown-converter.d.ts +8 -0
  16. package/dist/lib/markdown-converter.d.ts.map +1 -0
  17. package/dist/lib/markdown-converter.js +95 -0
  18. package/dist/lib/markdown-converter.js.map +1 -0
  19. package/dist/lib/notion-syncer.d.ts +27 -0
  20. package/dist/lib/notion-syncer.d.ts.map +1 -0
  21. package/dist/lib/notion-syncer.js +212 -0
  22. package/dist/lib/notion-syncer.js.map +1 -0
  23. package/dist/lib/parser.d.ts +13 -0
  24. package/dist/lib/parser.d.ts.map +1 -0
  25. package/dist/lib/parser.js +88 -0
  26. package/dist/lib/parser.js.map +1 -0
  27. package/dist/lib/property-parser.d.ts +23 -0
  28. package/dist/lib/property-parser.d.ts.map +1 -0
  29. package/dist/lib/property-parser.js +200 -0
  30. package/dist/lib/property-parser.js.map +1 -0
  31. package/dist/lib/response-formatter.d.ts +16 -0
  32. package/dist/lib/response-formatter.d.ts.map +1 -0
  33. package/dist/lib/response-formatter.js +173 -0
  34. package/dist/lib/response-formatter.js.map +1 -0
  35. package/dist/lib/sync-state-manager.d.ts +29 -0
  36. package/dist/lib/sync-state-manager.d.ts.map +1 -0
  37. package/dist/lib/sync-state-manager.js +45 -0
  38. package/dist/lib/sync-state-manager.js.map +1 -0
  39. package/dist/local-rag/DEMO.d.ts +22 -0
  40. package/dist/local-rag/DEMO.d.ts.map +1 -0
  41. package/dist/local-rag/DEMO.js +142 -0
  42. package/dist/local-rag/DEMO.js.map +1 -0
  43. package/dist/local-rag/chunker.d.ts +24 -0
  44. package/dist/local-rag/chunker.d.ts.map +1 -0
  45. package/dist/local-rag/chunker.js +58 -0
  46. package/dist/local-rag/chunker.js.map +1 -0
  47. package/dist/local-rag/embedder.d.ts +43 -0
  48. package/dist/local-rag/embedder.d.ts.map +1 -0
  49. package/dist/local-rag/embedder.js +74 -0
  50. package/dist/local-rag/embedder.js.map +1 -0
  51. package/dist/local-rag/embedder.service.d.ts +15 -0
  52. package/dist/local-rag/embedder.service.d.ts.map +1 -0
  53. package/dist/local-rag/embedder.service.js +84 -0
  54. package/dist/local-rag/embedder.service.js.map +1 -0
  55. package/dist/local-rag/embedder.worker.d.ts +2 -0
  56. package/dist/local-rag/embedder.worker.d.ts.map +1 -0
  57. package/dist/local-rag/embedder.worker.js +34 -0
  58. package/dist/local-rag/embedder.worker.js.map +1 -0
  59. package/dist/local-rag/errors.d.ts +31 -0
  60. package/dist/local-rag/errors.d.ts.map +1 -0
  61. package/dist/local-rag/errors.js +47 -0
  62. package/dist/local-rag/errors.js.map +1 -0
  63. package/dist/local-rag/html-parser.d.ts +2 -0
  64. package/dist/local-rag/html-parser.d.ts.map +1 -0
  65. package/dist/local-rag/html-parser.js +32 -0
  66. package/dist/local-rag/html-parser.js.map +1 -0
  67. package/dist/local-rag/index.d.ts +67 -0
  68. package/dist/local-rag/index.d.ts.map +1 -0
  69. package/dist/local-rag/index.js +410 -0
  70. package/dist/local-rag/index.js.map +1 -0
  71. package/dist/local-rag/parser.d.ts +59 -0
  72. package/dist/local-rag/parser.d.ts.map +1 -0
  73. package/dist/local-rag/parser.js +206 -0
  74. package/dist/local-rag/parser.js.map +1 -0
  75. package/dist/local-rag/types.d.ts +209 -0
  76. package/dist/local-rag/types.d.ts.map +1 -0
  77. package/dist/local-rag/types.js +5 -0
  78. package/dist/local-rag/types.js.map +1 -0
  79. package/dist/local-rag/utils/pool.d.ts +60 -0
  80. package/dist/local-rag/utils/pool.d.ts.map +1 -0
  81. package/dist/local-rag/utils/pool.js +140 -0
  82. package/dist/local-rag/utils/pool.js.map +1 -0
  83. package/dist/local-rag/utils/typed-emitter.d.ts +28 -0
  84. package/dist/local-rag/utils/typed-emitter.d.ts.map +1 -0
  85. package/dist/local-rag/utils/typed-emitter.js +44 -0
  86. package/dist/local-rag/utils/typed-emitter.js.map +1 -0
  87. package/dist/local-rag/vectordb/index.d.ts +91 -0
  88. package/dist/local-rag/vectordb/index.d.ts.map +1 -0
  89. package/dist/local-rag/vectordb/index.js +278 -0
  90. package/dist/local-rag/vectordb/index.js.map +1 -0
  91. package/dist/local-rag/vectordb/manager.d.ts +28 -0
  92. package/dist/local-rag/vectordb/manager.d.ts.map +1 -0
  93. package/dist/local-rag/vectordb/manager.js +91 -0
  94. package/dist/local-rag/vectordb/manager.js.map +1 -0
  95. package/dist/local-rag/vectordb/migration.d.ts +27 -0
  96. package/dist/local-rag/vectordb/migration.d.ts.map +1 -0
  97. package/dist/local-rag/vectordb/migration.js +121 -0
  98. package/dist/local-rag/vectordb/migration.js.map +1 -0
  99. package/dist/local-rag/vectordb/retriever.d.ts +51 -0
  100. package/dist/local-rag/vectordb/retriever.d.ts.map +1 -0
  101. package/dist/local-rag/vectordb/retriever.js +157 -0
  102. package/dist/local-rag/vectordb/retriever.js.map +1 -0
  103. package/dist/local-rag/vectordb/schema.d.ts +33 -0
  104. package/dist/local-rag/vectordb/schema.d.ts.map +1 -0
  105. package/dist/local-rag/vectordb/schema.js +102 -0
  106. package/dist/local-rag/vectordb/schema.js.map +1 -0
  107. package/dist/local-rag/watcher.d.ts +48 -0
  108. package/dist/local-rag/watcher.d.ts.map +1 -0
  109. package/dist/local-rag/watcher.js +102 -0
  110. package/dist/local-rag/watcher.js.map +1 -0
  111. package/dist/tools/create-pages.d.ts +4 -0
  112. package/dist/tools/create-pages.d.ts.map +1 -0
  113. package/dist/tools/create-pages.js +184 -0
  114. package/dist/tools/create-pages.js.map +1 -0
  115. package/dist/tools/fetch.d.ts +4 -0
  116. package/dist/tools/fetch.d.ts.map +1 -0
  117. package/dist/tools/fetch.js +90 -0
  118. package/dist/tools/fetch.js.map +1 -0
  119. package/dist/tools/query-datasource.d.ts +4 -0
  120. package/dist/tools/query-datasource.d.ts.map +1 -0
  121. package/dist/tools/{notion-query.js → query-datasource.js} +31 -38
  122. package/dist/tools/query-datasource.js.map +1 -0
  123. package/dist/tools/search.d.ts +12 -0
  124. package/dist/tools/search.d.ts.map +1 -0
  125. package/dist/tools/search.js +75 -0
  126. package/dist/tools/search.js.map +1 -0
  127. package/dist/tools/update-page.d.ts +4 -0
  128. package/dist/tools/update-page.d.ts.map +1 -0
  129. package/dist/tools/update-page.js +135 -0
  130. package/dist/tools/update-page.js.map +1 -0
  131. package/package.json +23 -8
  132. package/dist/tools/notion-query.d.ts +0 -3
  133. package/dist/tools/notion-query.d.ts.map +0 -1
  134. package/dist/tools/notion-query.js.map +0 -1
@@ -0,0 +1,206 @@
1
+ import { readFile, stat } from 'node:fs/promises';
2
+ import { extname, isAbsolute, resolve } from 'node:path';
3
+ import mammoth from 'mammoth';
4
+ import * as Parse from 'papaparse';
5
+ import { PDFParse } from 'pdf-parse';
6
+ import { FileOperationError, ValidationError } from './errors.js';
7
+ /**
8
+ * Supported code file extensions mapped to a language name.
9
+ */
10
+ const CODE_EXTENSIONS = {
11
+ '.ts': 'typescript',
12
+ '.tsx': 'typescript',
13
+ '.js': 'javascript',
14
+ '.jsx': 'javascript',
15
+ '.py': 'python',
16
+ '.java': 'java',
17
+ '.go': 'go',
18
+ '.rs': 'rust',
19
+ '.c': 'c',
20
+ '.cpp': 'cpp',
21
+ '.cc': 'cpp',
22
+ '.cxx': 'cpp',
23
+ '.h': 'c',
24
+ '.hpp': 'cpp',
25
+ '.rb': 'ruby',
26
+ '.php': 'php',
27
+ '.cs': 'csharp',
28
+ '.sh': 'bash',
29
+ '.sql': 'sql',
30
+ '.json': 'json',
31
+ '.csv': 'csv',
32
+ };
33
+ /**
34
+ * Parses text content from various file formats.
35
+ * Extracts basic metadata and transforms structured content for better RAG performance.
36
+ */
37
+ export class DocumentParser {
38
+ config;
39
+ constructor(config) {
40
+ this.config = config;
41
+ }
42
+ /**
43
+ * Parses a file, automatically detecting its format.
44
+ * @param filePath - The path to the file to parse.
45
+ * @returns The extracted text, detected language (for code files), and file size.
46
+ */
47
+ async parseFile(filePath) {
48
+ this.validateFilePath(filePath);
49
+ const { fileSize, fileCreatedAt, fileModifiedAt } = await this.validateAndGetFileStats(filePath);
50
+ const baseMetadata = { fileCreatedAt, fileModifiedAt };
51
+ const ext = extname(filePath).toLowerCase();
52
+ const language = this.detectLanguage(filePath);
53
+ let parsedContent;
54
+ if (language && !['.json', '.csv'].includes(ext)) {
55
+ parsedContent = { text: await this.parseAsPlainText(filePath) };
56
+ }
57
+ else {
58
+ switch (ext) {
59
+ case '.pdf':
60
+ parsedContent = await this.parsePdf(filePath);
61
+ break;
62
+ case '.docx':
63
+ parsedContent = await this.parseDocx(filePath);
64
+ break;
65
+ case '.json':
66
+ parsedContent = { text: await this.parseJson(filePath) };
67
+ break;
68
+ case '.csv':
69
+ parsedContent = { text: await this.parseStructuredCsv(filePath) };
70
+ break;
71
+ case '.txt':
72
+ case '.md':
73
+ case '.markdown':
74
+ parsedContent = { text: await this.parseAsPlainText(filePath) };
75
+ break;
76
+ default:
77
+ throw new ValidationError(`Unsupported file format: ${ext}`);
78
+ }
79
+ }
80
+ return {
81
+ text: parsedContent.text,
82
+ language,
83
+ fileSize,
84
+ metadata: { ...baseMetadata, ...parsedContent.metadata },
85
+ };
86
+ }
87
+ /**
88
+ * Returns a list of all supported file extensions.
89
+ * @returns An array of file extensions (e.g., ['.pdf', '.ts', '.md']).
90
+ */
91
+ getSupportedExtensions() {
92
+ const documentExts = ['.pdf', '.docx', '.txt', '.md', '.markdown', '.json', '.csv'];
93
+ const codeExts = Object.keys(CODE_EXTENSIONS);
94
+ return [...new Set([...documentExts, ...codeExts])];
95
+ }
96
+ /**
97
+ * Validates that a file path is absolute and within the configured `baseDir`.
98
+ * @param filePath - The absolute path to validate.
99
+ * @throws {ValidationError} If the path is invalid or outside the security boundary.
100
+ */
101
+ validateFilePath(filePath) {
102
+ if (!isAbsolute(filePath)) {
103
+ throw new ValidationError(`File path must be absolute. Received: ${filePath}`);
104
+ }
105
+ const baseDir = resolve(this.config.baseDir);
106
+ const normalizedPath = resolve(filePath);
107
+ if (!normalizedPath.startsWith(baseDir)) {
108
+ throw new ValidationError(`File path is outside the allowed base directory. Base: ${baseDir}, Path: ${filePath}`);
109
+ }
110
+ }
111
+ /**
112
+ * Validates that a file's size is within the configured limit.
113
+ * @param filePath - The path to the file to check.
114
+ * @returns The file size in bytes.
115
+ * @throws {ValidationError} If the file is too large.
116
+ * @throws {FileOperationError} If the file stats cannot be read.
117
+ */
118
+ async validateAndGetFileStats(filePath) {
119
+ try {
120
+ const stats = await stat(filePath);
121
+ if (stats.size > this.config.maxFileSize) {
122
+ throw new ValidationError(`File size ${stats.size} exceeds limit of ${this.config.maxFileSize} bytes.`);
123
+ }
124
+ return {
125
+ fileSize: stats.size,
126
+ fileCreatedAt: stats.birthtime.toISOString(),
127
+ fileModifiedAt: stats.mtime.toISOString(),
128
+ };
129
+ }
130
+ catch (error) {
131
+ if (error instanceof ValidationError)
132
+ throw error;
133
+ throw new FileOperationError(`Failed to get file stats: ${filePath}`, error);
134
+ }
135
+ }
136
+ detectLanguage(filePath) {
137
+ return CODE_EXTENSIONS[extname(filePath).toLowerCase()];
138
+ }
139
+ async parsePdf(filePath) {
140
+ try {
141
+ const buffer = await readFile(filePath);
142
+ const pdf = new PDFParse({ data: buffer });
143
+ const data = await pdf.getText();
144
+ const info = await pdf.getInfo();
145
+ const author = info.info.Author?.trim();
146
+ return { text: data.text, metadata: { author } };
147
+ }
148
+ catch (error) {
149
+ throw new FileOperationError(`Failed to parse PDF: ${filePath}`, error);
150
+ }
151
+ }
152
+ async parseDocx(filePath) {
153
+ try {
154
+ // Note: mammoth.extractRawText does not support metadata extraction (e.g., author).
155
+ // A more complex setup would be needed to get this information.
156
+ const { value } = await mammoth.extractRawText({ path: filePath });
157
+ return { text: value, metadata: {} };
158
+ }
159
+ catch (error) {
160
+ throw new FileOperationError(`Failed to parse DOCX: ${filePath}`, error);
161
+ }
162
+ }
163
+ async parseJson(filePath) {
164
+ try {
165
+ const content = await readFile(filePath, 'utf-8');
166
+ const data = JSON.parse(content);
167
+ // Pretty-print the JSON to make it more readable for the LLM.
168
+ return JSON.stringify(data, undefined, 2);
169
+ }
170
+ catch (error) {
171
+ throw new FileOperationError(`Failed to parse JSON: ${filePath}`, error);
172
+ }
173
+ }
174
+ /**
175
+ * Parses CSV into a text format where each row preserves the header context.
176
+ * This is crucial for the semantic meaning of chunks.
177
+ */
178
+ async parseStructuredCsv(filePath) {
179
+ try {
180
+ const content = await readFile(filePath, 'utf-8');
181
+ const result = Parse.parse(content, { header: true, skipEmptyLines: true });
182
+ if (result.errors.length > 0) {
183
+ throw new Error(`CSV parsing errors: ${result.errors.map((e) => e.message).join(', ')}`);
184
+ }
185
+ // Convert each row object into a descriptive string.
186
+ // E.g., { "name": "Alice", "age": 30 } becomes "name: Alice, age: 30"
187
+ return result.data
188
+ .map((row) => Object.entries(row)
189
+ .map(([key, value]) => `${key.trim()}: ${value?.trim() ?? ''}`)
190
+ .join(', '))
191
+ .join('\n');
192
+ }
193
+ catch (error) {
194
+ throw new FileOperationError(`Failed to parse CSV: ${filePath}`, error);
195
+ }
196
+ }
197
+ async parseAsPlainText(filePath) {
198
+ try {
199
+ return await readFile(filePath, 'utf-8');
200
+ }
201
+ catch (error) {
202
+ throw new FileOperationError(`Failed to read text file: ${filePath}`, error);
203
+ }
204
+ }
205
+ }
206
+ //# sourceMappingURL=parser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/local-rag/parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAExD,OAAO,OAAO,MAAM,SAAS,CAAA;AAC7B,OAAO,KAAK,KAAK,MAAM,WAAW,CAAA;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAEpC,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIjE;;GAEG;AACH,MAAM,eAAe,GAA2B;IAC9C,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,YAAY;IACpB,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,YAAY;IACpB,KAAK,EAAE,QAAQ;IACf,OAAO,EAAE,MAAM;IACf,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,MAAM;IACb,IAAI,EAAE,GAAG;IACT,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,KAAK;IACb,IAAI,EAAE,GAAG;IACT,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;CACL,CAAA;AAcV;;;GAGG;AACH,MAAM,OAAO,cAAc;IACR,MAAM,CAAc;IAErC,YAAY,MAAoB;QAC9B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,SAAS,CAAC,QAAgB;QACrC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;QAC/B,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,CAAA;QAEhG,MAAM,YAAY,GAA8B,EAAE,aAAa,EAAE,cAAc,EAAE,CAAA;QACjF,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAA;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAA;QAE9C,IAAI,aAAqE,CAAA;QAEzE,IAAI,QAAQ,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,aAAa,GAAG,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAA;QACjE,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,EAAE,CAAC;gBACZ,KAAK,MAAM;oBACT,aAAa,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;oBAC7C,MAAK;gBACP,KAAK,OAAO;oBACV,aAAa,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;oBAC9C,MAAK;gBACP,KAAK,OAAO;oBACV,aAAa,GAAG,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAA;oBACxD,MAAK;gBACP,KAAK,MAAM;oBACT,aAAa,GAAG,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAA;oBACjE,MAAK;gBACP,KAAK,MAAM,CAAC;gBACZ,KAAK,KAAK,CAAC;gBACX,KAAK,WAAW;oBACd,aAAa,GAAG,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAA;oBAC/D,MAAK;gBACP;oBACE,MAAM,IAAI,eAAe,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,aAAa,CAAC,IAAI;YACxB,QAAQ;YACR,QAAQ;YACR,QAAQ,EAAE,EAAE,GAAG,YAAY,EAAE,GAAG,aAAa,CAAC,QAAQ,EAAE;SACzD,CAAA;IACH,CAAC;IAED;;;OAGG;IACI,sBAAsB;QAC3B,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QACnF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;QAC7C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;IACrD,CAAC;IAED;;;;OAIG;IACI,gBAAgB,CAAC,QAAgB;QACtC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,eAAe,CAAC,yCAAyC,QAAQ,EAAE,CAAC,CAAA;QAChF,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;QAExC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,eAAe,CAAC,0DAA0D,OAAO,WAAW,QAAQ,EAAE,CAAC,CAAA;QACnH,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,uBAAuB,CAClC,QAAgB;QAEhB,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;YAClC,IAAI,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBACzC,MAAM,IAAI,eAAe,CAAC,aAAa,KAAK,CAAC,IAAI,qBAAqB,IAAI,CAAC,MAAM,CAAC,WAAW,SAAS,CAAC,CAAA;YACzG,CAAC;YACD,OAAO;gBACL,QAAQ,EAAE,KAAK,CAAC,IAAI;gBACpB,aAAa,EAAE,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;gBAC5C,cAAc,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;aAC1C,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,eAAe;gBAAE,MAAM,KAAK,CAAA;YACjD,MAAM,IAAI,kBAAkB,CAAC,6BAA6B,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAA;QAC9E,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,QAAgB;QACrC,OAAO,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;IACzD,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACvC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAA;YAC1C,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,CAAA;YAChC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,CAAA;YAChC,MAAM,MAAM,GAAI,IAAI,CAAC,IAA+B,CAAC,MAAM,EAAE,IAAI,EAAE,CAAA;YACnE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,EAAE,CAAA;QAClD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAAC,wBAAwB,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAA;QACzE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,QAAgB;QACtC,IAAI,CAAC;YACH,oFAAoF;YACpF,gEAAgE;YAChE,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAA;YAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAA;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAAC,yBAAyB,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAA;QAC1E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,QAAgB;QACtC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;YACjD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YAChC,8DAA8D;YAC9D,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAA;QAC3C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAAC,yBAAyB,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAA;QAC1E,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,kBAAkB,CAAC,QAAgB;QAC/C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAA;YAE3E,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,uBAAuB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAC1F,CAAC;YAED,qDAAqD;YACrD,sEAAsE;YACtE,OAAO,MAAM,CAAC,IAAI;iBACf,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACX,MAAM,CAAC,OAAO,CAAC,GAA6B,CAAC;iBAC1C,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,KAAK,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;iBAC9D,IAAI,CAAC,IAAI,CAAC,CACd;iBACA,IAAI,CAAC,IAAI,CAAC,CAAA;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAAC,wBAAwB,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAA;QACzE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QAC7C,IAAI,CAAC;YACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAAC,6BAA6B,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAA;QAC9E,CAAC;IACH,CAAC;CACF"}
@@ -0,0 +1,209 @@
1
+ /**
2
+ * Configuration options for the LocalRAG instance.
3
+ */
4
+ export interface LocalRAGConfig {
5
+ /** Path to the LanceDB database directory. */
6
+ dbPath: string;
7
+ /** Path to the directory for caching embedding models. */
8
+ cacheDir?: string;
9
+ /** The root directory to which file access is restricted for security. */
10
+ baseDir?: string;
11
+ /** HuggingFace model identifier for text embeddings. */
12
+ modelName?: string;
13
+ /** Maximum file size in bytes for ingestion. Defaults to 100MB. */
14
+ maxFileSize?: number;
15
+ /** The target size of each text chunk in characters. */
16
+ chunkSize?: number;
17
+ /** The number of characters to overlap between adjacent chunks. */
18
+ chunkOverlap?: number;
19
+ /**
20
+ * The debounce time in milliseconds for file watcher events to avoid rapid re-indexing.
21
+ * @default 5000
22
+ */
23
+ debounceMs?: number;
24
+ /**
25
+ * Interval in milliseconds to automatically run the cleanup job for expired documents.
26
+ * If not set, cleanup must be triggered manually.
27
+ * @example 86400000 (for 24 hours)
28
+ */
29
+ cleanupIntervalMs?: number;
30
+ /**
31
+ * Interval in milliseconds to automatically run the database optimization job (compacts files, rebuilds indexes).
32
+ * Recommended for write-heavy applications.
33
+ * @example 3600000 (for 1 hour)
34
+ */
35
+ optimizeIntervalMs?: number;
36
+ /** Advanced options for tuning the retrieval process. */
37
+ retrievalOptions?: RetrievalOptions;
38
+ /** Configuration for the embedding worker pool. */
39
+ poolConfig?: PoolConfig;
40
+ }
41
+ /**
42
+ * Configuration options for the dynamic worker pool.
43
+ */
44
+ export interface PoolConfig {
45
+ /**
46
+ * The maximum number of workers the pool can scale up to.
47
+ * Defaults to the number of CPU cores minus one.
48
+ */
49
+ maxWorkers?: number;
50
+ /**
51
+ * The minimum number of workers to keep alive, even if idle.
52
+ * @default 0
53
+ */
54
+ minWorkers?: number;
55
+ /**
56
+ * The time in milliseconds an idle worker will wait before being terminated.
57
+ * Set to 0 to disable automatic scaling down.
58
+ * @default 1800000 (30 minutes)
59
+ */
60
+ idleTimeoutMs?: number;
61
+ }
62
+ /**
63
+ * Represents a persisted watched path in the database.
64
+ */
65
+ export interface WatchedPath {
66
+ path: string;
67
+ type: 'file' | 'folder';
68
+ recursive: boolean;
69
+ addedAt: string;
70
+ }
71
+ /**
72
+ * Advanced options for tuning search and retrieval.
73
+ */
74
+ export interface RetrievalOptions {
75
+ /**
76
+ * The weight of keyword (BM25) search vs. semantic (vector) search in hybrid search.
77
+ * 0.0 = Purely semantic, 1.0 = Purely keyword.
78
+ * Recommended for code/technical docs: 0.6 - 0.7.
79
+ * @default 0.6
80
+ */
81
+ hybridWeight?: number;
82
+ /**
83
+ * Filters results by detecting relevance gaps.
84
+ * - 'similar': Returns only the most relevant group of results.
85
+ * - 'related': Returns the top two most relevant groups.
86
+ * - undefined: No grouping, returns all results up to the limit.
87
+ */
88
+ grouping?: GroupingMode;
89
+ /**
90
+ * An absolute distance threshold. Results with a score greater than this are discarded.
91
+ * For dot product, a lower score is better (0 = identical).
92
+ * E.g., a value of 0.5 would filter out less relevant matches.
93
+ */
94
+ maxDistance?: number;
95
+ }
96
+ export type GroupingMode = 'similar' | 'related';
97
+ /**
98
+ * Metadata associated with an ingested document or text snippet.
99
+ */
100
+ export interface DocumentMetadata {
101
+ fileName: string;
102
+ fileSize: number;
103
+ fileType: string;
104
+ language?: string;
105
+ tags?: string[];
106
+ project?: string;
107
+ memoryType?: 'file' | 'text' | 'url';
108
+ expiresAt?: string;
109
+ createdAt: string;
110
+ updatedAt: string;
111
+ sourceUrl?: string;
112
+ author?: string;
113
+ fileCreatedAt?: string;
114
+ fileModifiedAt?: string;
115
+ }
116
+ /**
117
+ * A chunk of text ready to be inserted into the vector store.
118
+ */
119
+ export interface VectorChunk {
120
+ id: string;
121
+ filePath: string;
122
+ chunkIndex: number;
123
+ text: string;
124
+ vector: number[];
125
+ metadata?: DocumentMetadata;
126
+ timestamp: string;
127
+ }
128
+ export interface WatchOptions {
129
+ /** For folders, specifies whether to watch subdirectories. @default false */
130
+ recursive?: boolean;
131
+ }
132
+ export interface IngestFileInput {
133
+ filePath: string;
134
+ tags?: string[];
135
+ project?: string;
136
+ /** If true, the file will be monitored for changes. */
137
+ watch?: boolean;
138
+ }
139
+ export interface IngestFolderInput {
140
+ folderPath: string;
141
+ tags?: string[];
142
+ project?: string;
143
+ /** If true, the folder will be monitored for new, changed, and deleted files. */
144
+ watch?: boolean;
145
+ /** If true, subdirectories will also be processed and watched. @default false */
146
+ recursive?: boolean;
147
+ }
148
+ export interface IngestTextInupt {
149
+ text: string;
150
+ label: string;
151
+ language?: string;
152
+ tags?: string[];
153
+ project?: string;
154
+ ttl?: string;
155
+ }
156
+ export interface IngestUrlInput {
157
+ url: string;
158
+ tags?: string[];
159
+ project?: string;
160
+ ttl?: string;
161
+ }
162
+ export interface QueryInput {
163
+ query: string;
164
+ limit?: number;
165
+ filters?: QueryFilters;
166
+ }
167
+ export interface QueryFilters {
168
+ type?: 'file' | 'text' | 'url';
169
+ tags?: string[];
170
+ project?: string;
171
+ fileName?: string;
172
+ }
173
+ export interface QueryResult {
174
+ filePath: string;
175
+ chunkIndex: number;
176
+ text: string;
177
+ score: number;
178
+ metadata?: DocumentMetadata;
179
+ }
180
+ export interface UpdateMemoryInput {
181
+ label: string;
182
+ mode?: 'replace' | 'append' | 'prepend';
183
+ text?: string;
184
+ tags?: string[];
185
+ addTags?: string[];
186
+ removeTags?: string[];
187
+ }
188
+ export interface ListOptions {
189
+ /** The number of items to return. @default 20 */
190
+ limit?: number;
191
+ /** The number of items to skip for pagination. @default 0 */
192
+ offset?: number;
193
+ filters?: QueryFilters;
194
+ }
195
+ export interface ListItem {
196
+ filePath: string;
197
+ chunkCount: number;
198
+ timestamp: string;
199
+ metadata?: DocumentMetadata;
200
+ }
201
+ export interface StatusReport {
202
+ documentCount: number;
203
+ chunkCount: number;
204
+ memoryUsageMb: number;
205
+ uptimeSeconds: number;
206
+ ftsIndexEnabled: boolean;
207
+ searchMode: 'hybrid' | 'vector-only';
208
+ }
209
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/local-rag/types.ts"],"names":[],"mappings":"AAIA;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8CAA8C;IAC9C,MAAM,EAAE,MAAM,CAAA;IACd,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,0EAA0E;IAC1E,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,mEAAmE;IACnE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,mEAAmE;IACnE,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,yDAAyD;IACzD,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC,mDAAmD;IACnD,UAAU,CAAC,EAAE,UAAU,CAAA;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAA;IACvB,SAAS,EAAE,OAAO,CAAA;IAClB,OAAO,EAAE,MAAM,CAAA;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,YAAY,CAAA;IACvB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,SAAS,CAAA;AAMhD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAA;IACpC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAGlB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAA;IACV,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IAC3B,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,6EAA6E;IAC7E,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAMD,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,uDAAuD;IACvD,KAAK,CAAC,EAAE,OAAO,CAAA;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,iFAAiF;IACjF,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,iFAAiF;IACjF,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,GAAG,CAAC,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,GAAG,CAAC,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,YAAY,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAA;IAC9B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,gBAAgB,CAAA;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,SAAS,GAAG,QAAQ,GAAG,SAAS,CAAA;IACvC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,iDAAiD;IACjD,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,6DAA6D;IAC7D,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,YAAY,CAAA;CACvB;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE,gBAAgB,CAAA;CAC5B;AAED,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;IACrB,eAAe,EAAE,OAAO,CAAA;IACxB,UAAU,EAAE,QAAQ,GAAG,aAAa,CAAA;CACrC"}
@@ -0,0 +1,5 @@
1
+ // ============================================
2
+ // Main Configuration
3
+ // ============================================
4
+ export {};
5
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/local-rag/types.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAC/C,qBAAqB;AACrB,+CAA+C"}
@@ -0,0 +1,60 @@
1
+ import type { PoolConfig } from '../types.js';
2
+ export interface SerializedError {
3
+ name: string;
4
+ message: string;
5
+ stack?: string;
6
+ cause?: unknown;
7
+ }
8
+ export type EmbeddingTask = {
9
+ type: 'embed';
10
+ payload: {
11
+ text: string;
12
+ };
13
+ } | {
14
+ type: 'embedBatch';
15
+ payload: {
16
+ texts: string[];
17
+ };
18
+ };
19
+ export type EmbeddingSuccess = {
20
+ taskId: number;
21
+ status: 'success';
22
+ type: 'embed';
23
+ result: number[];
24
+ } | {
25
+ taskId: number;
26
+ status: 'success';
27
+ type: 'embedBatch';
28
+ result: number[][];
29
+ };
30
+ export type EmbeddingResult = EmbeddingSuccess | {
31
+ taskId: number;
32
+ status: 'error';
33
+ error: SerializedError;
34
+ };
35
+ export declare class WorkerPool {
36
+ private readonly workerPath;
37
+ private readonly workerData;
38
+ private readonly config;
39
+ private workers;
40
+ private readonly taskQueue;
41
+ private taskIdCounter;
42
+ private workerIdCounter;
43
+ private readonly activeTasks;
44
+ private idleCheckInterval;
45
+ constructor(workerPath: URL, workerData: object, config?: PoolConfig);
46
+ runTask(task: EmbeddingTask & {
47
+ type: 'embed';
48
+ }): Promise<number[]>;
49
+ runTask(task: EmbeddingTask & {
50
+ type: 'embedBatch';
51
+ }): Promise<number[][]>;
52
+ private _dispatch;
53
+ private _createWorker;
54
+ private _assignTask;
55
+ private _handleTaskCompletion;
56
+ private _handleWorkerCrash;
57
+ private _reapIdleWorkers;
58
+ destroy(): Promise<void>;
59
+ }
60
+ //# sourceMappingURL=pool.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pool.d.ts","sourceRoot":"","sources":["../../../src/local-rag/utils/pool.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAE7C,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,OAAO,CAAA;CAChB;AACD,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,EAAE,CAAA;KAAE,CAAA;CAAE,CAAA;AAExD,MAAM,MAAM,gBAAgB,GACxB;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,GACtE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAA;CAAE,CAAA;AAEjF,MAAM,MAAM,eAAe,GAAG,gBAAgB,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,eAAe,CAAA;CAAE,CAAA;AAgB5G,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAK;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAE7C,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiB;IAC3C,OAAO,CAAC,aAAa,CAAI;IACzB,OAAO,CAAC,eAAe,CAAI;IAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA0D;IACtF,OAAO,CAAC,iBAAiB,CAA4B;gBAEzC,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAE,UAAe;IAejE,OAAO,CAAC,IAAI,EAAE,aAAa,GAAG;QAAE,IAAI,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IACnE,OAAO,CAAC,IAAI,EAAE,aAAa,GAAG;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAUjF,OAAO,CAAC,SAAS;IA8BjB,OAAO,CAAC,aAAa;IAoBrB,OAAO,CAAC,WAAW;IAKnB,OAAO,CAAC,qBAAqB;IAoB7B,OAAO,CAAC,kBAAkB;IAmB1B,OAAO,CAAC,gBAAgB;IAoBX,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAOtC"}
@@ -0,0 +1,140 @@
1
+ import { cpus } from 'node:os';
2
+ import { Worker } from 'node:worker_threads';
3
+ import { logger } from '@mcp-monorepo/shared';
4
+ import { EmbeddingError } from '../errors.js';
5
+ export class WorkerPool {
6
+ workerPath;
7
+ workerData;
8
+ config;
9
+ workers = [];
10
+ taskQueue = [];
11
+ taskIdCounter = 0;
12
+ workerIdCounter = 0;
13
+ activeTasks = new Map();
14
+ idleCheckInterval;
15
+ constructor(workerPath, workerData, config = {}) {
16
+ this.workerPath = workerPath;
17
+ this.workerData = workerData;
18
+ this.config = {
19
+ maxWorkers: config.maxWorkers ?? Math.max(1, cpus().length - 1),
20
+ minWorkers: config.minWorkers ?? 0,
21
+ idleTimeoutMs: config.idleTimeoutMs ?? 1_800_000,
22
+ };
23
+ if (this.config.idleTimeoutMs > 0) {
24
+ this.idleCheckInterval = setInterval(() => this._reapIdleWorkers(), this.config.idleTimeoutMs / 2);
25
+ }
26
+ }
27
+ runTask(task) {
28
+ const taskId = this.taskIdCounter++;
29
+ return new Promise((resolve, reject) => {
30
+ this.activeTasks.set(taskId, { resolve, reject });
31
+ this.taskQueue.push({ taskId, task, resolve, reject });
32
+ this._dispatch();
33
+ });
34
+ }
35
+ _dispatch() {
36
+ if (this.taskQueue.length === 0) {
37
+ return;
38
+ }
39
+ // 1. Prioritize using an existing idle worker
40
+ const availableWorker = this.workers.find((w) => w.currentTaskId === undefined);
41
+ if (availableWorker) {
42
+ const taskInfo = this.taskQueue.shift();
43
+ if (taskInfo) {
44
+ this._assignTask(availableWorker, taskInfo);
45
+ }
46
+ return;
47
+ }
48
+ // 2. If no idle worker, create a new one if we're not at the max limit
49
+ if (this.workers.length < this.config.maxWorkers) {
50
+ const taskInfo = this.taskQueue.shift();
51
+ if (taskInfo) {
52
+ logger.info(`Scaling up workers to ${this.workers.length + 1} to handle load...`);
53
+ const newWorker = this._createWorker();
54
+ this.workers.push(newWorker);
55
+ this._assignTask(newWorker, taskInfo);
56
+ }
57
+ }
58
+ // 3. If we are at the max limit and all workers are busy, the task remains queued.
59
+ // It will be picked up when a worker finishes its current task.
60
+ }
61
+ _createWorker() {
62
+ const worker = new Worker(this.workerPath, { workerData: this.workerData });
63
+ const trackedWorker = {
64
+ id: ++this.workerIdCounter,
65
+ worker,
66
+ currentTaskId: undefined,
67
+ lastUsed: Date.now(),
68
+ };
69
+ worker.on('message', (message) => {
70
+ this._handleTaskCompletion(trackedWorker, message);
71
+ });
72
+ worker.on('error', (err) => {
73
+ this._handleWorkerCrash(trackedWorker, err);
74
+ });
75
+ return trackedWorker;
76
+ }
77
+ _assignTask(worker, taskInfo) {
78
+ worker.currentTaskId = taskInfo.taskId;
79
+ worker.worker.postMessage({ task: taskInfo.task, taskId: taskInfo.taskId });
80
+ }
81
+ _handleTaskCompletion(worker, message) {
82
+ if (worker.currentTaskId === undefined)
83
+ return; // Should not happen
84
+ const taskCallbacks = this.activeTasks.get(worker.currentTaskId);
85
+ if (taskCallbacks) {
86
+ if (message.status === 'success') {
87
+ taskCallbacks.resolve(message.result);
88
+ }
89
+ else {
90
+ taskCallbacks.reject(new EmbeddingError(message.error.message, message.error));
91
+ }
92
+ this.activeTasks.delete(worker.currentTaskId);
93
+ }
94
+ worker.currentTaskId = undefined;
95
+ worker.lastUsed = Date.now();
96
+ // After finishing, check if there are more tasks to run.
97
+ this._dispatch();
98
+ }
99
+ _handleWorkerCrash(crashedWorker, err) {
100
+ logger.error(`Worker #${crashedWorker.id} crashed.`, err);
101
+ // Reject the task it was working on
102
+ if (crashedWorker.currentTaskId !== undefined) {
103
+ const taskCallbacks = this.activeTasks.get(crashedWorker.currentTaskId);
104
+ taskCallbacks?.reject(new EmbeddingError(`Worker #${crashedWorker.id} crashed while processing task.`, err));
105
+ this.activeTasks.delete(crashedWorker.currentTaskId);
106
+ }
107
+ // Remove the dead worker from the pool
108
+ const workerIndex = this.workers.findIndex((w) => w.id === crashedWorker.id);
109
+ if (workerIndex !== -1) {
110
+ this.workers.splice(workerIndex, 1);
111
+ }
112
+ // Immediately try to dispatch another task, which may create a new worker if needed
113
+ this._dispatch();
114
+ }
115
+ _reapIdleWorkers() {
116
+ const now = Date.now();
117
+ const idleWorkers = this.workers.filter((w) => w.currentTaskId === undefined);
118
+ for (const worker of idleWorkers) {
119
+ if (this.workers.length <= this.config.minWorkers) {
120
+ break; // Stop reaping if we're at the minimum
121
+ }
122
+ if (now - worker.lastUsed > this.config.idleTimeoutMs) {
123
+ logger.info(`Scaling down: Terminating idle worker #${worker.id}.`);
124
+ worker.worker.terminate();
125
+ const workerIndex = this.workers.findIndex((w) => w.id === worker.id);
126
+ if (workerIndex !== -1) {
127
+ this.workers.splice(workerIndex, 1);
128
+ }
129
+ }
130
+ }
131
+ }
132
+ async destroy() {
133
+ if (this.idleCheckInterval) {
134
+ clearInterval(this.idleCheckInterval);
135
+ }
136
+ await Promise.all(this.workers.map((w) => w.worker.terminate()));
137
+ this.workers = [];
138
+ }
139
+ }
140
+ //# sourceMappingURL=pool.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pool.js","sourceRoot":"","sources":["../../../src/local-rag/utils/pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAE5C,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAA;AAE7C,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAkC7C,MAAM,OAAO,UAAU;IACJ,UAAU,CAAK;IACf,UAAU,CAAQ;IAClB,MAAM,CAAsB;IAErC,OAAO,GAAoB,EAAE,CAAA;IACpB,SAAS,GAAe,EAAE,CAAA;IACnC,aAAa,GAAG,CAAC,CAAA;IACjB,eAAe,GAAG,CAAC,CAAA;IACV,WAAW,GAAG,IAAI,GAAG,EAAgD,CAAA;IAC9E,iBAAiB,CAA4B;IAErD,YAAY,UAAe,EAAE,UAAkB,EAAE,SAAqB,EAAE;QACtE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAC5B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAE5B,IAAI,CAAC,MAAM,GAAG;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/D,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC;YAClC,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,SAAS;SACjD,CAAA;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC,CAAA;QACpG,CAAC;IACH,CAAC;IAIM,OAAO,CAAC,IAAmB;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;QACnC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;YACjD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;YACtD,IAAI,CAAC,SAAS,EAAE,CAAA;QAClB,CAAC,CAAC,CAAA;IACJ,CAAC;IAEO,SAAS;QACf,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAM;QACR,CAAC;QAED,8CAA8C;QAC9C,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,SAAS,CAAC,CAAA;QAC/E,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;YACvC,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,CAAC,WAAW,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAA;YAC7C,CAAC;YACD,OAAM;QACR,CAAC;QAED,uEAAuE;QACvE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;YACvC,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,oBAAoB,CAAC,CAAA;gBACjF,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;gBACtC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBAC5B,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;YACvC,CAAC;QACH,CAAC;QAED,mFAAmF;QACnF,gEAAgE;IAClE,CAAC;IAEO,aAAa;QACnB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAA;QAC3E,MAAM,aAAa,GAAkB;YACnC,EAAE,EAAE,EAAE,IAAI,CAAC,eAAe;YAC1B,MAAM;YACN,aAAa,EAAE,SAAS;YACxB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;SACrB,CAAA;QAED,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAwB,EAAE,EAAE;YAChD,IAAI,CAAC,qBAAqB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAA;QACpD,CAAC,CAAC,CAAA;QAEF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,IAAI,CAAC,kBAAkB,CAAC,aAAa,EAAE,GAAG,CAAC,CAAA;QAC7C,CAAC,CAAC,CAAA;QAEF,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,WAAW,CAAC,MAAqB,EAAE,QAAkB;QAC3D,MAAM,CAAC,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAA;QACtC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;IAC7E,CAAC;IAEO,qBAAqB,CAAC,MAAqB,EAAE,OAAwB;QAC3E,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS;YAAE,OAAM,CAAC,oBAAoB;QAEnE,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAChE,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACjC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACvC,CAAC;iBAAM,CAAC;gBACN,aAAa,CAAC,MAAM,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;YAChF,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC/C,CAAC;QAED,MAAM,CAAC,aAAa,GAAG,SAAS,CAAA;QAChC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE5B,yDAAyD;QACzD,IAAI,CAAC,SAAS,EAAE,CAAA;IAClB,CAAC;IAEO,kBAAkB,CAAC,aAA4B,EAAE,GAAU;QACjE,MAAM,CAAC,KAAK,CAAC,WAAW,aAAa,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,CAAA;QACzD,oCAAoC;QACpC,IAAI,aAAa,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,aAAa,CAAC,CAAA;YACvE,aAAa,EAAE,MAAM,CAAC,IAAI,cAAc,CAAC,WAAW,aAAa,CAAC,EAAE,iCAAiC,EAAE,GAAG,CAAC,CAAC,CAAA;YAC5G,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,CAAA;QACtD,CAAC;QAED,uCAAuC;QACvC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,aAAa,CAAC,EAAE,CAAC,CAAA;QAC5E,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;QACrC,CAAC;QAED,oFAAoF;QACpF,IAAI,CAAC,SAAS,EAAE,CAAA;IAClB,CAAC;IAEO,gBAAgB;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,SAAS,CAAC,CAAA;QAE7E,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;gBAClD,MAAK,CAAC,uCAAuC;YAC/C,CAAC;YAED,IAAI,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC,0CAA0C,MAAM,CAAC,EAAE,GAAG,CAAC,CAAA;gBACnE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAA;gBACzB,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,CAAC,CAAA;gBACrE,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;oBACvB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,OAAO;QAClB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACvC,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;QAChE,IAAI,CAAC,OAAO,GAAG,EAAE,CAAA;IACnB,CAAC;CACF"}