@ngao/search 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. package/.claude/settings.local.json +10 -0
  2. package/.env.example +7 -0
  3. package/.eslintrc.json +20 -0
  4. package/.github/workflows/build.yml +39 -0
  5. package/.github/workflows/release.yml +34 -0
  6. package/.github/workflows/test.yml +35 -0
  7. package/.mcp-config.json +14 -0
  8. package/.prettierrc.json +10 -0
  9. package/LICENSE +17 -0
  10. package/Makefile +26 -0
  11. package/README.md +57 -172
  12. package/config.example.json +8 -0
  13. package/dist/backend/api/search-engine.d.ts +40 -0
  14. package/dist/backend/api/search-engine.d.ts.map +1 -0
  15. package/dist/backend/api/search-engine.js +227 -0
  16. package/dist/backend/api/search-engine.js.map +1 -0
  17. package/dist/backend/core/block-impl.d.ts +32 -0
  18. package/dist/backend/core/block-impl.d.ts.map +1 -0
  19. package/dist/backend/core/block-impl.js +33 -0
  20. package/dist/backend/core/block-impl.js.map +1 -0
  21. package/dist/backend/core/config-loader.d.ts +68 -0
  22. package/dist/backend/core/config-loader.d.ts.map +1 -0
  23. package/dist/backend/core/config-loader.js +234 -0
  24. package/dist/backend/core/config-loader.js.map +1 -0
  25. package/dist/backend/core/constants.d.ts +39 -0
  26. package/dist/backend/core/constants.d.ts.map +1 -0
  27. package/dist/backend/core/constants.js +57 -0
  28. package/dist/backend/core/constants.js.map +1 -0
  29. package/dist/backend/core/enums.d.ts +54 -0
  30. package/dist/backend/core/enums.d.ts.map +1 -0
  31. package/dist/backend/core/enums.js +61 -0
  32. package/dist/backend/core/enums.js.map +1 -0
  33. package/dist/backend/core/errors.d.ts +83 -0
  34. package/dist/backend/core/errors.d.ts.map +1 -0
  35. package/dist/backend/core/errors.js +151 -0
  36. package/dist/backend/core/errors.js.map +1 -0
  37. package/dist/backend/core/logger.d.ts +68 -0
  38. package/dist/backend/core/logger.d.ts.map +1 -0
  39. package/dist/backend/core/logger.js +151 -0
  40. package/dist/backend/core/logger.js.map +1 -0
  41. package/dist/backend/core/models.d.ts +332 -0
  42. package/dist/backend/core/models.d.ts.map +1 -0
  43. package/dist/backend/core/models.js +6 -0
  44. package/dist/backend/core/models.js.map +1 -0
  45. package/dist/backend/core/service-types.d.ts +184 -0
  46. package/dist/backend/core/service-types.d.ts.map +1 -0
  47. package/dist/backend/core/service-types.js +7 -0
  48. package/dist/backend/core/service-types.js.map +1 -0
  49. package/dist/backend/core/types.d.ts +219 -0
  50. package/dist/backend/core/types.d.ts.map +1 -0
  51. package/dist/backend/core/types.js +109 -0
  52. package/dist/backend/core/types.js.map +1 -0
  53. package/dist/backend/index.d.ts +5 -0
  54. package/dist/backend/index.d.ts.map +1 -0
  55. package/dist/backend/index.js +13 -0
  56. package/dist/backend/index.js.map +1 -0
  57. package/dist/backend/indexing/block-extractor.d.ts +22 -0
  58. package/dist/backend/indexing/block-extractor.d.ts.map +1 -0
  59. package/dist/backend/indexing/block-extractor.js +52 -0
  60. package/dist/backend/indexing/block-extractor.js.map +1 -0
  61. package/dist/backend/indexing/index-builder.d.ts +26 -0
  62. package/dist/backend/indexing/index-builder.d.ts.map +1 -0
  63. package/dist/backend/indexing/index-builder.js +71 -0
  64. package/dist/backend/indexing/index-builder.js.map +1 -0
  65. package/dist/backend/parsers/base-file-parser.d.ts +134 -0
  66. package/dist/backend/parsers/base-file-parser.d.ts.map +1 -0
  67. package/dist/backend/parsers/base-file-parser.js +149 -0
  68. package/dist/backend/parsers/base-file-parser.js.map +1 -0
  69. package/dist/backend/parsers/javascript-parser.d.ts +36 -0
  70. package/dist/backend/parsers/javascript-parser.d.ts.map +1 -0
  71. package/dist/backend/parsers/javascript-parser.js +194 -0
  72. package/dist/backend/parsers/javascript-parser.js.map +1 -0
  73. package/dist/backend/parsers/json-parser.d.ts +15 -0
  74. package/dist/backend/parsers/json-parser.d.ts.map +1 -0
  75. package/dist/backend/parsers/json-parser.js +75 -0
  76. package/dist/backend/parsers/json-parser.js.map +1 -0
  77. package/dist/backend/parsers/markdown-parser.d.ts +17 -0
  78. package/dist/backend/parsers/markdown-parser.d.ts.map +1 -0
  79. package/dist/backend/parsers/markdown-parser.js +94 -0
  80. package/dist/backend/parsers/markdown-parser.js.map +1 -0
  81. package/dist/backend/parsers/parser-factory.d.ts +43 -0
  82. package/dist/backend/parsers/parser-factory.d.ts.map +1 -0
  83. package/dist/backend/parsers/parser-factory.js +149 -0
  84. package/dist/backend/parsers/parser-factory.js.map +1 -0
  85. package/dist/backend/parsers/python-parser.d.ts +21 -0
  86. package/dist/backend/parsers/python-parser.d.ts.map +1 -0
  87. package/dist/backend/parsers/python-parser.js +185 -0
  88. package/dist/backend/parsers/python-parser.js.map +1 -0
  89. package/dist/backend/parsers/yaml-parser.d.ts +16 -0
  90. package/dist/backend/parsers/yaml-parser.d.ts.map +1 -0
  91. package/dist/backend/parsers/yaml-parser.js +81 -0
  92. package/dist/backend/parsers/yaml-parser.js.map +1 -0
  93. package/dist/backend/repositories/implementations/lancedb-block-repository.d.ts +125 -0
  94. package/dist/backend/repositories/implementations/lancedb-block-repository.d.ts.map +1 -0
  95. package/dist/backend/repositories/implementations/lancedb-block-repository.js +505 -0
  96. package/dist/backend/repositories/implementations/lancedb-block-repository.js.map +1 -0
  97. package/dist/backend/repositories/implementations/lancedb-metadata-repository.d.ts +107 -0
  98. package/dist/backend/repositories/implementations/lancedb-metadata-repository.d.ts.map +1 -0
  99. package/dist/backend/repositories/implementations/lancedb-metadata-repository.js +275 -0
  100. package/dist/backend/repositories/implementations/lancedb-metadata-repository.js.map +1 -0
  101. package/dist/backend/repositories/implementations/memory-cache.d.ts +18 -0
  102. package/dist/backend/repositories/implementations/memory-cache.d.ts.map +1 -0
  103. package/dist/backend/repositories/implementations/memory-cache.js +53 -0
  104. package/dist/backend/repositories/implementations/memory-cache.js.map +1 -0
  105. package/dist/backend/repositories/repository.interface.d.ts +334 -0
  106. package/dist/backend/repositories/repository.interface.d.ts.map +1 -0
  107. package/dist/backend/repositories/repository.interface.js +7 -0
  108. package/dist/backend/repositories/repository.interface.js.map +1 -0
  109. package/dist/backend/search/context-extractor.d.ts +29 -0
  110. package/dist/backend/search/context-extractor.d.ts.map +1 -0
  111. package/dist/backend/search/context-extractor.js +106 -0
  112. package/dist/backend/search/context-extractor.js.map +1 -0
  113. package/dist/backend/search/multi-index-searcher.d.ts +28 -0
  114. package/dist/backend/search/multi-index-searcher.d.ts.map +1 -0
  115. package/dist/backend/search/multi-index-searcher.js +81 -0
  116. package/dist/backend/search/multi-index-searcher.js.map +1 -0
  117. package/dist/backend/search/query-parser.d.ts +37 -0
  118. package/dist/backend/search/query-parser.d.ts.map +1 -0
  119. package/dist/backend/search/query-parser.js +145 -0
  120. package/dist/backend/search/query-parser.js.map +1 -0
  121. package/dist/backend/search/ranking-engine.d.ts +31 -0
  122. package/dist/backend/search/ranking-engine.d.ts.map +1 -0
  123. package/dist/backend/search/ranking-engine.js +165 -0
  124. package/dist/backend/search/ranking-engine.js.map +1 -0
  125. package/dist/backend/search/result-formatter.d.ts +29 -0
  126. package/dist/backend/search/result-formatter.d.ts.map +1 -0
  127. package/dist/backend/search/result-formatter.js +70 -0
  128. package/dist/backend/search/result-formatter.js.map +1 -0
  129. package/dist/backend/service-types.d.ts +184 -0
  130. package/dist/backend/service-types.d.ts.map +1 -0
  131. package/dist/backend/service-types.js +7 -0
  132. package/dist/backend/service-types.js.map +1 -0
  133. package/dist/backend/services/embedding-service.d.ts +75 -0
  134. package/dist/backend/services/embedding-service.d.ts.map +1 -0
  135. package/dist/backend/services/embedding-service.js +298 -0
  136. package/dist/backend/services/embedding-service.js.map +1 -0
  137. package/dist/backend/services/file-watcher.d.ts +17 -0
  138. package/dist/backend/services/file-watcher.d.ts.map +1 -0
  139. package/dist/backend/services/file-watcher.js +92 -0
  140. package/dist/backend/services/file-watcher.js.map +1 -0
  141. package/dist/backend/services/index-information-service.d.ts +114 -0
  142. package/dist/backend/services/index-information-service.d.ts.map +1 -0
  143. package/dist/backend/services/index-information-service.js +104 -0
  144. package/dist/backend/services/index-information-service.js.map +1 -0
  145. package/dist/backend/services/ngao-search-service.d.ts +107 -0
  146. package/dist/backend/services/ngao-search-service.d.ts.map +1 -0
  147. package/dist/backend/services/ngao-search-service.js +384 -0
  148. package/dist/backend/services/ngao-search-service.js.map +1 -0
  149. package/dist/backend/services/quantization-service.d.ts +53 -0
  150. package/dist/backend/services/quantization-service.d.ts.map +1 -0
  151. package/dist/backend/services/quantization-service.js +84 -0
  152. package/dist/backend/services/quantization-service.js.map +1 -0
  153. package/dist/backend/services/reindex-manager.d.ts +25 -0
  154. package/dist/backend/services/reindex-manager.d.ts.map +1 -0
  155. package/dist/backend/services/reindex-manager.js +78 -0
  156. package/dist/backend/services/reindex-manager.js.map +1 -0
  157. package/dist/backend/services/session-manager.d.ts +115 -0
  158. package/dist/backend/services/session-manager.d.ts.map +1 -0
  159. package/dist/backend/services/session-manager.js +150 -0
  160. package/dist/backend/services/session-manager.js.map +1 -0
  161. package/dist/backend/services/vector-search-service.d.ts +81 -0
  162. package/dist/backend/services/vector-search-service.d.ts.map +1 -0
  163. package/dist/backend/services/vector-search-service.js +143 -0
  164. package/dist/backend/services/vector-search-service.js.map +1 -0
  165. package/dist/backend/utils/file-utils.d.ts +92 -0
  166. package/dist/backend/utils/file-utils.d.ts.map +1 -0
  167. package/dist/backend/utils/file-utils.js +247 -0
  168. package/dist/backend/utils/file-utils.js.map +1 -0
  169. package/dist/cli/setup.d.ts +4 -0
  170. package/dist/cli/setup.d.ts.map +1 -0
  171. package/dist/cli/setup.js +138 -0
  172. package/dist/cli/setup.js.map +1 -0
  173. package/dist/index.d.ts +6 -0
  174. package/dist/index.d.ts.map +1 -0
  175. package/dist/index.js +22 -0
  176. package/dist/index.js.map +1 -0
  177. package/dist/main.d.ts +14 -0
  178. package/dist/main.d.ts.map +1 -0
  179. package/dist/main.js +7 -67075
  180. package/dist/main.js.map +1 -0
  181. package/dist/mcp/tool-schemas.d.ts +205 -0
  182. package/dist/mcp/tool-schemas.d.ts.map +1 -0
  183. package/dist/mcp/tool-schemas.js +391 -0
  184. package/dist/mcp/tool-schemas.js.map +1 -0
  185. package/dist/server/logger.d.ts +50 -0
  186. package/dist/server/logger.d.ts.map +1 -0
  187. package/dist/server/logger.js +77 -0
  188. package/dist/server/logger.js.map +1 -0
  189. package/dist/server/tool-registry.d.ts +64 -0
  190. package/dist/server/tool-registry.d.ts.map +1 -0
  191. package/dist/server/tool-registry.js +93 -0
  192. package/dist/server/tool-registry.js.map +1 -0
  193. package/dist/server/transports/mcp-transport.d.ts +31 -0
  194. package/dist/server/transports/mcp-transport.d.ts.map +1 -0
  195. package/dist/server/transports/mcp-transport.js +331 -0
  196. package/dist/server/transports/mcp-transport.js.map +1 -0
  197. package/dist/server/transports/rest-transport.d.ts +36 -0
  198. package/dist/server/transports/rest-transport.d.ts.map +1 -0
  199. package/dist/server/transports/rest-transport.js +250 -0
  200. package/dist/server/transports/rest-transport.js.map +1 -0
  201. package/docs/API.md +116 -0
  202. package/docs/ARCHITECTURE.md +101 -0
  203. package/docs/FILE_WATCHING.md +120 -0
  204. package/docs/INSTALLATION.md +87 -0
  205. package/docs/MCP_INTEGRATION.md +108 -0
  206. package/docs/README.md +288 -0
  207. package/docs/USAGE.md +123 -0
  208. package/docs/architecture-design-standards/01_ARCHITECTURE.md +863 -0
  209. package/docs/architecture-design-standards/02_SEARCH_ENGINE_DESIGN.md +958 -0
  210. package/docs/architecture-design-standards/03_DATAFLOW.md +1000 -0
  211. package/docs/architecture-design-standards/04_VISUAL_GUIDE.md +922 -0
  212. package/docs/architecture-design-standards/05_REPOSITORY_PATTERN_GUIDE.md +503 -0
  213. package/docs/architecture-design-standards/06_IMPLEMENTATION_PATTERNS.md +1026 -0
  214. package/docs/architecture-design-standards/07_TYPESCRIPT_GUIDE.md +1027 -0
  215. package/docs/architecture-design-standards/08_CODING_STANDARDS.md +1274 -0
  216. package/docs/reference/01_START_HERE.md +108 -0
  217. package/docs/reference/02_QUICK_REFERENCE.md +363 -0
  218. package/docs/reference/03_DOCUMENTATION_INDEX.md +293 -0
  219. package/docs/reference/04_DELIVERY_SUMMARY.md +463 -0
  220. package/docs/reference/05_IMPLEMENTATION_OVERVIEW.md +319 -0
  221. package/docs/reference/06_RESEARCH_SUMMARY.md +519 -0
  222. package/docs/tracking/03_IMPLEMENTATION_ROADMAP.md +788 -0
  223. package/jest.config.json +12 -0
  224. package/package.json +46 -53
  225. package/prepend-shebang.js +18 -0
  226. package/scripts/setup-mcp.sh +66 -0
  227. package/src/backend/index.ts +5 -0
  228. package/src/backend/service-types.ts +219 -0
  229. package/src/backend/services/file-watcher.ts +79 -0
  230. package/src/backend/services/ngao-search-service.ts +430 -0
  231. package/src/backend/services/reindex-manager.ts +90 -0
  232. package/src/backend/services/session-manager.ts +214 -0
  233. package/src/cli/setup.ts +122 -0
  234. package/src/index.ts +6 -0
  235. package/src/main.ts +225 -0
  236. package/src/mcp/tool-schemas.ts +439 -0
  237. package/src/server/logger.ts +88 -0
  238. package/src/server/tool-registry.ts +117 -0
  239. package/src/server/transports/mcp-transport.ts +374 -0
  240. package/src/server/transports/rest-transport.ts +258 -0
  241. package/tests/unit/agent-tools.test.ts +454 -0
  242. package/tests/unit/file-watcher.test.d.ts +2 -0
  243. package/tests/unit/file-watcher.test.d.ts.map +1 -0
  244. package/tests/unit/file-watcher.test.js +9 -0
  245. package/tests/unit/file-watcher.test.js.map +1 -0
  246. package/tests/unit/file-watcher.test.ts +7 -0
  247. package/tests/unit/search-integration.test.ts +256 -0
  248. package/tests/unit/services.test.d.ts +2 -0
  249. package/tests/unit/services.test.d.ts.map +1 -0
  250. package/tests/unit/services.test.js +9 -0
  251. package/tests/unit/services.test.js.map +1 -0
  252. package/tests/unit/services.test.ts +7 -0
  253. package/tsconfig.json +23 -0
  254. package/webpack.backend.config.js +60 -0
  255. package/webpack.config.js +34 -0
  256. package/models/Xenova/all-MiniLM-L6-v2/config.json +0 -25
  257. package/models/Xenova/all-MiniLM-L6-v2/onnx/model_quantized.onnx +0 -0
  258. package/models/Xenova/all-MiniLM-L6-v2/tokenizer.json +0 -30686
  259. package/models/Xenova/all-MiniLM-L6-v2/tokenizer_config.json +0 -15
@@ -0,0 +1 @@
1
+ {"version":3,"file":"service-types.js","sourceRoot":"","sources":["../../src/backend/service-types.ts"],"names":[],"mappings":";AAAA;;;GAGG"}
@@ -0,0 +1,75 @@
1
+ /**
2
+ * Embedding Service
3
+ * Generates semantic embeddings for code blocks using local transformers
4
+ *
5
+ * Note: Uses 'any' type for transformers.js API due to lack of complete TypeScript definitions
6
+ */
7
+ import { ILogger } from '../core/types';
8
+ /**
9
+ * Service for generating semantic embeddings using local transformers
10
+ * Uses: Xenova/all-MiniLM-L6-v2 (384-dimensional, lightweight ~80MB)
11
+ * Runtime: ~10-20ms per embedding on modern hardware
12
+ * Lightweight and efficient model for semantic search
13
+ */
14
+ export declare class EmbeddingService {
15
+ private _model;
16
+ private _tokenizer;
17
+ private _isInitialized;
18
+ private _logger;
19
+ private _modelName;
20
+ private _maxLength;
21
+ constructor(logger?: ILogger);
22
+ /**
23
+ * Initialize embedding model (loads transformer model)
24
+ * First checks for bundled model cache, then falls back to HuggingFace download
25
+ * Optimized for fast initialization with quantized ONNX models
26
+ *
27
+ * @throws Error if model loading fails
28
+ */
29
+ initialize(): Promise<void>;
30
+ /**
31
+ * Setup cache directory for transformers.js library
32
+ * Prioritizes bundled models in the package, falls back to downloaded cache
33
+ * Checks multiple possible locations for bundled models
34
+ */
35
+ private _setupCacheDirectory;
36
+ /**
37
+ * Generate embedding for a single text
38
+ *
39
+ * @param text - Text to embed
40
+ * @returns 384-dimensional float32 array
41
+ * @throws Error if model not initialized or embedding fails
42
+ */
43
+ generate(text: string): Promise<number[]>;
44
+ /**
45
+ * Generate embeddings for multiple texts (batch processing)
46
+ *
47
+ * @param texts - Array of texts to embed
48
+ * @param batchSize - Process this many at once (default: 32)
49
+ * @returns Array of 384-dimensional embeddings
50
+ */
51
+ generateBatch(texts: string[], batchSize?: number): Promise<number[][]>;
52
+ /**
53
+ * Check if model is initialized
54
+ */
55
+ isInitialized(): boolean;
56
+ /**
57
+ * Get embedding dimension (384 for all-MiniLM-L6-v2)
58
+ */
59
+ getDimension(): number;
60
+ /**
61
+ * Get model name
62
+ */
63
+ getModelName(): string;
64
+ /**
65
+ * Mean pooling: average token embeddings to get sentence embedding
66
+ * Private helper method
67
+ */
68
+ private _meanPooling;
69
+ /**
70
+ * L2 normalization: normalize vector to unit length
71
+ * Private helper method
72
+ */
73
+ private _normalize;
74
+ }
75
+ //# sourceMappingURL=embedding-service.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embedding-service.d.ts","sourceRoot":"","sources":["../../../src/backend/services/embedding-service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAMxC;;;;;GAKG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,UAAU,CAAe;gBAEd,MAAM,CAAC,EAAE,OAAO;IAInC;;;;;;OAMG;IACU,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA0CxC;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAiE5B;;;;;;OAMG;IACU,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA2CtD;;;;;;OAMG;IACU,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAW,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IA2BxF;;OAEG;IACI,aAAa,IAAI,OAAO;IAI/B;;OAEG;IACI,YAAY,IAAI,MAAM;IAI7B;;OAEG;IACI,YAAY,IAAI,MAAM;IAI7B;;;OAGG;IACH,OAAO,CAAC,YAAY;IA4CpB;;;OAGG;IACH,OAAO,CAAC,UAAU;CAiBnB"}
@@ -0,0 +1,298 @@
1
+ "use strict";
2
+ /**
3
+ * Embedding Service
4
+ * Generates semantic embeddings for code blocks using local transformers
5
+ *
6
+ * Note: Uses 'any' type for transformers.js API due to lack of complete TypeScript definitions
7
+ */
8
+ /* eslint-disable @typescript-eslint/no-explicit-any */
9
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ var desc = Object.getOwnPropertyDescriptor(m, k);
12
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
13
+ desc = { enumerable: true, get: function() { return m[k]; } };
14
+ }
15
+ Object.defineProperty(o, k2, desc);
16
+ }) : (function(o, m, k, k2) {
17
+ if (k2 === undefined) k2 = k;
18
+ o[k2] = m[k];
19
+ }));
20
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
21
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
22
+ }) : function(o, v) {
23
+ o["default"] = v;
24
+ });
25
+ var __importStar = (this && this.__importStar) || function (mod) {
26
+ if (mod && mod.__esModule) return mod;
27
+ var result = {};
28
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
29
+ __setModuleDefault(result, mod);
30
+ return result;
31
+ };
32
+ Object.defineProperty(exports, "__esModule", { value: true });
33
+ exports.EmbeddingService = void 0;
34
+ const logger_1 = require("../core/logger");
35
+ const path = __importStar(require("path"));
36
+ const fs = __importStar(require("fs"));
37
+ const os = __importStar(require("os"));
38
+ /**
39
+ * Service for generating semantic embeddings using local transformers
40
+ * Uses: Xenova/all-MiniLM-L6-v2 (384-dimensional, lightweight ~80MB)
41
+ * Runtime: ~10-20ms per embedding on modern hardware
42
+ * Lightweight and efficient model for semantic search
43
+ */
44
+ class EmbeddingService {
45
+ constructor(logger) {
46
+ this._isInitialized = false;
47
+ this._modelName = 'Xenova/all-MiniLM-L6-v2';
48
+ this._maxLength = 512; // Max tokens for embedding
49
+ this._logger = logger || new logger_1.NoOpLogger();
50
+ }
51
+ /**
52
+ * Initialize embedding model (loads transformer model)
53
+ * First checks for bundled model cache, then falls back to HuggingFace download
54
+ * Optimized for fast initialization with quantized ONNX models
55
+ *
56
+ * @throws Error if model loading fails
57
+ */
58
+ async initialize() {
59
+ if (this._isInitialized) {
60
+ return;
61
+ }
62
+ try {
63
+ this._logger.debug('Initializing embedding model', { model: this._modelName });
64
+ // Set cache directory to use bundled models if available
65
+ this._setupCacheDirectory();
66
+ // Optimize transformers.js for faster inference
67
+ // Use WASM backend for better performance on CPU
68
+ process.env.TRANSFORMERS_CACHE_DIR = process.env.TRANSFORMERS_CACHE;
69
+ // Dynamically import transformers to avoid hard dependency
70
+ // Users must install @xenova/transformers separately
71
+ const transformersModule = await Promise.resolve().then(() => __importStar(require('@xenova/transformers')));
72
+ // Load pre-trained model and tokenizer
73
+ // Will use cached version if available, otherwise download from HuggingFace
74
+ // Using from_pretrained which automatically uses ONNX if available
75
+ const startTime = Date.now();
76
+ this._tokenizer = await transformersModule.AutoTokenizer.from_pretrained(this._modelName);
77
+ this._model = await transformersModule.AutoModel.from_pretrained(this._modelName, {
78
+ quantized: true, // Use quantized model if available (faster)
79
+ });
80
+ const initTime = Date.now() - startTime;
81
+ this._isInitialized = true;
82
+ this._logger.info('Embedding model initialized', {
83
+ model: this._modelName,
84
+ initTimeMs: initTime,
85
+ });
86
+ }
87
+ catch (error) {
88
+ const message = `Failed to initialize embedding model: ${String(error)}`;
89
+ this._logger.error(message, error);
90
+ throw new Error(message);
91
+ }
92
+ }
93
+ /**
94
+ * Setup cache directory for transformers.js library
95
+ * Prioritizes bundled models in the package, falls back to downloaded cache
96
+ * Checks multiple possible locations for bundled models
97
+ */
98
+ _setupCacheDirectory() {
99
+ const possibleBundledPaths = [];
100
+ // Find package root by walking up from current file location
101
+ let currentDir = __dirname;
102
+ for (let i = 0; i < 10; i++) {
103
+ const packageJsonPath = path.join(currentDir, 'package.json');
104
+ if (fs.existsSync(packageJsonPath)) {
105
+ const modelsPath = path.join(currentDir, 'models');
106
+ possibleBundledPaths.push(modelsPath);
107
+ break;
108
+ }
109
+ currentDir = path.dirname(currentDir);
110
+ }
111
+ // Also check common locations
112
+ possibleBundledPaths.push(
113
+ // Development: src/backend/services/../../../models (3 levels up)
114
+ path.join(__dirname, '..', '..', '..', 'models'),
115
+ // Distribution/npm package: dist/backend/services/../../../../models (4 levels up)
116
+ path.join(__dirname, '..', '..', '..', '..', 'models'),
117
+ // Bundled distribution from process entry point
118
+ path.join(path.dirname(process.argv[1] || ''), '..', 'models'),
119
+ // Project working directory
120
+ path.join(process.cwd(), 'models'));
121
+ // Check each possible bundled path
122
+ for (const bundledModelsDir of possibleBundledPaths) {
123
+ if (!bundledModelsDir || !fs.existsSync(bundledModelsDir)) {
124
+ continue;
125
+ }
126
+ const modelPath = path.join(bundledModelsDir, 'Xenova', 'all-MiniLM-L6-v2');
127
+ if (fs.existsSync(modelPath)) {
128
+ this._logger.info('Using bundled model cache', {
129
+ cacheDir: bundledModelsDir,
130
+ modelPath: modelPath
131
+ });
132
+ process.env.TRANSFORMERS_CACHE = bundledModelsDir;
133
+ process.env.HF_HOME = bundledModelsDir;
134
+ return;
135
+ }
136
+ }
137
+ // If no bundled models found, fall back to standard HuggingFace cache
138
+ this._logger.warn('Bundled models not found, will use downloaded cache');
139
+ const homeDir = process.env.HOME || process.env.USERPROFILE || os.homedir?.() || '.';
140
+ const cacheDir = path.join(homeDir, '.cache', 'huggingface', 'transformers');
141
+ // Create cache directory if it doesn't exist
142
+ try {
143
+ if (!fs.existsSync(cacheDir)) {
144
+ fs.mkdirSync(cacheDir, { recursive: true });
145
+ }
146
+ }
147
+ catch (error) {
148
+ this._logger.debug('Could not create cache directory', error);
149
+ }
150
+ // Set environment variables for transformers.js cache
151
+ process.env.TRANSFORMERS_CACHE = cacheDir;
152
+ process.env.HF_HOME = cacheDir;
153
+ this._logger.debug('Model cache directory', { path: cacheDir });
154
+ }
155
+ /**
156
+ * Generate embedding for a single text
157
+ *
158
+ * @param text - Text to embed
159
+ * @returns 384-dimensional float32 array
160
+ * @throws Error if model not initialized or embedding fails
161
+ */
162
+ async generate(text) {
163
+ if (!this._isInitialized) {
164
+ await this.initialize();
165
+ }
166
+ try {
167
+ // Validate input
168
+ if (!text || text.trim().length === 0) {
169
+ this._logger.warn('Empty text provided for embedding');
170
+ // Return zero vector for empty text
171
+ return new Array(384).fill(0);
172
+ }
173
+ // Truncate text if too long
174
+ const truncated = text.length > 5000 ? text.substring(0, 5000) : text;
175
+ // Tokenize
176
+ const tokens = this._tokenizer(truncated, {
177
+ padding: true,
178
+ truncation: true,
179
+ max_length: this._maxLength,
180
+ });
181
+ // Generate embeddings
182
+ const { last_hidden_state } = await this._model(tokens);
183
+ // Pool embeddings (mean pooling)
184
+ const embedding = this._meanPooling(last_hidden_state.data, last_hidden_state.shape, tokens.input_ids.data);
185
+ // Normalize (L2 normalization)
186
+ const normalized = this._normalize(embedding);
187
+ return normalized;
188
+ }
189
+ catch (error) {
190
+ this._logger.error('Error generating embedding', error);
191
+ throw new Error(`Failed to generate embedding: ${String(error)}`);
192
+ }
193
+ }
194
+ /**
195
+ * Generate embeddings for multiple texts (batch processing)
196
+ *
197
+ * @param texts - Array of texts to embed
198
+ * @param batchSize - Process this many at once (default: 32)
199
+ * @returns Array of 384-dimensional embeddings
200
+ */
201
+ async generateBatch(texts, batchSize = 32) {
202
+ if (!this._isInitialized) {
203
+ await this.initialize();
204
+ }
205
+ const embeddings = [];
206
+ this._logger.debug('Generating batch embeddings', {
207
+ textCount: texts.length,
208
+ batchSize,
209
+ });
210
+ // Process in batches to manage memory
211
+ for (let i = 0; i < texts.length; i += batchSize) {
212
+ const batch = texts.slice(i, Math.min(i + batchSize, texts.length));
213
+ const batchEmbeddings = await Promise.all(batch.map((text) => this.generate(text)));
214
+ embeddings.push(...batchEmbeddings);
215
+ this._logger.debug('Batch progress', {
216
+ processed: Math.min(i + batchSize, texts.length),
217
+ total: texts.length,
218
+ });
219
+ }
220
+ return embeddings;
221
+ }
222
+ /**
223
+ * Check if model is initialized
224
+ */
225
+ isInitialized() {
226
+ return this._isInitialized;
227
+ }
228
+ /**
229
+ * Get embedding dimension (384 for all-MiniLM-L6-v2)
230
+ */
231
+ getDimension() {
232
+ return 384;
233
+ }
234
+ /**
235
+ * Get model name
236
+ */
237
+ getModelName() {
238
+ return this._modelName;
239
+ }
240
+ /**
241
+ * Mean pooling: average token embeddings to get sentence embedding
242
+ * Private helper method
243
+ */
244
+ _meanPooling(embeddings, shape, inputIds) {
245
+ // Handle different shape formats
246
+ let shapeArray;
247
+ if (Array.isArray(shape)) {
248
+ shapeArray = shape;
249
+ }
250
+ else if (shape && shape.dims && Array.isArray(shape.dims)) {
251
+ shapeArray = shape.dims;
252
+ }
253
+ else {
254
+ // Fallback: assume shape is [1, seqLength, 384]
255
+ const seqLength = inputIds.length;
256
+ shapeArray = [1, seqLength, 384];
257
+ }
258
+ const [, seqLength, hiddenDim] = shapeArray;
259
+ const result = new Array(hiddenDim).fill(0);
260
+ let tokenCount = 0;
261
+ for (let i = 0; i < seqLength; i++) {
262
+ // Skip padding tokens ([PAD] = 0)
263
+ if (inputIds[i] === 0) {
264
+ continue;
265
+ }
266
+ for (let j = 0; j < hiddenDim; j++) {
267
+ result[j] += embeddings[i * hiddenDim + j];
268
+ }
269
+ tokenCount++;
270
+ }
271
+ // Average
272
+ if (tokenCount > 0) {
273
+ for (let j = 0; j < hiddenDim; j++) {
274
+ result[j] /= tokenCount;
275
+ }
276
+ }
277
+ return result;
278
+ }
279
+ /**
280
+ * L2 normalization: normalize vector to unit length
281
+ * Private helper method
282
+ */
283
+ _normalize(vector) {
284
+ let norm = 0;
285
+ // Calculate L2 norm
286
+ for (const val of vector) {
287
+ norm += val * val;
288
+ }
289
+ norm = Math.sqrt(norm);
290
+ // Normalize (if norm is 0, return as-is)
291
+ if (norm === 0) {
292
+ return vector;
293
+ }
294
+ return vector.map((val) => val / norm);
295
+ }
296
+ }
297
+ exports.EmbeddingService = EmbeddingService;
298
+ //# sourceMappingURL=embedding-service.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embedding-service.js","sourceRoot":"","sources":["../../../src/backend/services/embedding-service.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;AACH,uDAAuD;;;;;;;;;;;;;;;;;;;;;;;;;;AAGvD,2CAA4C;AAC5C,2CAA6B;AAC7B,uCAAyB;AACzB,uCAAyB;AAEzB;;;;;GAKG;AACH,MAAa,gBAAgB;IAQ3B,YAAmB,MAAgB;QAL3B,mBAAc,GAAY,KAAK,CAAC;QAEhC,eAAU,GAAW,yBAAyB,CAAC;QAC/C,eAAU,GAAW,GAAG,CAAC,CAAC,2BAA2B;QAG3D,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,IAAI,mBAAU,EAAE,CAAC;IAC5C,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,UAAU;QACrB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;YAE/E,yDAAyD;YACzD,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAE5B,gDAAgD;YAChD,iDAAiD;YACjD,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;YAEpE,2DAA2D;YAC3D,qDAAqD;YACrD,MAAM,kBAAkB,GAAG,wDAAa,sBAAsB,GAAC,CAAC;YAEhE,uCAAuC;YACvC,4EAA4E;YAC5E,mEAAmE;YACnE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE7B,IAAI,CAAC,UAAU,GAAG,MAAO,kBAA0B,CAAC,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACnG,IAAI,CAAC,MAAM,GAAG,MAAO,kBAA0B,CAAC,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,UAAU,EAAE;gBACzF,SAAS,EAAE,IAAI,EAAE,4CAA4C;aAC9D,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACxC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,6BAA6B,EAAE;gBAC/C,KAAK,EAAE,IAAI,CAAC,UAAU;gBACtB,UAAU,EAAE,QAAQ;aACrB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,yCAAyC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAc,CAAC,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,oBAAoB;QAC1B,MAAM,oBAAoB,GAAa,EAAE,CAAC;QAE1C,6DAA6D;QAC7D,IAAI,UAAU,GAAG,SAAS,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YAC9D,IAAI,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;gBACnC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;gBACnD,oBAAoB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACtC,MAAM;YACR,CAAC;YACD,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACxC,CAAC;QAED,8BAA8B;QAC9B,oBAAoB,CAAC,IAAI;QACvB,kEAAkE;QAClE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC;QAChD,mFAAmF;QACnF,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC;QACtD,gDAAgD;QAChD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC;QAC9D,4BAA4B;QAC5B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CACnC,CAAC;QAEF,mCAAmC;QACnC,KAAK,MAAM,gBAAgB,IAAI,oBAAoB,EAAE,CAAC;YACpD,IAAI,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC1D,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;YAC5E,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,2BAA2B,EAAE;oBAC7C,QAAQ,EAAE,gBAAgB;oBAC1B,SAAS,EAAE,SAAS;iBACrB,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,gBAAgB,CAAC;gBAClD,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,gBAAgB,CAAC;gBACvC,OAAO;YACT,CAAC;QACH,CAAC;QAED,sEAAsE;QACtE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,IAAI,GAAG,CAAC;QACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;QAE7E,6CAA6C;QAC7C,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC7B,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,KAAc,CAAC,CAAC;QACzE,CAAC;QAED,sDAAsD;QACtD,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,QAAQ,CAAC;QAC/B,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAClE,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,QAAQ,CAAC,IAAY;QAChC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC;YACH,iBAAiB;YACjB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;gBACvD,oCAAoC;gBACpC,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChC,CAAC;YAED,4BAA4B;YAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAEtE,WAAW;YACX,MAAM,MAAM,GAAI,IAAI,CAAC,UAAkB,CAAC,SAAS,EAAE;gBACjD,OAAO,EAAE,IAAI;gBACb,UAAU,EAAE,IAAI;gBAChB,UAAU,EAAE,IAAI,CAAC,UAAU;aAC5B,CAAC,CAAC;YAEH,sBAAsB;YACtB,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAO,IAAI,CAAC,MAAc,CAAC,MAAM,CAAC,CAAC;YAEjE,iCAAiC;YACjC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CACjC,iBAAiB,CAAC,IAAI,EACtB,iBAAiB,CAAC,KAAK,EACvB,MAAM,CAAC,SAAS,CAAC,IAAI,CACtB,CAAC;YAEF,+BAA+B;YAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAE9C,OAAO,UAAU,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAc,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,iCAAiC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,aAAa,CAAC,KAAe,EAAE,YAAoB,EAAE;QAChE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,UAAU,GAAe,EAAE,CAAC;QAElC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE;YAChD,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,SAAS;SACV,CAAC,CAAC;QAEH,sCAAsC;QACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YACpE,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpF,UAAU,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAEpC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE;gBACnC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC;gBAChD,KAAK,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACI,aAAa;QAClB,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;OAEG;IACI,YAAY;QACjB,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACI,YAAY;QACjB,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED;;;OAGG;IACK,YAAY,CAClB,UAAwB,EACxB,KAA2C,EAC3C,QAAqB;QAErB,iCAAiC;QACjC,IAAI,UAAoB,CAAC;QACzB,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,UAAU,GAAG,KAAK,CAAC;QACrB,CAAC;aAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5D,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,UAAU,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QACnC,CAAC;QAED,MAAM,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,GAAG,UAAU,CAAC;QAC5C,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,kCAAkC;YAClC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC;YAC7C,CAAC;YAED,UAAU,EAAE,CAAC;QACf,CAAC;QAED,UAAU;QACV,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACK,UAAU,CAAC,MAAgB;QACjC,IAAI,IAAI,GAAG,CAAC,CAAC;QAEb,oBAAoB;QACpB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,IAAI,IAAI,GAAG,GAAG,GAAG,CAAC;QACpB,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvB,yCAAyC;QACzC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YACf,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;IACzC,CAAC;CACF;AAjTD,4CAiTC"}
@@ -0,0 +1,17 @@
1
+ import { SearchEngine } from '@ngao/search-core';
2
+ export interface FileWatcherConfig {
3
+ ignored?: string | string[];
4
+ stabilityThreshold?: number;
5
+ pollInterval?: number;
6
+ }
7
+ export declare class FileWatcher {
8
+ private searchEngine;
9
+ private config?;
10
+ private watcher;
11
+ private isWatching;
12
+ constructor(searchEngine: SearchEngine, config?: FileWatcherConfig | undefined);
13
+ watch(dirPath: string): Promise<void>;
14
+ stop(): Promise<void>;
15
+ isActive(): boolean;
16
+ }
17
+ //# sourceMappingURL=file-watcher.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"file-watcher.d.ts","sourceRoot":"","sources":["../../../src/backend/services/file-watcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,MAAM,WAAW,iBAAiB;IAChC,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,qBAAa,WAAW;IAIV,OAAO,CAAC,YAAY;IAAgB,OAAO,CAAC,MAAM,CAAC;IAH/D,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,UAAU,CAAQ;gBAEN,YAAY,EAAE,YAAY,EAAU,MAAM,CAAC,+BAAmB;IAE5E,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmDrC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAS3B,QAAQ,IAAI,OAAO;CAGpB"}
@@ -0,0 +1,92 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.FileWatcher = void 0;
27
+ const chokidar = __importStar(require("chokidar"));
28
+ class FileWatcher {
29
+ constructor(searchEngine, config) {
30
+ this.searchEngine = searchEngine;
31
+ this.config = config;
32
+ this.watcher = null;
33
+ this.isWatching = false;
34
+ }
35
+ async watch(dirPath) {
36
+ if (this.isWatching) {
37
+ throw new Error('File watcher is already active');
38
+ }
39
+ this.watcher = chokidar.watch(dirPath, {
40
+ persistent: true,
41
+ ignored: this.config?.ignored || /(^|[\/\\])\.|node_modules/,
42
+ ignoreInitial: true,
43
+ awaitWriteFinish: {
44
+ stabilityThreshold: this.config?.stabilityThreshold || 100,
45
+ pollInterval: this.config?.pollInterval || 100,
46
+ },
47
+ });
48
+ this.watcher.on('change', (filePath) => {
49
+ console.log(`File changed: ${filePath}`);
50
+ // TODO: @ngao/search-core doesn't have reindexFile method
51
+ // For now, we'll need to reindex the directory
52
+ const path = require('path');
53
+ const dirPath = path.dirname(filePath);
54
+ this.searchEngine.indexDirectory(dirPath).catch((err) => {
55
+ console.error(`Failed to reindex ${filePath}:`, err);
56
+ });
57
+ });
58
+ this.watcher.on('add', (filePath) => {
59
+ console.log(`File added: ${filePath}`);
60
+ // TODO: @ngao/search-core doesn't have indexFile method
61
+ // For now, we'll need to index the directory
62
+ const path = require('path');
63
+ const dirPath = path.dirname(filePath);
64
+ this.searchEngine.indexDirectory(dirPath).catch((err) => {
65
+ console.error(`Failed to index ${filePath}:`, err);
66
+ });
67
+ });
68
+ this.watcher.on('unlink', async (filePath) => {
69
+ console.log(`File deleted: ${filePath}`);
70
+ // Note: Core library should handle file deletion
71
+ // For now, we just log it
72
+ });
73
+ this.watcher.on('error', (error) => {
74
+ console.error('File watcher error:', error);
75
+ });
76
+ this.isWatching = true;
77
+ console.log(`Watching directory: ${dirPath}`);
78
+ }
79
+ async stop() {
80
+ if (this.watcher) {
81
+ await this.watcher.close();
82
+ this.watcher = null;
83
+ this.isWatching = false;
84
+ console.log('File watcher stopped');
85
+ }
86
+ }
87
+ isActive() {
88
+ return this.isWatching;
89
+ }
90
+ }
91
+ exports.FileWatcher = FileWatcher;
92
+ //# sourceMappingURL=file-watcher.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"file-watcher.js","sourceRoot":"","sources":["../../../src/backend/services/file-watcher.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AACA,mDAAqC;AAQrC,MAAa,WAAW;IAItB,YAAoB,YAA0B,EAAU,MAA0B;QAA9D,iBAAY,GAAZ,YAAY,CAAc;QAAU,WAAM,GAAN,MAAM,CAAoB;QAH1E,YAAO,GAA8B,IAAI,CAAA;QACzC,eAAU,GAAG,KAAK,CAAA;IAE2D,CAAC;IAEtF,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAA;QACnD,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE;YACrC,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,2BAA2B;YAC5D,aAAa,EAAE,IAAI;YACnB,gBAAgB,EAAE;gBAChB,kBAAkB,EAAE,IAAI,CAAC,MAAM,EAAE,kBAAkB,IAAI,GAAG;gBAC1D,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,IAAI,GAAG;aAC/C;SACF,CAAC,CAAA;QAEF,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC7C,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAA;YACxC,0DAA0D;YAC1D,+CAA+C;YAC/C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;YAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,GAAU,EAAE,EAAE;gBAC7D,OAAO,CAAC,KAAK,CAAC,qBAAqB,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAA;YACtD,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC1C,OAAO,CAAC,GAAG,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAA;YACtC,wDAAwD;YACxD,6CAA6C;YAC7C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;YAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,GAAU,EAAE,EAAE;gBAC7D,OAAO,CAAC,KAAK,CAAC,mBAAmB,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAA;YACpD,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;YAC3C,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAA;YACxC,iDAAiD;YACjD,0BAA0B;QAC5B,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAA;QAC7C,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;QACtB,OAAO,CAAC,GAAG,CAAC,uBAAuB,OAAO,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;YAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;YACnB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;YACvB,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAA;QACrC,CAAC;IACH,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,UAAU,CAAA;IACxB,CAAC;CACF;AArED,kCAqEC"}
@@ -0,0 +1,114 @@
1
+ /**
2
+ * Index Information Service for NGAO Search
3
+ * Aggregates and formats detailed index statistics
4
+ */
5
+ import { IndexStats } from '../core/models';
6
+ import { IMetadataRepository } from '../repositories/repository.interface';
7
+ import { SearchEngine } from '../api/search-engine';
8
+ /**
9
+ * File type definition
10
+ */
11
+ export type FileType = '.ts' | '.js' | '.py' | '.json' | '.md' | '.yml' | '.yaml' | string;
12
+ /**
13
+ * Detailed information about indexed files
14
+ */
15
+ export interface FileStatistics {
16
+ /** File path */
17
+ filePath: string;
18
+ /** File type (python, markdown, etc.) */
19
+ fileType: FileType;
20
+ /** Number of blocks in file */
21
+ blockCount: number;
22
+ /** Breakdown of blocks by type */
23
+ blocksByType: Record<string, number>;
24
+ /** File size in bytes */
25
+ fileSizeBytes: number;
26
+ /** Unix timestamp when indexed */
27
+ indexedAt: number;
28
+ /** SHA256 hash for change detection */
29
+ hash: string;
30
+ /** Whether embeddings are present for this file */
31
+ embeddingsPresent: boolean;
32
+ }
33
+ /**
34
+ * File index metadata
35
+ */
36
+ export interface FileIndexMetadata {
37
+ /** File path */
38
+ filePath: string;
39
+ /** File type */
40
+ fileType: FileType;
41
+ /** File hash */
42
+ fileHash: string;
43
+ /** File size in bytes */
44
+ fileSizeBytes: number;
45
+ /** When indexed */
46
+ indexedAt: number;
47
+ /** Number of blocks */
48
+ blockCount: number;
49
+ /** Blocks by type */
50
+ blocksByType: Record<string, number>;
51
+ /** Whether embeddings are present */
52
+ embeddingPresent: boolean;
53
+ }
54
+ /**
55
+ * Detailed index information including breakdown
56
+ */
57
+ export interface DetailedIndexInfo {
58
+ /** Summary statistics */
59
+ summary: IndexStats;
60
+ /** Per-file breakdown */
61
+ fileBreakdown: FileStatistics[];
62
+ /** Index metadata */
63
+ indexMetadata: {
64
+ /** Index format version */
65
+ indexVersion: string;
66
+ /** When index was created */
67
+ createdAt: number;
68
+ /** Number of times reindexed */
69
+ totalReindexes: number;
70
+ /** Average reindex time in ms */
71
+ averageReindexTimeMs: number;
72
+ /** Indexing strategies used */
73
+ indexingStrategies: string[];
74
+ };
75
+ }
76
+ /**
77
+ * Service for aggregating and formatting index information
78
+ */
79
+ export declare class IndexInformationService {
80
+ private _metadataRepository;
81
+ private _searchEngine;
82
+ /**
83
+ * Constructor with dependency injection
84
+ * @param blockRepository - Block data access
85
+ * @param metadataRepository - Metadata data access
86
+ * @param searchEngine - Search engine for aggregate data
87
+ */
88
+ constructor(_metadataRepository: IMetadataRepository, _searchEngine: SearchEngine);
89
+ /**
90
+ * Get detailed index information
91
+ * @returns Detailed index info with breakdown
92
+ */
93
+ getDetailedIndexInfo(): Promise<DetailedIndexInfo>;
94
+ /**
95
+ * Get file statistics
96
+ * @param filePath - Optional specific file path
97
+ * @returns Array of file statistics
98
+ */
99
+ getFileStatistics(filePath?: string): Promise<FileStatistics[]>;
100
+ /**
101
+ * Get file breakdown by aggregating statistics
102
+ * @private
103
+ * @returns Array of file statistics
104
+ */
105
+ private _getFileBreakdown;
106
+ /**
107
+ * Format file metadata into file statistics
108
+ * @private
109
+ * @param metadata - File metadata
110
+ * @returns Formatted file statistics
111
+ */
112
+ private _formatFileStatistic;
113
+ }
114
+ //# sourceMappingURL=index-information-service.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-information-service.d.ts","sourceRoot":"","sources":["../../../src/backend/services/index-information-service.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,sCAAsC,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAE3F;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,QAAQ,EAAE,QAAQ,CAAC;IACnB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,yBAAyB;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,mDAAmD;IACnD,iBAAiB,EAAE,OAAO,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB;IAChB,QAAQ,EAAE,QAAQ,CAAC;IACnB,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,uBAAuB;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,qBAAqB;IACrB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,qCAAqC;IACrC,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,yBAAyB;IACzB,OAAO,EAAE,UAAU,CAAC;IACpB,yBAAyB;IACzB,aAAa,EAAE,cAAc,EAAE,CAAC;IAChC,qBAAqB;IACrB,aAAa,EAAE;QACb,2BAA2B;QAC3B,YAAY,EAAE,MAAM,CAAC;QACrB,6BAA6B;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,gCAAgC;QAChC,cAAc,EAAE,MAAM,CAAC;QACvB,iCAAiC;QACjC,oBAAoB,EAAE,MAAM,CAAC;QAC7B,+BAA+B;QAC/B,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH;AAED;;GAEG;AACH,qBAAa,uBAAuB;IAQhC,OAAO,CAAC,mBAAmB;IAC3B,OAAO,CAAC,aAAa;IARvB;;;;;OAKG;gBAEO,mBAAmB,EAAE,mBAAmB,EACxC,aAAa,EAAE,YAAY;IAGrC;;;OAGG;IACU,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,CAAC;IA8B/D;;;;OAIG;IACU,iBAAiB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAe5E;;;;OAIG;YACW,iBAAiB;IAU/B;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;CAY7B"}