@ngao/search 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +10 -0
- package/.env.example +7 -0
- package/.eslintrc.json +20 -0
- package/.github/workflows/build.yml +39 -0
- package/.github/workflows/release.yml +34 -0
- package/.github/workflows/test.yml +35 -0
- package/.mcp-config.json +14 -0
- package/.prettierrc.json +10 -0
- package/LICENSE +17 -0
- package/Makefile +26 -0
- package/README.md +57 -172
- package/config.example.json +8 -0
- package/dist/backend/api/search-engine.d.ts +40 -0
- package/dist/backend/api/search-engine.d.ts.map +1 -0
- package/dist/backend/api/search-engine.js +227 -0
- package/dist/backend/api/search-engine.js.map +1 -0
- package/dist/backend/core/block-impl.d.ts +32 -0
- package/dist/backend/core/block-impl.d.ts.map +1 -0
- package/dist/backend/core/block-impl.js +33 -0
- package/dist/backend/core/block-impl.js.map +1 -0
- package/dist/backend/core/config-loader.d.ts +68 -0
- package/dist/backend/core/config-loader.d.ts.map +1 -0
- package/dist/backend/core/config-loader.js +234 -0
- package/dist/backend/core/config-loader.js.map +1 -0
- package/dist/backend/core/constants.d.ts +39 -0
- package/dist/backend/core/constants.d.ts.map +1 -0
- package/dist/backend/core/constants.js +57 -0
- package/dist/backend/core/constants.js.map +1 -0
- package/dist/backend/core/enums.d.ts +54 -0
- package/dist/backend/core/enums.d.ts.map +1 -0
- package/dist/backend/core/enums.js +61 -0
- package/dist/backend/core/enums.js.map +1 -0
- package/dist/backend/core/errors.d.ts +83 -0
- package/dist/backend/core/errors.d.ts.map +1 -0
- package/dist/backend/core/errors.js +151 -0
- package/dist/backend/core/errors.js.map +1 -0
- package/dist/backend/core/logger.d.ts +68 -0
- package/dist/backend/core/logger.d.ts.map +1 -0
- package/dist/backend/core/logger.js +151 -0
- package/dist/backend/core/logger.js.map +1 -0
- package/dist/backend/core/models.d.ts +332 -0
- package/dist/backend/core/models.d.ts.map +1 -0
- package/dist/backend/core/models.js +6 -0
- package/dist/backend/core/models.js.map +1 -0
- package/dist/backend/core/service-types.d.ts +184 -0
- package/dist/backend/core/service-types.d.ts.map +1 -0
- package/dist/backend/core/service-types.js +7 -0
- package/dist/backend/core/service-types.js.map +1 -0
- package/dist/backend/core/types.d.ts +219 -0
- package/dist/backend/core/types.d.ts.map +1 -0
- package/dist/backend/core/types.js +109 -0
- package/dist/backend/core/types.js.map +1 -0
- package/dist/backend/index.d.ts +5 -0
- package/dist/backend/index.d.ts.map +1 -0
- package/dist/backend/index.js +13 -0
- package/dist/backend/index.js.map +1 -0
- package/dist/backend/indexing/block-extractor.d.ts +22 -0
- package/dist/backend/indexing/block-extractor.d.ts.map +1 -0
- package/dist/backend/indexing/block-extractor.js +52 -0
- package/dist/backend/indexing/block-extractor.js.map +1 -0
- package/dist/backend/indexing/index-builder.d.ts +26 -0
- package/dist/backend/indexing/index-builder.d.ts.map +1 -0
- package/dist/backend/indexing/index-builder.js +71 -0
- package/dist/backend/indexing/index-builder.js.map +1 -0
- package/dist/backend/parsers/base-file-parser.d.ts +134 -0
- package/dist/backend/parsers/base-file-parser.d.ts.map +1 -0
- package/dist/backend/parsers/base-file-parser.js +149 -0
- package/dist/backend/parsers/base-file-parser.js.map +1 -0
- package/dist/backend/parsers/javascript-parser.d.ts +36 -0
- package/dist/backend/parsers/javascript-parser.d.ts.map +1 -0
- package/dist/backend/parsers/javascript-parser.js +194 -0
- package/dist/backend/parsers/javascript-parser.js.map +1 -0
- package/dist/backend/parsers/json-parser.d.ts +15 -0
- package/dist/backend/parsers/json-parser.d.ts.map +1 -0
- package/dist/backend/parsers/json-parser.js +75 -0
- package/dist/backend/parsers/json-parser.js.map +1 -0
- package/dist/backend/parsers/markdown-parser.d.ts +17 -0
- package/dist/backend/parsers/markdown-parser.d.ts.map +1 -0
- package/dist/backend/parsers/markdown-parser.js +94 -0
- package/dist/backend/parsers/markdown-parser.js.map +1 -0
- package/dist/backend/parsers/parser-factory.d.ts +43 -0
- package/dist/backend/parsers/parser-factory.d.ts.map +1 -0
- package/dist/backend/parsers/parser-factory.js +149 -0
- package/dist/backend/parsers/parser-factory.js.map +1 -0
- package/dist/backend/parsers/python-parser.d.ts +21 -0
- package/dist/backend/parsers/python-parser.d.ts.map +1 -0
- package/dist/backend/parsers/python-parser.js +185 -0
- package/dist/backend/parsers/python-parser.js.map +1 -0
- package/dist/backend/parsers/yaml-parser.d.ts +16 -0
- package/dist/backend/parsers/yaml-parser.d.ts.map +1 -0
- package/dist/backend/parsers/yaml-parser.js +81 -0
- package/dist/backend/parsers/yaml-parser.js.map +1 -0
- package/dist/backend/repositories/implementations/lancedb-block-repository.d.ts +125 -0
- package/dist/backend/repositories/implementations/lancedb-block-repository.d.ts.map +1 -0
- package/dist/backend/repositories/implementations/lancedb-block-repository.js +505 -0
- package/dist/backend/repositories/implementations/lancedb-block-repository.js.map +1 -0
- package/dist/backend/repositories/implementations/lancedb-metadata-repository.d.ts +107 -0
- package/dist/backend/repositories/implementations/lancedb-metadata-repository.d.ts.map +1 -0
- package/dist/backend/repositories/implementations/lancedb-metadata-repository.js +275 -0
- package/dist/backend/repositories/implementations/lancedb-metadata-repository.js.map +1 -0
- package/dist/backend/repositories/implementations/memory-cache.d.ts +18 -0
- package/dist/backend/repositories/implementations/memory-cache.d.ts.map +1 -0
- package/dist/backend/repositories/implementations/memory-cache.js +53 -0
- package/dist/backend/repositories/implementations/memory-cache.js.map +1 -0
- package/dist/backend/repositories/repository.interface.d.ts +334 -0
- package/dist/backend/repositories/repository.interface.d.ts.map +1 -0
- package/dist/backend/repositories/repository.interface.js +7 -0
- package/dist/backend/repositories/repository.interface.js.map +1 -0
- package/dist/backend/search/context-extractor.d.ts +29 -0
- package/dist/backend/search/context-extractor.d.ts.map +1 -0
- package/dist/backend/search/context-extractor.js +106 -0
- package/dist/backend/search/context-extractor.js.map +1 -0
- package/dist/backend/search/multi-index-searcher.d.ts +28 -0
- package/dist/backend/search/multi-index-searcher.d.ts.map +1 -0
- package/dist/backend/search/multi-index-searcher.js +81 -0
- package/dist/backend/search/multi-index-searcher.js.map +1 -0
- package/dist/backend/search/query-parser.d.ts +37 -0
- package/dist/backend/search/query-parser.d.ts.map +1 -0
- package/dist/backend/search/query-parser.js +145 -0
- package/dist/backend/search/query-parser.js.map +1 -0
- package/dist/backend/search/ranking-engine.d.ts +31 -0
- package/dist/backend/search/ranking-engine.d.ts.map +1 -0
- package/dist/backend/search/ranking-engine.js +165 -0
- package/dist/backend/search/ranking-engine.js.map +1 -0
- package/dist/backend/search/result-formatter.d.ts +29 -0
- package/dist/backend/search/result-formatter.d.ts.map +1 -0
- package/dist/backend/search/result-formatter.js +70 -0
- package/dist/backend/search/result-formatter.js.map +1 -0
- package/dist/backend/service-types.d.ts +184 -0
- package/dist/backend/service-types.d.ts.map +1 -0
- package/dist/backend/service-types.js +7 -0
- package/dist/backend/service-types.js.map +1 -0
- package/dist/backend/services/embedding-service.d.ts +75 -0
- package/dist/backend/services/embedding-service.d.ts.map +1 -0
- package/dist/backend/services/embedding-service.js +298 -0
- package/dist/backend/services/embedding-service.js.map +1 -0
- package/dist/backend/services/file-watcher.d.ts +17 -0
- package/dist/backend/services/file-watcher.d.ts.map +1 -0
- package/dist/backend/services/file-watcher.js +92 -0
- package/dist/backend/services/file-watcher.js.map +1 -0
- package/dist/backend/services/index-information-service.d.ts +114 -0
- package/dist/backend/services/index-information-service.d.ts.map +1 -0
- package/dist/backend/services/index-information-service.js +104 -0
- package/dist/backend/services/index-information-service.js.map +1 -0
- package/dist/backend/services/ngao-search-service.d.ts +107 -0
- package/dist/backend/services/ngao-search-service.d.ts.map +1 -0
- package/dist/backend/services/ngao-search-service.js +384 -0
- package/dist/backend/services/ngao-search-service.js.map +1 -0
- package/dist/backend/services/quantization-service.d.ts +53 -0
- package/dist/backend/services/quantization-service.d.ts.map +1 -0
- package/dist/backend/services/quantization-service.js +84 -0
- package/dist/backend/services/quantization-service.js.map +1 -0
- package/dist/backend/services/reindex-manager.d.ts +25 -0
- package/dist/backend/services/reindex-manager.d.ts.map +1 -0
- package/dist/backend/services/reindex-manager.js +78 -0
- package/dist/backend/services/reindex-manager.js.map +1 -0
- package/dist/backend/services/session-manager.d.ts +115 -0
- package/dist/backend/services/session-manager.d.ts.map +1 -0
- package/dist/backend/services/session-manager.js +150 -0
- package/dist/backend/services/session-manager.js.map +1 -0
- package/dist/backend/services/vector-search-service.d.ts +81 -0
- package/dist/backend/services/vector-search-service.d.ts.map +1 -0
- package/dist/backend/services/vector-search-service.js +143 -0
- package/dist/backend/services/vector-search-service.js.map +1 -0
- package/dist/backend/utils/file-utils.d.ts +92 -0
- package/dist/backend/utils/file-utils.d.ts.map +1 -0
- package/dist/backend/utils/file-utils.js +247 -0
- package/dist/backend/utils/file-utils.js.map +1 -0
- package/dist/cli/setup.d.ts +4 -0
- package/dist/cli/setup.d.ts.map +1 -0
- package/dist/cli/setup.js +138 -0
- package/dist/cli/setup.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +22 -0
- package/dist/index.js.map +1 -0
- package/dist/main.d.ts +14 -0
- package/dist/main.d.ts.map +1 -0
- package/dist/main.js +7 -67075
- package/dist/main.js.map +1 -0
- package/dist/mcp/tool-schemas.d.ts +205 -0
- package/dist/mcp/tool-schemas.d.ts.map +1 -0
- package/dist/mcp/tool-schemas.js +391 -0
- package/dist/mcp/tool-schemas.js.map +1 -0
- package/dist/server/logger.d.ts +50 -0
- package/dist/server/logger.d.ts.map +1 -0
- package/dist/server/logger.js +77 -0
- package/dist/server/logger.js.map +1 -0
- package/dist/server/tool-registry.d.ts +64 -0
- package/dist/server/tool-registry.d.ts.map +1 -0
- package/dist/server/tool-registry.js +93 -0
- package/dist/server/tool-registry.js.map +1 -0
- package/dist/server/transports/mcp-transport.d.ts +31 -0
- package/dist/server/transports/mcp-transport.d.ts.map +1 -0
- package/dist/server/transports/mcp-transport.js +331 -0
- package/dist/server/transports/mcp-transport.js.map +1 -0
- package/dist/server/transports/rest-transport.d.ts +36 -0
- package/dist/server/transports/rest-transport.d.ts.map +1 -0
- package/dist/server/transports/rest-transport.js +250 -0
- package/dist/server/transports/rest-transport.js.map +1 -0
- package/docs/API.md +116 -0
- package/docs/ARCHITECTURE.md +101 -0
- package/docs/FILE_WATCHING.md +120 -0
- package/docs/INSTALLATION.md +87 -0
- package/docs/MCP_INTEGRATION.md +108 -0
- package/docs/README.md +288 -0
- package/docs/USAGE.md +123 -0
- package/docs/architecture-design-standards/01_ARCHITECTURE.md +863 -0
- package/docs/architecture-design-standards/02_SEARCH_ENGINE_DESIGN.md +958 -0
- package/docs/architecture-design-standards/03_DATAFLOW.md +1000 -0
- package/docs/architecture-design-standards/04_VISUAL_GUIDE.md +922 -0
- package/docs/architecture-design-standards/05_REPOSITORY_PATTERN_GUIDE.md +503 -0
- package/docs/architecture-design-standards/06_IMPLEMENTATION_PATTERNS.md +1026 -0
- package/docs/architecture-design-standards/07_TYPESCRIPT_GUIDE.md +1027 -0
- package/docs/architecture-design-standards/08_CODING_STANDARDS.md +1274 -0
- package/docs/reference/01_START_HERE.md +108 -0
- package/docs/reference/02_QUICK_REFERENCE.md +363 -0
- package/docs/reference/03_DOCUMENTATION_INDEX.md +293 -0
- package/docs/reference/04_DELIVERY_SUMMARY.md +463 -0
- package/docs/reference/05_IMPLEMENTATION_OVERVIEW.md +319 -0
- package/docs/reference/06_RESEARCH_SUMMARY.md +519 -0
- package/docs/tracking/03_IMPLEMENTATION_ROADMAP.md +788 -0
- package/jest.config.json +12 -0
- package/package.json +46 -53
- package/prepend-shebang.js +18 -0
- package/scripts/setup-mcp.sh +66 -0
- package/src/backend/index.ts +5 -0
- package/src/backend/service-types.ts +219 -0
- package/src/backend/services/file-watcher.ts +79 -0
- package/src/backend/services/ngao-search-service.ts +430 -0
- package/src/backend/services/reindex-manager.ts +90 -0
- package/src/backend/services/session-manager.ts +214 -0
- package/src/cli/setup.ts +122 -0
- package/src/index.ts +6 -0
- package/src/main.ts +225 -0
- package/src/mcp/tool-schemas.ts +439 -0
- package/src/server/logger.ts +88 -0
- package/src/server/tool-registry.ts +117 -0
- package/src/server/transports/mcp-transport.ts +374 -0
- package/src/server/transports/rest-transport.ts +258 -0
- package/tests/unit/agent-tools.test.ts +454 -0
- package/tests/unit/file-watcher.test.d.ts +2 -0
- package/tests/unit/file-watcher.test.d.ts.map +1 -0
- package/tests/unit/file-watcher.test.js +9 -0
- package/tests/unit/file-watcher.test.js.map +1 -0
- package/tests/unit/file-watcher.test.ts +7 -0
- package/tests/unit/search-integration.test.ts +256 -0
- package/tests/unit/services.test.d.ts +2 -0
- package/tests/unit/services.test.d.ts.map +1 -0
- package/tests/unit/services.test.js +9 -0
- package/tests/unit/services.test.js.map +1 -0
- package/tests/unit/services.test.ts +7 -0
- package/tsconfig.json +23 -0
- package/webpack.backend.config.js +60 -0
- package/webpack.config.js +34 -0
- package/models/Xenova/all-MiniLM-L6-v2/config.json +0 -25
- package/models/Xenova/all-MiniLM-L6-v2/onnx/model_quantized.onnx +0 -0
- package/models/Xenova/all-MiniLM-L6-v2/tokenizer.json +0 -30686
- package/models/Xenova/all-MiniLM-L6-v2/tokenizer_config.json +0 -15
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"service-types.js","sourceRoot":"","sources":["../../src/backend/service-types.ts"],"names":[],"mappings":";AAAA;;;GAGG"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding Service
|
|
3
|
+
* Generates semantic embeddings for code blocks using local transformers
|
|
4
|
+
*
|
|
5
|
+
* Note: Uses 'any' type for transformers.js API due to lack of complete TypeScript definitions
|
|
6
|
+
*/
|
|
7
|
+
import { ILogger } from '../core/types';
|
|
8
|
+
/**
|
|
9
|
+
* Service for generating semantic embeddings using local transformers
|
|
10
|
+
* Uses: Xenova/all-MiniLM-L6-v2 (384-dimensional, lightweight ~80MB)
|
|
11
|
+
* Runtime: ~10-20ms per embedding on modern hardware
|
|
12
|
+
* Lightweight and efficient model for semantic search
|
|
13
|
+
*/
|
|
14
|
+
export declare class EmbeddingService {
|
|
15
|
+
private _model;
|
|
16
|
+
private _tokenizer;
|
|
17
|
+
private _isInitialized;
|
|
18
|
+
private _logger;
|
|
19
|
+
private _modelName;
|
|
20
|
+
private _maxLength;
|
|
21
|
+
constructor(logger?: ILogger);
|
|
22
|
+
/**
|
|
23
|
+
* Initialize embedding model (loads transformer model)
|
|
24
|
+
* First checks for bundled model cache, then falls back to HuggingFace download
|
|
25
|
+
* Optimized for fast initialization with quantized ONNX models
|
|
26
|
+
*
|
|
27
|
+
* @throws Error if model loading fails
|
|
28
|
+
*/
|
|
29
|
+
initialize(): Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Setup cache directory for transformers.js library
|
|
32
|
+
* Prioritizes bundled models in the package, falls back to downloaded cache
|
|
33
|
+
* Checks multiple possible locations for bundled models
|
|
34
|
+
*/
|
|
35
|
+
private _setupCacheDirectory;
|
|
36
|
+
/**
|
|
37
|
+
* Generate embedding for a single text
|
|
38
|
+
*
|
|
39
|
+
* @param text - Text to embed
|
|
40
|
+
* @returns 384-dimensional float32 array
|
|
41
|
+
* @throws Error if model not initialized or embedding fails
|
|
42
|
+
*/
|
|
43
|
+
generate(text: string): Promise<number[]>;
|
|
44
|
+
/**
|
|
45
|
+
* Generate embeddings for multiple texts (batch processing)
|
|
46
|
+
*
|
|
47
|
+
* @param texts - Array of texts to embed
|
|
48
|
+
* @param batchSize - Process this many at once (default: 32)
|
|
49
|
+
* @returns Array of 384-dimensional embeddings
|
|
50
|
+
*/
|
|
51
|
+
generateBatch(texts: string[], batchSize?: number): Promise<number[][]>;
|
|
52
|
+
/**
|
|
53
|
+
* Check if model is initialized
|
|
54
|
+
*/
|
|
55
|
+
isInitialized(): boolean;
|
|
56
|
+
/**
|
|
57
|
+
* Get embedding dimension (384 for all-MiniLM-L6-v2)
|
|
58
|
+
*/
|
|
59
|
+
getDimension(): number;
|
|
60
|
+
/**
|
|
61
|
+
* Get model name
|
|
62
|
+
*/
|
|
63
|
+
getModelName(): string;
|
|
64
|
+
/**
|
|
65
|
+
* Mean pooling: average token embeddings to get sentence embedding
|
|
66
|
+
* Private helper method
|
|
67
|
+
*/
|
|
68
|
+
private _meanPooling;
|
|
69
|
+
/**
|
|
70
|
+
* L2 normalization: normalize vector to unit length
|
|
71
|
+
* Private helper method
|
|
72
|
+
*/
|
|
73
|
+
private _normalize;
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=embedding-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-service.d.ts","sourceRoot":"","sources":["../../../src/backend/services/embedding-service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAMxC;;;;;GAKG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,UAAU,CAAe;gBAEd,MAAM,CAAC,EAAE,OAAO;IAInC;;;;;;OAMG;IACU,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA0CxC;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAiE5B;;;;;;OAMG;IACU,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA2CtD;;;;;;OAMG;IACU,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAW,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IA2BxF;;OAEG;IACI,aAAa,IAAI,OAAO;IAI/B;;OAEG;IACI,YAAY,IAAI,MAAM;IAI7B;;OAEG;IACI,YAAY,IAAI,MAAM;IAI7B;;;OAGG;IACH,OAAO,CAAC,YAAY;IA4CpB;;;OAGG;IACH,OAAO,CAAC,UAAU;CAiBnB"}
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Embedding Service
|
|
4
|
+
* Generates semantic embeddings for code blocks using local transformers
|
|
5
|
+
*
|
|
6
|
+
* Note: Uses 'any' type for transformers.js API due to lack of complete TypeScript definitions
|
|
7
|
+
*/
|
|
8
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
9
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
12
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
13
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
14
|
+
}
|
|
15
|
+
Object.defineProperty(o, k2, desc);
|
|
16
|
+
}) : (function(o, m, k, k2) {
|
|
17
|
+
if (k2 === undefined) k2 = k;
|
|
18
|
+
o[k2] = m[k];
|
|
19
|
+
}));
|
|
20
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
21
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
22
|
+
}) : function(o, v) {
|
|
23
|
+
o["default"] = v;
|
|
24
|
+
});
|
|
25
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
26
|
+
if (mod && mod.__esModule) return mod;
|
|
27
|
+
var result = {};
|
|
28
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
29
|
+
__setModuleDefault(result, mod);
|
|
30
|
+
return result;
|
|
31
|
+
};
|
|
32
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
33
|
+
exports.EmbeddingService = void 0;
|
|
34
|
+
const logger_1 = require("../core/logger");
|
|
35
|
+
const path = __importStar(require("path"));
|
|
36
|
+
const fs = __importStar(require("fs"));
|
|
37
|
+
const os = __importStar(require("os"));
|
|
38
|
+
/**
|
|
39
|
+
* Service for generating semantic embeddings using local transformers
|
|
40
|
+
* Uses: Xenova/all-MiniLM-L6-v2 (384-dimensional, lightweight ~80MB)
|
|
41
|
+
* Runtime: ~10-20ms per embedding on modern hardware
|
|
42
|
+
* Lightweight and efficient model for semantic search
|
|
43
|
+
*/
|
|
44
|
+
class EmbeddingService {
|
|
45
|
+
constructor(logger) {
|
|
46
|
+
this._isInitialized = false;
|
|
47
|
+
this._modelName = 'Xenova/all-MiniLM-L6-v2';
|
|
48
|
+
this._maxLength = 512; // Max tokens for embedding
|
|
49
|
+
this._logger = logger || new logger_1.NoOpLogger();
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Initialize embedding model (loads transformer model)
|
|
53
|
+
* First checks for bundled model cache, then falls back to HuggingFace download
|
|
54
|
+
* Optimized for fast initialization with quantized ONNX models
|
|
55
|
+
*
|
|
56
|
+
* @throws Error if model loading fails
|
|
57
|
+
*/
|
|
58
|
+
async initialize() {
|
|
59
|
+
if (this._isInitialized) {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
try {
|
|
63
|
+
this._logger.debug('Initializing embedding model', { model: this._modelName });
|
|
64
|
+
// Set cache directory to use bundled models if available
|
|
65
|
+
this._setupCacheDirectory();
|
|
66
|
+
// Optimize transformers.js for faster inference
|
|
67
|
+
// Use WASM backend for better performance on CPU
|
|
68
|
+
process.env.TRANSFORMERS_CACHE_DIR = process.env.TRANSFORMERS_CACHE;
|
|
69
|
+
// Dynamically import transformers to avoid hard dependency
|
|
70
|
+
// Users must install @xenova/transformers separately
|
|
71
|
+
const transformersModule = await Promise.resolve().then(() => __importStar(require('@xenova/transformers')));
|
|
72
|
+
// Load pre-trained model and tokenizer
|
|
73
|
+
// Will use cached version if available, otherwise download from HuggingFace
|
|
74
|
+
// Using from_pretrained which automatically uses ONNX if available
|
|
75
|
+
const startTime = Date.now();
|
|
76
|
+
this._tokenizer = await transformersModule.AutoTokenizer.from_pretrained(this._modelName);
|
|
77
|
+
this._model = await transformersModule.AutoModel.from_pretrained(this._modelName, {
|
|
78
|
+
quantized: true, // Use quantized model if available (faster)
|
|
79
|
+
});
|
|
80
|
+
const initTime = Date.now() - startTime;
|
|
81
|
+
this._isInitialized = true;
|
|
82
|
+
this._logger.info('Embedding model initialized', {
|
|
83
|
+
model: this._modelName,
|
|
84
|
+
initTimeMs: initTime,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
const message = `Failed to initialize embedding model: ${String(error)}`;
|
|
89
|
+
this._logger.error(message, error);
|
|
90
|
+
throw new Error(message);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Setup cache directory for transformers.js library
|
|
95
|
+
* Prioritizes bundled models in the package, falls back to downloaded cache
|
|
96
|
+
* Checks multiple possible locations for bundled models
|
|
97
|
+
*/
|
|
98
|
+
_setupCacheDirectory() {
|
|
99
|
+
const possibleBundledPaths = [];
|
|
100
|
+
// Find package root by walking up from current file location
|
|
101
|
+
let currentDir = __dirname;
|
|
102
|
+
for (let i = 0; i < 10; i++) {
|
|
103
|
+
const packageJsonPath = path.join(currentDir, 'package.json');
|
|
104
|
+
if (fs.existsSync(packageJsonPath)) {
|
|
105
|
+
const modelsPath = path.join(currentDir, 'models');
|
|
106
|
+
possibleBundledPaths.push(modelsPath);
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
currentDir = path.dirname(currentDir);
|
|
110
|
+
}
|
|
111
|
+
// Also check common locations
|
|
112
|
+
possibleBundledPaths.push(
|
|
113
|
+
// Development: src/backend/services/../../../models (3 levels up)
|
|
114
|
+
path.join(__dirname, '..', '..', '..', 'models'),
|
|
115
|
+
// Distribution/npm package: dist/backend/services/../../../../models (4 levels up)
|
|
116
|
+
path.join(__dirname, '..', '..', '..', '..', 'models'),
|
|
117
|
+
// Bundled distribution from process entry point
|
|
118
|
+
path.join(path.dirname(process.argv[1] || ''), '..', 'models'),
|
|
119
|
+
// Project working directory
|
|
120
|
+
path.join(process.cwd(), 'models'));
|
|
121
|
+
// Check each possible bundled path
|
|
122
|
+
for (const bundledModelsDir of possibleBundledPaths) {
|
|
123
|
+
if (!bundledModelsDir || !fs.existsSync(bundledModelsDir)) {
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
const modelPath = path.join(bundledModelsDir, 'Xenova', 'all-MiniLM-L6-v2');
|
|
127
|
+
if (fs.existsSync(modelPath)) {
|
|
128
|
+
this._logger.info('Using bundled model cache', {
|
|
129
|
+
cacheDir: bundledModelsDir,
|
|
130
|
+
modelPath: modelPath
|
|
131
|
+
});
|
|
132
|
+
process.env.TRANSFORMERS_CACHE = bundledModelsDir;
|
|
133
|
+
process.env.HF_HOME = bundledModelsDir;
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
// If no bundled models found, fall back to standard HuggingFace cache
|
|
138
|
+
this._logger.warn('Bundled models not found, will use downloaded cache');
|
|
139
|
+
const homeDir = process.env.HOME || process.env.USERPROFILE || os.homedir?.() || '.';
|
|
140
|
+
const cacheDir = path.join(homeDir, '.cache', 'huggingface', 'transformers');
|
|
141
|
+
// Create cache directory if it doesn't exist
|
|
142
|
+
try {
|
|
143
|
+
if (!fs.existsSync(cacheDir)) {
|
|
144
|
+
fs.mkdirSync(cacheDir, { recursive: true });
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
catch (error) {
|
|
148
|
+
this._logger.debug('Could not create cache directory', error);
|
|
149
|
+
}
|
|
150
|
+
// Set environment variables for transformers.js cache
|
|
151
|
+
process.env.TRANSFORMERS_CACHE = cacheDir;
|
|
152
|
+
process.env.HF_HOME = cacheDir;
|
|
153
|
+
this._logger.debug('Model cache directory', { path: cacheDir });
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Generate embedding for a single text
|
|
157
|
+
*
|
|
158
|
+
* @param text - Text to embed
|
|
159
|
+
* @returns 384-dimensional float32 array
|
|
160
|
+
* @throws Error if model not initialized or embedding fails
|
|
161
|
+
*/
|
|
162
|
+
async generate(text) {
|
|
163
|
+
if (!this._isInitialized) {
|
|
164
|
+
await this.initialize();
|
|
165
|
+
}
|
|
166
|
+
try {
|
|
167
|
+
// Validate input
|
|
168
|
+
if (!text || text.trim().length === 0) {
|
|
169
|
+
this._logger.warn('Empty text provided for embedding');
|
|
170
|
+
// Return zero vector for empty text
|
|
171
|
+
return new Array(384).fill(0);
|
|
172
|
+
}
|
|
173
|
+
// Truncate text if too long
|
|
174
|
+
const truncated = text.length > 5000 ? text.substring(0, 5000) : text;
|
|
175
|
+
// Tokenize
|
|
176
|
+
const tokens = this._tokenizer(truncated, {
|
|
177
|
+
padding: true,
|
|
178
|
+
truncation: true,
|
|
179
|
+
max_length: this._maxLength,
|
|
180
|
+
});
|
|
181
|
+
// Generate embeddings
|
|
182
|
+
const { last_hidden_state } = await this._model(tokens);
|
|
183
|
+
// Pool embeddings (mean pooling)
|
|
184
|
+
const embedding = this._meanPooling(last_hidden_state.data, last_hidden_state.shape, tokens.input_ids.data);
|
|
185
|
+
// Normalize (L2 normalization)
|
|
186
|
+
const normalized = this._normalize(embedding);
|
|
187
|
+
return normalized;
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
this._logger.error('Error generating embedding', error);
|
|
191
|
+
throw new Error(`Failed to generate embedding: ${String(error)}`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Generate embeddings for multiple texts (batch processing)
|
|
196
|
+
*
|
|
197
|
+
* @param texts - Array of texts to embed
|
|
198
|
+
* @param batchSize - Process this many at once (default: 32)
|
|
199
|
+
* @returns Array of 384-dimensional embeddings
|
|
200
|
+
*/
|
|
201
|
+
async generateBatch(texts, batchSize = 32) {
|
|
202
|
+
if (!this._isInitialized) {
|
|
203
|
+
await this.initialize();
|
|
204
|
+
}
|
|
205
|
+
const embeddings = [];
|
|
206
|
+
this._logger.debug('Generating batch embeddings', {
|
|
207
|
+
textCount: texts.length,
|
|
208
|
+
batchSize,
|
|
209
|
+
});
|
|
210
|
+
// Process in batches to manage memory
|
|
211
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
212
|
+
const batch = texts.slice(i, Math.min(i + batchSize, texts.length));
|
|
213
|
+
const batchEmbeddings = await Promise.all(batch.map((text) => this.generate(text)));
|
|
214
|
+
embeddings.push(...batchEmbeddings);
|
|
215
|
+
this._logger.debug('Batch progress', {
|
|
216
|
+
processed: Math.min(i + batchSize, texts.length),
|
|
217
|
+
total: texts.length,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
return embeddings;
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Check if model is initialized
|
|
224
|
+
*/
|
|
225
|
+
isInitialized() {
|
|
226
|
+
return this._isInitialized;
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Get embedding dimension (384 for all-MiniLM-L6-v2)
|
|
230
|
+
*/
|
|
231
|
+
getDimension() {
|
|
232
|
+
return 384;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Get model name
|
|
236
|
+
*/
|
|
237
|
+
getModelName() {
|
|
238
|
+
return this._modelName;
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Mean pooling: average token embeddings to get sentence embedding
|
|
242
|
+
* Private helper method
|
|
243
|
+
*/
|
|
244
|
+
_meanPooling(embeddings, shape, inputIds) {
|
|
245
|
+
// Handle different shape formats
|
|
246
|
+
let shapeArray;
|
|
247
|
+
if (Array.isArray(shape)) {
|
|
248
|
+
shapeArray = shape;
|
|
249
|
+
}
|
|
250
|
+
else if (shape && shape.dims && Array.isArray(shape.dims)) {
|
|
251
|
+
shapeArray = shape.dims;
|
|
252
|
+
}
|
|
253
|
+
else {
|
|
254
|
+
// Fallback: assume shape is [1, seqLength, 384]
|
|
255
|
+
const seqLength = inputIds.length;
|
|
256
|
+
shapeArray = [1, seqLength, 384];
|
|
257
|
+
}
|
|
258
|
+
const [, seqLength, hiddenDim] = shapeArray;
|
|
259
|
+
const result = new Array(hiddenDim).fill(0);
|
|
260
|
+
let tokenCount = 0;
|
|
261
|
+
for (let i = 0; i < seqLength; i++) {
|
|
262
|
+
// Skip padding tokens ([PAD] = 0)
|
|
263
|
+
if (inputIds[i] === 0) {
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
for (let j = 0; j < hiddenDim; j++) {
|
|
267
|
+
result[j] += embeddings[i * hiddenDim + j];
|
|
268
|
+
}
|
|
269
|
+
tokenCount++;
|
|
270
|
+
}
|
|
271
|
+
// Average
|
|
272
|
+
if (tokenCount > 0) {
|
|
273
|
+
for (let j = 0; j < hiddenDim; j++) {
|
|
274
|
+
result[j] /= tokenCount;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
return result;
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* L2 normalization: normalize vector to unit length
|
|
281
|
+
* Private helper method
|
|
282
|
+
*/
|
|
283
|
+
_normalize(vector) {
|
|
284
|
+
let norm = 0;
|
|
285
|
+
// Calculate L2 norm
|
|
286
|
+
for (const val of vector) {
|
|
287
|
+
norm += val * val;
|
|
288
|
+
}
|
|
289
|
+
norm = Math.sqrt(norm);
|
|
290
|
+
// Normalize (if norm is 0, return as-is)
|
|
291
|
+
if (norm === 0) {
|
|
292
|
+
return vector;
|
|
293
|
+
}
|
|
294
|
+
return vector.map((val) => val / norm);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
exports.EmbeddingService = EmbeddingService;
|
|
298
|
+
//# sourceMappingURL=embedding-service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-service.js","sourceRoot":"","sources":["../../../src/backend/services/embedding-service.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;AACH,uDAAuD;;;;;;;;;;;;;;;;;;;;;;;;;;AAGvD,2CAA4C;AAC5C,2CAA6B;AAC7B,uCAAyB;AACzB,uCAAyB;AAEzB;;;;;GAKG;AACH,MAAa,gBAAgB;IAQ3B,YAAmB,MAAgB;QAL3B,mBAAc,GAAY,KAAK,CAAC;QAEhC,eAAU,GAAW,yBAAyB,CAAC;QAC/C,eAAU,GAAW,GAAG,CAAC,CAAC,2BAA2B;QAG3D,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,IAAI,mBAAU,EAAE,CAAC;IAC5C,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,UAAU;QACrB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;YAE/E,yDAAyD;YACzD,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAE5B,gDAAgD;YAChD,iDAAiD;YACjD,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;YAEpE,2DAA2D;YAC3D,qDAAqD;YACrD,MAAM,kBAAkB,GAAG,wDAAa,sBAAsB,GAAC,CAAC;YAEhE,uCAAuC;YACvC,4EAA4E;YAC5E,mEAAmE;YACnE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE7B,IAAI,CAAC,UAAU,GAAG,MAAO,kBAA0B,CAAC,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACnG,IAAI,CAAC,MAAM,GAAG,MAAO,kBAA0B,CAAC,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,UAAU,EAAE;gBACzF,SAAS,EAAE,IAAI,EAAE,4CAA4C;aAC9D,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACxC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,6BAA6B,EAAE;gBAC/C,KAAK,EAAE,IAAI,CAAC,UAAU;gBACtB,UAAU,EAAE,QAAQ;aACrB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,yCAAyC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAc,CAAC,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,oBAAoB;QAC1B,MAAM,oBAAoB,GAAa,EAAE,CAAC;QAE1C,6DAA6D;QAC7D,IAAI,UAAU,GAAG,SAAS,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YAC9D,IAAI,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;gBACnC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;gBACnD,oBAAoB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACtC,MAAM;YACR,CAAC;YACD,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACxC,CAAC;QAED,8BAA8B;QAC9B,oBAAoB,CAAC,IAAI;QACvB,kEAAkE;QAClE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC;QAChD,mFAAmF;QACnF,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC;QACtD,gDAAgD;QAChD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC;QAC9D,4BAA4B;QAC5B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CACnC,CAAC;QAEF,mCAAmC;QACnC,KAAK,MAAM,gBAAgB,IAAI,oBAAoB,EAAE,CAAC;YACpD,IAAI,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC1D,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;YAC5E,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,2BAA2B,EAAE;oBAC7C,QAAQ,EAAE,gBAAgB;oBAC1B,SAAS,EAAE,SAAS;iBACrB,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,gBAAgB,CAAC;gBAClD,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,gBAAgB,CAAC;gBACvC,OAAO;YACT,CAAC;QACH,CAAC;QAED,sEAAsE;QACtE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,IAAI,GAAG,CAAC;QACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;QAE7E,6CAA6C;QAC7C,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC7B,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,KAAc,CAAC,CAAC;QACzE,CAAC;QAED,sDAAsD;QACtD,OAAO,CAAC,GAAG,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,QAAQ,CAAC;QAC/B,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAClE,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,QAAQ,CAAC,IAAY;QAChC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC;YACH,iBAAiB;YACjB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;gBACvD,oCAAoC;gBACpC,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChC,CAAC;YAED,4BAA4B;YAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAEtE,WAAW;YACX,MAAM,MAAM,GAAI,IAAI,CAAC,UAAkB,CAAC,SAAS,EAAE;gBACjD,OAAO,EAAE,IAAI;gBACb,UAAU,EAAE,IAAI;gBAChB,UAAU,EAAE,IAAI,CAAC,UAAU;aAC5B,CAAC,CAAC;YAEH,sBAAsB;YACtB,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAO,IAAI,CAAC,MAAc,CAAC,MAAM,CAAC,CAAC;YAEjE,iCAAiC;YACjC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CACjC,iBAAiB,CAAC,IAAI,EACtB,iBAAiB,CAAC,KAAK,EACvB,MAAM,CAAC,SAAS,CAAC,IAAI,CACtB,CAAC;YAEF,+BAA+B;YAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAE9C,OAAO,UAAU,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAc,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,iCAAiC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,aAAa,CAAC,KAAe,EAAE,YAAoB,EAAE;QAChE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,UAAU,GAAe,EAAE,CAAC;QAElC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE;YAChD,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,SAAS;SACV,CAAC,CAAC;QAEH,sCAAsC;QACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YACpE,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpF,UAAU,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAEpC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE;gBACnC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC;gBAChD,KAAK,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACI,aAAa;QAClB,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;OAEG;IACI,YAAY;QACjB,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACI,YAAY;QACjB,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED;;;OAGG;IACK,YAAY,CAClB,UAAwB,EACxB,KAA2C,EAC3C,QAAqB;QAErB,iCAAiC;QACjC,IAAI,UAAoB,CAAC;QACzB,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,UAAU,GAAG,KAAK,CAAC;QACrB,CAAC;aAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5D,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,UAAU,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QACnC,CAAC;QAED,MAAM,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,GAAG,UAAU,CAAC;QAC5C,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,kCAAkC;YAClC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC;YAC7C,CAAC;YAED,UAAU,EAAE,CAAC;QACf,CAAC;QAED,UAAU;QACV,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACK,UAAU,CAAC,MAAgB;QACjC,IAAI,IAAI,GAAG,CAAC,CAAC;QAEb,oBAAoB;QACpB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,IAAI,IAAI,GAAG,GAAG,GAAG,CAAC;QACpB,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvB,yCAAyC;QACzC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YACf,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;IACzC,CAAC;CACF;AAjTD,4CAiTC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { SearchEngine } from '@ngao/search-core';
|
|
2
|
+
export interface FileWatcherConfig {
|
|
3
|
+
ignored?: string | string[];
|
|
4
|
+
stabilityThreshold?: number;
|
|
5
|
+
pollInterval?: number;
|
|
6
|
+
}
|
|
7
|
+
export declare class FileWatcher {
|
|
8
|
+
private searchEngine;
|
|
9
|
+
private config?;
|
|
10
|
+
private watcher;
|
|
11
|
+
private isWatching;
|
|
12
|
+
constructor(searchEngine: SearchEngine, config?: FileWatcherConfig | undefined);
|
|
13
|
+
watch(dirPath: string): Promise<void>;
|
|
14
|
+
stop(): Promise<void>;
|
|
15
|
+
isActive(): boolean;
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=file-watcher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-watcher.d.ts","sourceRoot":"","sources":["../../../src/backend/services/file-watcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,MAAM,WAAW,iBAAiB;IAChC,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,qBAAa,WAAW;IAIV,OAAO,CAAC,YAAY;IAAgB,OAAO,CAAC,MAAM,CAAC;IAH/D,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,UAAU,CAAQ;gBAEN,YAAY,EAAE,YAAY,EAAU,MAAM,CAAC,+BAAmB;IAE5E,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmDrC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAS3B,QAAQ,IAAI,OAAO;CAGpB"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.FileWatcher = void 0;
|
|
27
|
+
const chokidar = __importStar(require("chokidar"));
|
|
28
|
+
class FileWatcher {
|
|
29
|
+
constructor(searchEngine, config) {
|
|
30
|
+
this.searchEngine = searchEngine;
|
|
31
|
+
this.config = config;
|
|
32
|
+
this.watcher = null;
|
|
33
|
+
this.isWatching = false;
|
|
34
|
+
}
|
|
35
|
+
async watch(dirPath) {
|
|
36
|
+
if (this.isWatching) {
|
|
37
|
+
throw new Error('File watcher is already active');
|
|
38
|
+
}
|
|
39
|
+
this.watcher = chokidar.watch(dirPath, {
|
|
40
|
+
persistent: true,
|
|
41
|
+
ignored: this.config?.ignored || /(^|[\/\\])\.|node_modules/,
|
|
42
|
+
ignoreInitial: true,
|
|
43
|
+
awaitWriteFinish: {
|
|
44
|
+
stabilityThreshold: this.config?.stabilityThreshold || 100,
|
|
45
|
+
pollInterval: this.config?.pollInterval || 100,
|
|
46
|
+
},
|
|
47
|
+
});
|
|
48
|
+
this.watcher.on('change', (filePath) => {
|
|
49
|
+
console.log(`File changed: ${filePath}`);
|
|
50
|
+
// TODO: @ngao/search-core doesn't have reindexFile method
|
|
51
|
+
// For now, we'll need to reindex the directory
|
|
52
|
+
const path = require('path');
|
|
53
|
+
const dirPath = path.dirname(filePath);
|
|
54
|
+
this.searchEngine.indexDirectory(dirPath).catch((err) => {
|
|
55
|
+
console.error(`Failed to reindex ${filePath}:`, err);
|
|
56
|
+
});
|
|
57
|
+
});
|
|
58
|
+
this.watcher.on('add', (filePath) => {
|
|
59
|
+
console.log(`File added: ${filePath}`);
|
|
60
|
+
// TODO: @ngao/search-core doesn't have indexFile method
|
|
61
|
+
// For now, we'll need to index the directory
|
|
62
|
+
const path = require('path');
|
|
63
|
+
const dirPath = path.dirname(filePath);
|
|
64
|
+
this.searchEngine.indexDirectory(dirPath).catch((err) => {
|
|
65
|
+
console.error(`Failed to index ${filePath}:`, err);
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
this.watcher.on('unlink', async (filePath) => {
|
|
69
|
+
console.log(`File deleted: ${filePath}`);
|
|
70
|
+
// Note: Core library should handle file deletion
|
|
71
|
+
// For now, we just log it
|
|
72
|
+
});
|
|
73
|
+
this.watcher.on('error', (error) => {
|
|
74
|
+
console.error('File watcher error:', error);
|
|
75
|
+
});
|
|
76
|
+
this.isWatching = true;
|
|
77
|
+
console.log(`Watching directory: ${dirPath}`);
|
|
78
|
+
}
|
|
79
|
+
async stop() {
|
|
80
|
+
if (this.watcher) {
|
|
81
|
+
await this.watcher.close();
|
|
82
|
+
this.watcher = null;
|
|
83
|
+
this.isWatching = false;
|
|
84
|
+
console.log('File watcher stopped');
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
isActive() {
|
|
88
|
+
return this.isWatching;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
exports.FileWatcher = FileWatcher;
|
|
92
|
+
//# sourceMappingURL=file-watcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-watcher.js","sourceRoot":"","sources":["../../../src/backend/services/file-watcher.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AACA,mDAAqC;AAQrC,MAAa,WAAW;IAItB,YAAoB,YAA0B,EAAU,MAA0B;QAA9D,iBAAY,GAAZ,YAAY,CAAc;QAAU,WAAM,GAAN,MAAM,CAAoB;QAH1E,YAAO,GAA8B,IAAI,CAAA;QACzC,eAAU,GAAG,KAAK,CAAA;IAE2D,CAAC;IAEtF,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAA;QACnD,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE;YACrC,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,2BAA2B;YAC5D,aAAa,EAAE,IAAI;YACnB,gBAAgB,EAAE;gBAChB,kBAAkB,EAAE,IAAI,CAAC,MAAM,EAAE,kBAAkB,IAAI,GAAG;gBAC1D,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,IAAI,GAAG;aAC/C;SACF,CAAC,CAAA;QAEF,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC7C,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAA;YACxC,0DAA0D;YAC1D,+CAA+C;YAC/C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;YAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,GAAU,EAAE,EAAE;gBAC7D,OAAO,CAAC,KAAK,CAAC,qBAAqB,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAA;YACtD,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC1C,OAAO,CAAC,GAAG,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAA;YACtC,wDAAwD;YACxD,6CAA6C;YAC7C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;YAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,GAAU,EAAE,EAAE;gBAC7D,OAAO,CAAC,KAAK,CAAC,mBAAmB,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAA;YACpD,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;YAC3C,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAA;YACxC,iDAAiD;YACjD,0BAA0B;QAC5B,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAA;QAC7C,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;QACtB,OAAO,CAAC,GAAG,CAAC,uBAAuB,OAAO,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;YAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;YACnB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;YACvB,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAA;QACrC,CAAC;IACH,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,UAAU,CAAA;IACxB,CAAC;CACF;AArED,kCAqEC"}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index Information Service for NGAO Search
|
|
3
|
+
* Aggregates and formats detailed index statistics
|
|
4
|
+
*/
|
|
5
|
+
import { IndexStats } from '../core/models';
|
|
6
|
+
import { IMetadataRepository } from '../repositories/repository.interface';
|
|
7
|
+
import { SearchEngine } from '../api/search-engine';
|
|
8
|
+
/**
|
|
9
|
+
* File type definition
|
|
10
|
+
*/
|
|
11
|
+
export type FileType = '.ts' | '.js' | '.py' | '.json' | '.md' | '.yml' | '.yaml' | string;
|
|
12
|
+
/**
|
|
13
|
+
* Detailed information about indexed files
|
|
14
|
+
*/
|
|
15
|
+
export interface FileStatistics {
|
|
16
|
+
/** File path */
|
|
17
|
+
filePath: string;
|
|
18
|
+
/** File type (python, markdown, etc.) */
|
|
19
|
+
fileType: FileType;
|
|
20
|
+
/** Number of blocks in file */
|
|
21
|
+
blockCount: number;
|
|
22
|
+
/** Breakdown of blocks by type */
|
|
23
|
+
blocksByType: Record<string, number>;
|
|
24
|
+
/** File size in bytes */
|
|
25
|
+
fileSizeBytes: number;
|
|
26
|
+
/** Unix timestamp when indexed */
|
|
27
|
+
indexedAt: number;
|
|
28
|
+
/** SHA256 hash for change detection */
|
|
29
|
+
hash: string;
|
|
30
|
+
/** Whether embeddings are present for this file */
|
|
31
|
+
embeddingsPresent: boolean;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* File index metadata
|
|
35
|
+
*/
|
|
36
|
+
export interface FileIndexMetadata {
|
|
37
|
+
/** File path */
|
|
38
|
+
filePath: string;
|
|
39
|
+
/** File type */
|
|
40
|
+
fileType: FileType;
|
|
41
|
+
/** File hash */
|
|
42
|
+
fileHash: string;
|
|
43
|
+
/** File size in bytes */
|
|
44
|
+
fileSizeBytes: number;
|
|
45
|
+
/** When indexed */
|
|
46
|
+
indexedAt: number;
|
|
47
|
+
/** Number of blocks */
|
|
48
|
+
blockCount: number;
|
|
49
|
+
/** Blocks by type */
|
|
50
|
+
blocksByType: Record<string, number>;
|
|
51
|
+
/** Whether embeddings are present */
|
|
52
|
+
embeddingPresent: boolean;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Detailed index information including breakdown
|
|
56
|
+
*/
|
|
57
|
+
export interface DetailedIndexInfo {
|
|
58
|
+
/** Summary statistics */
|
|
59
|
+
summary: IndexStats;
|
|
60
|
+
/** Per-file breakdown */
|
|
61
|
+
fileBreakdown: FileStatistics[];
|
|
62
|
+
/** Index metadata */
|
|
63
|
+
indexMetadata: {
|
|
64
|
+
/** Index format version */
|
|
65
|
+
indexVersion: string;
|
|
66
|
+
/** When index was created */
|
|
67
|
+
createdAt: number;
|
|
68
|
+
/** Number of times reindexed */
|
|
69
|
+
totalReindexes: number;
|
|
70
|
+
/** Average reindex time in ms */
|
|
71
|
+
averageReindexTimeMs: number;
|
|
72
|
+
/** Indexing strategies used */
|
|
73
|
+
indexingStrategies: string[];
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Service for aggregating and formatting index information
|
|
78
|
+
*/
|
|
79
|
+
export declare class IndexInformationService {
|
|
80
|
+
private _metadataRepository;
|
|
81
|
+
private _searchEngine;
|
|
82
|
+
/**
|
|
83
|
+
* Constructor with dependency injection
|
|
84
|
+
* @param blockRepository - Block data access
|
|
85
|
+
* @param metadataRepository - Metadata data access
|
|
86
|
+
* @param searchEngine - Search engine for aggregate data
|
|
87
|
+
*/
|
|
88
|
+
constructor(_metadataRepository: IMetadataRepository, _searchEngine: SearchEngine);
|
|
89
|
+
/**
|
|
90
|
+
* Get detailed index information
|
|
91
|
+
* @returns Detailed index info with breakdown
|
|
92
|
+
*/
|
|
93
|
+
getDetailedIndexInfo(): Promise<DetailedIndexInfo>;
|
|
94
|
+
/**
|
|
95
|
+
* Get file statistics
|
|
96
|
+
* @param filePath - Optional specific file path
|
|
97
|
+
* @returns Array of file statistics
|
|
98
|
+
*/
|
|
99
|
+
getFileStatistics(filePath?: string): Promise<FileStatistics[]>;
|
|
100
|
+
/**
|
|
101
|
+
* Get file breakdown by aggregating statistics
|
|
102
|
+
* @private
|
|
103
|
+
* @returns Array of file statistics
|
|
104
|
+
*/
|
|
105
|
+
private _getFileBreakdown;
|
|
106
|
+
/**
|
|
107
|
+
* Format file metadata into file statistics
|
|
108
|
+
* @private
|
|
109
|
+
* @param metadata - File metadata
|
|
110
|
+
* @returns Formatted file statistics
|
|
111
|
+
*/
|
|
112
|
+
private _formatFileStatistic;
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=index-information-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-information-service.d.ts","sourceRoot":"","sources":["../../../src/backend/services/index-information-service.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,sCAAsC,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAE3F;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,QAAQ,EAAE,QAAQ,CAAC;IACnB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,yBAAyB;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,mDAAmD;IACnD,iBAAiB,EAAE,OAAO,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB;IAChB,QAAQ,EAAE,QAAQ,CAAC;IACnB,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,uBAAuB;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,qBAAqB;IACrB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,qCAAqC;IACrC,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,yBAAyB;IACzB,OAAO,EAAE,UAAU,CAAC;IACpB,yBAAyB;IACzB,aAAa,EAAE,cAAc,EAAE,CAAC;IAChC,qBAAqB;IACrB,aAAa,EAAE;QACb,2BAA2B;QAC3B,YAAY,EAAE,MAAM,CAAC;QACrB,6BAA6B;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,gCAAgC;QAChC,cAAc,EAAE,MAAM,CAAC;QACvB,iCAAiC;QACjC,oBAAoB,EAAE,MAAM,CAAC;QAC7B,+BAA+B;QAC/B,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH;AAED;;GAEG;AACH,qBAAa,uBAAuB;IAQhC,OAAO,CAAC,mBAAmB;IAC3B,OAAO,CAAC,aAAa;IARvB;;;;;OAKG;gBAEO,mBAAmB,EAAE,mBAAmB,EACxC,aAAa,EAAE,YAAY;IAGrC;;;OAGG;IACU,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,CAAC;IA8B/D;;;;OAIG;IACU,iBAAiB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAe5E;;;;OAIG;YACW,iBAAiB;IAU/B;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;CAY7B"}
|