codesearch-mcp 0.0.17-alpha.1 → 0.1.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/README.md +95 -6
- package/dist/app.js +70 -10
- package/dist/src/commands/ConfigCommand.d.ts +1 -0
- package/dist/src/commands/ConfigCommand.js +84 -4
- package/dist/src/commands/IndexCommand.js +35 -7
- package/dist/src/commands/McpCommand.js +9 -4
- package/dist/src/commands/{SetupCommand.d.ts → WebCommand.d.ts} +3 -2
- package/dist/src/commands/WebCommand.js +56 -0
- package/dist/src/config-store.d.ts +85 -4
- package/dist/src/config-store.js +157 -32
- package/dist/src/config.d.ts +32 -8
- package/dist/src/config.js +93 -30
- package/dist/src/constants.d.ts +35 -0
- package/dist/src/constants.js +57 -4
- package/dist/src/container.d.ts +10 -6
- package/dist/src/container.js +10 -5
- package/dist/src/db/index.d.ts +1 -1
- package/dist/src/db/index.js +2 -1
- package/dist/src/db/schema.d.ts +111 -0
- package/dist/src/db/schema.js +15 -1
- package/dist/src/graph.d.ts +60 -0
- package/dist/src/graph.js +239 -120
- package/dist/src/help.js +20 -5
- package/dist/src/interfaces/IAstParser.d.ts +20 -3
- package/dist/src/interfaces/IChunker.d.ts +24 -0
- package/dist/src/interfaces/IChunker.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +27 -0
- package/dist/src/interfaces/IDatabase.js +2 -0
- package/dist/src/interfaces/IEmbedder.d.ts +15 -2
- package/dist/src/interfaces/ILogger.d.ts +10 -0
- package/dist/src/interfaces/ILogger.js +2 -0
- package/dist/src/interfaces/IMcpToolHandler.d.ts +22 -0
- package/dist/src/interfaces/IMcpToolHandler.js +2 -0
- package/dist/src/interfaces/IMonorepoResolver.d.ts +33 -0
- package/dist/src/interfaces/IMonorepoResolver.js +2 -0
- package/dist/src/interfaces/IProjectDiscovery.d.ts +28 -0
- package/dist/src/interfaces/IProjectDiscovery.js +2 -0
- package/dist/src/interfaces/ISearcher.d.ts +37 -0
- package/dist/src/interfaces/ISearcher.js +2 -0
- package/dist/src/interfaces/index.d.ts +8 -0
- package/dist/src/interfaces/index.js +2 -0
- package/dist/src/owner-path.d.ts +10 -0
- package/dist/src/owner-path.js +27 -0
- package/dist/src/project-selector.d.ts +1 -1
- package/dist/src/services/AstChunker.d.ts +31 -0
- package/dist/src/services/AstChunker.js +114 -0
- package/dist/src/services/AstParserService.d.ts +10 -6
- package/dist/src/services/AstParserService.js +60 -15
- package/dist/src/services/DatabaseService.d.ts +14 -1
- package/dist/src/services/DatabaseService.js +70 -25
- package/dist/src/services/EmbedderFactory.d.ts +9 -0
- package/dist/src/services/EmbedderFactory.js +29 -0
- package/dist/src/services/IndexerService.d.ts +22 -12
- package/dist/src/services/IndexerService.js +106 -46
- package/dist/src/services/InfinityEmbedder.d.ts +79 -0
- package/dist/src/services/InfinityEmbedder.js +199 -0
- package/dist/src/services/LoggerService.d.ts +2 -1
- package/dist/src/services/McpServer.d.ts +16 -7
- package/dist/src/services/McpServer.js +54 -367
- package/dist/src/services/MonorepoAwareDiscoveryService.d.ts +62 -0
- package/dist/src/services/MonorepoAwareDiscoveryService.js +125 -0
- package/dist/src/services/NxMonorepoResolver.d.ts +45 -0
- package/dist/src/services/NxMonorepoResolver.js +308 -0
- package/dist/src/services/OllamaEmbedder.d.ts +19 -5
- package/dist/src/services/OllamaEmbedder.js +91 -36
- package/dist/src/services/OpenAICompatibleEmbedder.d.ts +25 -0
- package/dist/src/services/OpenAICompatibleEmbedder.js +75 -0
- package/dist/src/services/ProjectDiscoveryService.d.ts +8 -0
- package/dist/src/{env-loader.js → services/ProjectDiscoveryService.js} +16 -5
- package/dist/src/services/SearcherService.d.ts +46 -7
- package/dist/src/services/SearcherService.js +183 -53
- package/dist/src/services/VoyageEmbedder.d.ts +22 -0
- package/dist/src/services/VoyageEmbedder.js +72 -0
- package/dist/src/services/WebServerService.d.ts +77 -0
- package/dist/src/services/WebServerService.js +267 -0
- package/dist/src/services/http-retry.d.ts +19 -0
- package/dist/src/services/http-retry.js +54 -0
- package/dist/src/services/reranker.d.ts +15 -0
- package/dist/src/services/reranker.js +44 -0
- package/dist/src/setup.d.ts +6 -0
- package/dist/src/setup.js +249 -31
- package/dist/src/tools/FindSymbolTool.d.ts +8 -0
- package/dist/src/tools/FindSymbolTool.js +45 -0
- package/dist/src/tools/GetCallersTool.d.ts +8 -0
- package/dist/src/tools/GetCallersTool.js +42 -0
- package/dist/src/tools/GetDependenciesTool.d.ts +8 -0
- package/dist/src/tools/GetDependenciesTool.js +38 -0
- package/dist/src/tools/GetDependentsTool.d.ts +8 -0
- package/dist/src/tools/GetDependentsTool.js +57 -0
- package/dist/src/tools/ListFilesTool.d.ts +8 -0
- package/dist/src/tools/ListFilesTool.js +39 -0
- package/dist/src/tools/ListProjectsTool.d.ts +8 -0
- package/dist/src/tools/ListProjectsTool.js +44 -0
- package/dist/src/tools/ReadFileRangeTool.d.ts +8 -0
- package/dist/src/tools/ReadFileRangeTool.js +44 -0
- package/dist/src/tools/ReadFileTool.d.ts +8 -0
- package/dist/src/tools/ReadFileTool.js +42 -0
- package/dist/src/tools/SearchCodebaseTool.d.ts +13 -0
- package/dist/src/tools/SearchCodebaseTool.js +81 -0
- package/dist/src/tools/format-helpers.d.ts +8 -0
- package/dist/src/tools/format-helpers.js +58 -0
- package/dist/src/tools/index.d.ts +9 -0
- package/dist/src/tools/index.js +21 -0
- package/dist/src/web/dashboard.d.ts +9 -0
- package/dist/src/web/dashboard.js +342 -0
- package/docker/docker-compose.infinity-gpu.yml +19 -0
- package/docker/docker-compose.yml +25 -0
- package/docs/web-dashboard.md +187 -0
- package/package.json +25 -3
- package/dist/src/ast.d.ts +0 -2
- package/dist/src/ast.js +0 -110
- package/dist/src/commands/SetupCommand.js +0 -14
- package/dist/src/db/data-source.d.ts +0 -2
- package/dist/src/db/data-source.js +0 -13
- package/dist/src/db/entities/code-chunk.entity.d.ts +0 -9
- package/dist/src/db/entities/code-chunk.entity.js +0 -54
- package/dist/src/db/entities/code-dependency.entity.d.ts +0 -9
- package/dist/src/db/entities/code-dependency.entity.js +0 -54
- package/dist/src/db/entities/code-symbol.entity.d.ts +0 -10
- package/dist/src/db/entities/code-symbol.entity.js +0 -59
- package/dist/src/db/init.d.ts +0 -2
- package/dist/src/db/init.js +0 -62
- package/dist/src/env-loader.d.ts +0 -1
- package/dist/src/indexer.d.ts +0 -17
- package/dist/src/indexer.js +0 -242
- package/dist/src/mcp.d.ts +0 -1
- package/dist/src/mcp.js +0 -403
- package/dist/src/ollama.d.ts +0 -2
- package/dist/src/ollama.js +0 -6
- package/dist/src/searcher.d.ts +0 -74
- package/dist/src/searcher.js +0 -446
- package/dist/vitest.config.d.ts +0 -2
- package/dist/vitest.config.js +0 -9
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MonorepoAwareDiscoveryService = void 0;
|
|
4
|
+
exports.toLeafProjects = toLeafProjects;
|
|
5
|
+
const ProjectDiscoveryService_1 = require("./ProjectDiscoveryService");
|
|
6
|
+
/**
|
|
7
|
+
* Expand discovery units into the leaf projects to index: a nested monorepo
|
|
8
|
+
* unit (with `children`) becomes its sub-projects; an ordinary project stays
|
|
9
|
+
* itself. Shared by IndexCommand (indexing loop) and SearcherService (name→path
|
|
10
|
+
* resolution) so both agree on the leaf set.
|
|
11
|
+
*/
|
|
12
|
+
function toLeafProjects(projects) {
|
|
13
|
+
return projects.flatMap(p => (p.children && p.children.length > 0 ? p.children : [p]));
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Project discovery that is monorepo-aware.
|
|
17
|
+
*
|
|
18
|
+
* When a monorepo is detected (via the resolver), it uses the resolver
|
|
19
|
+
* to discover projects. Otherwise, it falls back to the original
|
|
20
|
+
* flat-workspace discovery (directories with package.json). Cross-project
|
|
21
|
+
* import resolution relies on the tsconfig `paths` attached via
|
|
22
|
+
* `rootTsConfigPath` — no separate scope map is built.
|
|
23
|
+
*
|
|
24
|
+
* Single Responsibility: only discovers projects and workspace metadata.
|
|
25
|
+
* Does not index or search.
|
|
26
|
+
*
|
|
27
|
+
* Implements {@link IProjectDiscovery} so it can be wired as the default
|
|
28
|
+
* discovery in the container. `discover()` returns one entry per top-level
|
|
29
|
+
* project: when the root itself is a monorepo, each entry carries the
|
|
30
|
+
* workspace-wide `rootTsConfigPath`; under a flat root, an ordinary project
|
|
31
|
+
* gets `rootTsConfigPath: null`, while a top-level directory that is itself a
|
|
32
|
+
* nested monorepo is returned as a single unit with expanded `children` (see
|
|
33
|
+
* `toLeafProjects`). `analyzeWorkspace()` exposes the full workspace metadata
|
|
34
|
+
* (monorepo flag, root tsconfig, project list).
|
|
35
|
+
*/
|
|
36
|
+
class MonorepoAwareDiscoveryService {
|
|
37
|
+
resolvers;
|
|
38
|
+
logger;
|
|
39
|
+
flatDiscovery;
|
|
40
|
+
constructor(resolvers, logger,
|
|
41
|
+
/** Discovery used when no monorepo is detected. Defaults to flat package.json scan. */
|
|
42
|
+
flatDiscovery = new ProjectDiscoveryService_1.ProjectDiscoveryService()) {
|
|
43
|
+
this.resolvers = resolvers;
|
|
44
|
+
this.logger = logger;
|
|
45
|
+
this.flatDiscovery = flatDiscovery;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* IProjectDiscovery contract: enumerate projects in the workspace.
|
|
49
|
+
* Attaches the workspace-wide root tsconfig to every project so the
|
|
50
|
+
* indexer can resolve cross-project path aliases.
|
|
51
|
+
*/
|
|
52
|
+
discover(rootDir) {
|
|
53
|
+
const workspace = this.analyzeWorkspace(rootDir);
|
|
54
|
+
// Case A: workspaceRoot itself is a monorepo — attach the workspace-wide
|
|
55
|
+
// tsconfig to every project (original behavior).
|
|
56
|
+
if (workspace.isMonorepo) {
|
|
57
|
+
return workspace.projects.map(p => ({ ...p, rootTsConfigPath: workspace.rootTsConfigPath }));
|
|
58
|
+
}
|
|
59
|
+
// Case B: flat workspaceRoot — a top-level project directory may itself be a
|
|
60
|
+
// nested monorepo (e.g. an Nx repo). Expand such a directory into its
|
|
61
|
+
// sub-projects as `children`, keeping the repo a single selectable unit.
|
|
62
|
+
return workspace.projects.map(p => this.expandIfNestedMonorepo(p));
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* If `project`'s directory is itself a monorepo (resolver detects it),
|
|
66
|
+
* return it with expanded `children` named `<repo>/<sub-project>`, each
|
|
67
|
+
* carrying the repo's workspace-wide tsconfig. Otherwise return it unchanged.
|
|
68
|
+
*/
|
|
69
|
+
expandIfNestedMonorepo(project) {
|
|
70
|
+
// Ordinary flat project — preserve the original contract (null root tsconfig).
|
|
71
|
+
const flat = { ...project, rootTsConfigPath: null };
|
|
72
|
+
const resolver = this.resolvers.find(r => r.isMonorepo(project.path));
|
|
73
|
+
if (!resolver)
|
|
74
|
+
return flat;
|
|
75
|
+
const rootTsConfigPath = resolver.getRootTsConfigPath(project.path);
|
|
76
|
+
const children = resolver.discoverProjects(project.path).map(c => ({
|
|
77
|
+
name: `${project.name}/${c.name}`,
|
|
78
|
+
path: c.path,
|
|
79
|
+
packageName: c.packageName,
|
|
80
|
+
type: c.type,
|
|
81
|
+
rootTsConfigPath,
|
|
82
|
+
}));
|
|
83
|
+
if (children.length === 0)
|
|
84
|
+
return flat;
|
|
85
|
+
this.logger.info('MonorepoAwareDiscovery: nested monorepo expanded', {
|
|
86
|
+
repo: project.name,
|
|
87
|
+
projects: children.length,
|
|
88
|
+
rootTsConfigPath,
|
|
89
|
+
});
|
|
90
|
+
return { ...project, children };
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Analyze the workspace root and return full workspace information:
|
|
94
|
+
* monorepo detection flag, root tsconfig path, and project list.
|
|
95
|
+
*/
|
|
96
|
+
analyzeWorkspace(rootDir) {
|
|
97
|
+
for (const resolver of this.resolvers) {
|
|
98
|
+
if (resolver.isMonorepo(rootDir)) {
|
|
99
|
+
this.logger.info('MonorepoAwareDiscovery: monorepo detected', { rootDir });
|
|
100
|
+
const monoProjects = resolver.discoverProjects(rootDir);
|
|
101
|
+
const rootTsConfigPath = resolver.getRootTsConfigPath(rootDir);
|
|
102
|
+
const projects = monoProjects.map(p => ({
|
|
103
|
+
name: p.name,
|
|
104
|
+
path: p.path,
|
|
105
|
+
packageName: p.packageName,
|
|
106
|
+
type: p.type,
|
|
107
|
+
}));
|
|
108
|
+
this.logger.debug('MonorepoAwareDiscovery: workspace analyzed', {
|
|
109
|
+
projects: projects.length,
|
|
110
|
+
rootTsConfigPath,
|
|
111
|
+
});
|
|
112
|
+
return { isMonorepo: true, rootTsConfigPath, projects };
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
// Fallback: flat workspace (original behavior) — delegated to the default discovery.
|
|
116
|
+
this.logger.debug('MonorepoAwareDiscovery: no monorepo detected, using flat discovery', { rootDir });
|
|
117
|
+
const projects = this.flatDiscovery.discover(rootDir);
|
|
118
|
+
return {
|
|
119
|
+
isMonorepo: false,
|
|
120
|
+
rootTsConfigPath: null,
|
|
121
|
+
projects,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
exports.MonorepoAwareDiscoveryService = MonorepoAwareDiscoveryService;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { IMonorepoResolver, MonorepoProject } from '../interfaces/IMonorepoResolver';
|
|
2
|
+
import type { ILogger } from '../interfaces';
|
|
3
|
+
/**
|
|
4
|
+
* Nx monorepo resolver.
|
|
5
|
+
*
|
|
6
|
+
* Detection: presence of `nx.json` at the workspace root.
|
|
7
|
+
*
|
|
8
|
+
* Project discovery strategies (results deduplicated by absolute path):
|
|
9
|
+
* 1. Read `workspace.json` or `angular.json` → projects map
|
|
10
|
+
* 2. Scan standard Nx directories (`apps/`, `libs/`, `packages/`) for
|
|
11
|
+
* `project.json`, then `package.json` (only when strategy 1 found nothing)
|
|
12
|
+
* 3. `tsconfig.base.json` `paths` — root-level libs referenced only by path
|
|
13
|
+
* alias (e.g. `contracts/`, `interfaces/`) that the directory scan misses
|
|
14
|
+
* 4. Root-level fallback: treat the workspace root itself as a project
|
|
15
|
+
* (standalone Nx setup) when nothing else matched
|
|
16
|
+
*
|
|
17
|
+
* tsconfig resolution:
|
|
18
|
+
* - Returns `tsconfig.base.json` at workspace root (Nx convention)
|
|
19
|
+
* - Falls back to `tsconfig.json` at workspace root
|
|
20
|
+
*
|
|
21
|
+
* Cross-project import resolution is handled by the AST parser via the
|
|
22
|
+
* workspace-wide tsconfig `paths` (tsconfig.base.json), so no scope map is built here.
|
|
23
|
+
*/
|
|
24
|
+
export declare class NxMonorepoResolver implements IMonorepoResolver {
|
|
25
|
+
private logger;
|
|
26
|
+
constructor(logger: ILogger);
|
|
27
|
+
isMonorepo(rootDir: string): boolean;
|
|
28
|
+
discoverProjects(rootDir: string): MonorepoProject[];
|
|
29
|
+
/**
|
|
30
|
+
* Discover projects from `tsconfig.base.json` `compilerOptions.paths`. Each
|
|
31
|
+
* alias (e.g. `@scope/interfaces` → `interfaces/src/index.ts`) resolves to a
|
|
32
|
+
* lib root (the path before `/src/`), which for real Nx libs contains a
|
|
33
|
+
* project.json. Aliases pointing at non-projects (no project.json/package.json,
|
|
34
|
+
* e.g. `@env` → `environments/environment.ts`) are skipped.
|
|
35
|
+
*/
|
|
36
|
+
private discoverFromTsconfigPaths;
|
|
37
|
+
/** Parse JSON that may contain comments / trailing commas (tsconfig style). */
|
|
38
|
+
private parseJsonc;
|
|
39
|
+
getRootTsConfigPath(rootDir: string): string | null;
|
|
40
|
+
private readWorkspaceJson;
|
|
41
|
+
private scanDirectory;
|
|
42
|
+
private tryReadProject;
|
|
43
|
+
private readPackageName;
|
|
44
|
+
private inferProjectType;
|
|
45
|
+
}
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.NxMonorepoResolver = void 0;
|
|
37
|
+
const fs = __importStar(require("fs"));
|
|
38
|
+
const path = __importStar(require("path"));
|
|
39
|
+
const file_utils_1 = require("../file-utils");
|
|
40
|
+
/**
|
|
41
|
+
* Nx monorepo resolver.
|
|
42
|
+
*
|
|
43
|
+
* Detection: presence of `nx.json` at the workspace root.
|
|
44
|
+
*
|
|
45
|
+
* Project discovery strategies (results deduplicated by absolute path):
|
|
46
|
+
* 1. Read `workspace.json` or `angular.json` → projects map
|
|
47
|
+
* 2. Scan standard Nx directories (`apps/`, `libs/`, `packages/`) for
|
|
48
|
+
* `project.json`, then `package.json` (only when strategy 1 found nothing)
|
|
49
|
+
* 3. `tsconfig.base.json` `paths` — root-level libs referenced only by path
|
|
50
|
+
* alias (e.g. `contracts/`, `interfaces/`) that the directory scan misses
|
|
51
|
+
* 4. Root-level fallback: treat the workspace root itself as a project
|
|
52
|
+
* (standalone Nx setup) when nothing else matched
|
|
53
|
+
*
|
|
54
|
+
* tsconfig resolution:
|
|
55
|
+
* - Returns `tsconfig.base.json` at workspace root (Nx convention)
|
|
56
|
+
* - Falls back to `tsconfig.json` at workspace root
|
|
57
|
+
*
|
|
58
|
+
* Cross-project import resolution is handled by the AST parser via the
|
|
59
|
+
* workspace-wide tsconfig `paths` (tsconfig.base.json), so no scope map is built here.
|
|
60
|
+
*/
|
|
61
|
+
class NxMonorepoResolver {
|
|
62
|
+
logger;
|
|
63
|
+
constructor(logger) {
|
|
64
|
+
this.logger = logger;
|
|
65
|
+
}
|
|
66
|
+
isMonorepo(rootDir) {
|
|
67
|
+
return fs.existsSync(path.join(rootDir, 'nx.json'));
|
|
68
|
+
}
|
|
69
|
+
discoverProjects(rootDir) {
|
|
70
|
+
// Deduplicate across strategies by absolute project path.
|
|
71
|
+
const byPath = new Map();
|
|
72
|
+
const add = (p) => { if (p && !byPath.has(p.path))
|
|
73
|
+
byPath.set(p.path, p); };
|
|
74
|
+
// Strategy 1: workspace.json / angular.json (authoritative when present)
|
|
75
|
+
const workspaceProjects = this.readWorkspaceJson(rootDir);
|
|
76
|
+
for (const p of workspaceProjects)
|
|
77
|
+
add(p);
|
|
78
|
+
// Strategy 2 & 3: scan standard directories (only when no workspace.json map)
|
|
79
|
+
if (byPath.size === 0) {
|
|
80
|
+
for (const dir of ['apps', 'libs', 'packages']) {
|
|
81
|
+
const fullDir = path.join(rootDir, dir);
|
|
82
|
+
if (!fs.existsSync(fullDir))
|
|
83
|
+
continue;
|
|
84
|
+
const type = dir === 'apps' ? 'app' : 'lib';
|
|
85
|
+
for (const p of this.scanDirectory(fullDir, type, rootDir))
|
|
86
|
+
add(p);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
// Strategy 4: tsconfig.base.json `paths` — captures libs that live at the
|
|
90
|
+
// repo root (e.g. `contracts/`, `interfaces/`) referenced by path aliases,
|
|
91
|
+
// which the apps/libs/packages scan misses. Only real Nx projects (with a
|
|
92
|
+
// project.json/package.json) are added; dedup handles overlap with the scan.
|
|
93
|
+
for (const p of this.discoverFromTsconfigPaths(rootDir))
|
|
94
|
+
add(p);
|
|
95
|
+
// Root-level fallback (standalone Nx setup)
|
|
96
|
+
if (byPath.size === 0) {
|
|
97
|
+
const rootProject = this.tryReadProject(rootDir, rootDir);
|
|
98
|
+
if (rootProject)
|
|
99
|
+
add(rootProject);
|
|
100
|
+
}
|
|
101
|
+
const projects = [...byPath.values()];
|
|
102
|
+
this.logger.debug('NxMonorepoResolver: discovered projects', { count: projects.length });
|
|
103
|
+
return projects;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Discover projects from `tsconfig.base.json` `compilerOptions.paths`. Each
|
|
107
|
+
* alias (e.g. `@scope/interfaces` → `interfaces/src/index.ts`) resolves to a
|
|
108
|
+
* lib root (the path before `/src/`), which for real Nx libs contains a
|
|
109
|
+
* project.json. Aliases pointing at non-projects (no project.json/package.json,
|
|
110
|
+
* e.g. `@env` → `environments/environment.ts`) are skipped.
|
|
111
|
+
*/
|
|
112
|
+
discoverFromTsconfigPaths(rootDir) {
|
|
113
|
+
const tsconfigPath = this.getRootTsConfigPath(rootDir);
|
|
114
|
+
if (!tsconfigPath)
|
|
115
|
+
return [];
|
|
116
|
+
let json;
|
|
117
|
+
try {
|
|
118
|
+
json = this.parseJsonc(fs.readFileSync(tsconfigPath, 'utf-8'));
|
|
119
|
+
}
|
|
120
|
+
catch (err) {
|
|
121
|
+
this.logger.warn('NxMonorepoResolver: failed to parse tsconfig paths', { file: tsconfigPath, reason: err.message });
|
|
122
|
+
return [];
|
|
123
|
+
}
|
|
124
|
+
const co = json?.compilerOptions ?? {};
|
|
125
|
+
const baseUrl = typeof co.baseUrl === 'string' ? co.baseUrl : '.';
|
|
126
|
+
const paths = co.paths;
|
|
127
|
+
if (!paths || typeof paths !== 'object')
|
|
128
|
+
return [];
|
|
129
|
+
const projects = [];
|
|
130
|
+
for (const targets of Object.values(paths)) {
|
|
131
|
+
const target = Array.isArray(targets) ? targets[0] : targets;
|
|
132
|
+
if (typeof target !== 'string')
|
|
133
|
+
continue;
|
|
134
|
+
const srcIdx = target.indexOf('/src/');
|
|
135
|
+
const relRoot = srcIdx >= 0 ? target.slice(0, srcIdx) : path.dirname(target);
|
|
136
|
+
const libRoot = path.resolve(rootDir, baseUrl, relRoot);
|
|
137
|
+
const project = this.tryReadProject(libRoot, rootDir); // null unless a real project marker exists
|
|
138
|
+
if (project)
|
|
139
|
+
projects.push(project);
|
|
140
|
+
}
|
|
141
|
+
return projects;
|
|
142
|
+
}
|
|
143
|
+
/** Parse JSON that may contain comments / trailing commas (tsconfig style). */
|
|
144
|
+
parseJsonc(text) {
|
|
145
|
+
try {
|
|
146
|
+
return JSON.parse(text);
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
const stripped = text
|
|
150
|
+
.replace(/\/\*[\s\S]*?\*\//g, '')
|
|
151
|
+
.replace(/(^|[^:])\/\/.*$/gm, '$1')
|
|
152
|
+
.replace(/,(\s*[}\]])/g, '$1');
|
|
153
|
+
return JSON.parse(stripped);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
getRootTsConfigPath(rootDir) {
|
|
157
|
+
// Nx convention: tsconfig.base.json at workspace root
|
|
158
|
+
const basePath = path.join(rootDir, 'tsconfig.base.json');
|
|
159
|
+
if (fs.existsSync(basePath))
|
|
160
|
+
return basePath;
|
|
161
|
+
// Fallback: tsconfig.json at workspace root
|
|
162
|
+
const rootPath = path.join(rootDir, 'tsconfig.json');
|
|
163
|
+
if (fs.existsSync(rootPath))
|
|
164
|
+
return rootPath;
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
// ── Private helpers ─────────────────────────────────────────────────────────
|
|
168
|
+
readWorkspaceJson(rootDir) {
|
|
169
|
+
const projects = [];
|
|
170
|
+
for (const filename of ['workspace.json', 'angular.json']) {
|
|
171
|
+
const filePath = path.join(rootDir, filename);
|
|
172
|
+
if (!fs.existsSync(filePath))
|
|
173
|
+
continue;
|
|
174
|
+
try {
|
|
175
|
+
const content = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
|
176
|
+
const projectsMap = content.projects ?? {};
|
|
177
|
+
for (const [name, value] of Object.entries(projectsMap)) {
|
|
178
|
+
// value can be a string (path) or an object with a `root` property.
|
|
179
|
+
// Guard per-entry: a malformed entry (object without `root`) must not
|
|
180
|
+
// abort discovery of the remaining projects in this file.
|
|
181
|
+
const rawRoot = typeof value === 'string'
|
|
182
|
+
? value
|
|
183
|
+
: value?.root;
|
|
184
|
+
if (typeof rawRoot !== 'string' || rawRoot.length === 0) {
|
|
185
|
+
this.logger.warn('NxMonorepoResolver: skipping workspace entry without valid root', { file: filePath, name });
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
const projectPath = path.resolve(rootDir, rawRoot);
|
|
189
|
+
if (!fs.existsSync(projectPath))
|
|
190
|
+
continue;
|
|
191
|
+
const type = this.inferProjectType(projectPath, name);
|
|
192
|
+
const packageName = this.readPackageName(projectPath);
|
|
193
|
+
projects.push({ name, path: projectPath, packageName, type });
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
catch (err) {
|
|
197
|
+
this.logger.warn('NxMonorepoResolver: failed to parse workspace file', {
|
|
198
|
+
file: filePath,
|
|
199
|
+
reason: err.message,
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return projects;
|
|
204
|
+
}
|
|
205
|
+
scanDirectory(dir, defaultType, rootDir) {
|
|
206
|
+
const projects = [];
|
|
207
|
+
let entries;
|
|
208
|
+
try {
|
|
209
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
210
|
+
}
|
|
211
|
+
catch {
|
|
212
|
+
return projects;
|
|
213
|
+
}
|
|
214
|
+
for (const entry of entries) {
|
|
215
|
+
if (!entry.isDirectory())
|
|
216
|
+
continue;
|
|
217
|
+
if (file_utils_1.IGNORE_DIRS.has(entry.name))
|
|
218
|
+
continue;
|
|
219
|
+
const projectPath = path.join(dir, entry.name);
|
|
220
|
+
const project = this.tryReadProject(projectPath, rootDir, defaultType);
|
|
221
|
+
if (project) {
|
|
222
|
+
projects.push(project);
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
// Check for nested grouping directories (e.g. libs/shared/utils)
|
|
226
|
+
const nested = this.scanDirectory(projectPath, defaultType, rootDir);
|
|
227
|
+
projects.push(...nested);
|
|
228
|
+
}
|
|
229
|
+
return projects;
|
|
230
|
+
}
|
|
231
|
+
tryReadProject(projectPath, rootDir, defaultType = 'lib') {
|
|
232
|
+
// Check for project.json (Nx project marker)
|
|
233
|
+
const projectJsonPath = path.join(projectPath, 'project.json');
|
|
234
|
+
const packageJsonPath = path.join(projectPath, 'package.json');
|
|
235
|
+
const hasProjectJson = fs.existsSync(projectJsonPath);
|
|
236
|
+
const hasPackageJson = fs.existsSync(packageJsonPath);
|
|
237
|
+
if (!hasProjectJson && !hasPackageJson)
|
|
238
|
+
return null;
|
|
239
|
+
let name = path.relative(rootDir, projectPath).replace(/\\/g, '/');
|
|
240
|
+
let type = defaultType;
|
|
241
|
+
let packageName = null;
|
|
242
|
+
// Read project.json for name and type
|
|
243
|
+
if (hasProjectJson) {
|
|
244
|
+
try {
|
|
245
|
+
const projectJson = JSON.parse(fs.readFileSync(projectJsonPath, 'utf-8'));
|
|
246
|
+
if (projectJson.name)
|
|
247
|
+
name = projectJson.name;
|
|
248
|
+
if (projectJson.projectType === 'application')
|
|
249
|
+
type = 'app';
|
|
250
|
+
else if (projectJson.projectType === 'library')
|
|
251
|
+
type = 'lib';
|
|
252
|
+
}
|
|
253
|
+
catch (err) {
|
|
254
|
+
this.logger.warn('NxMonorepoResolver: failed to parse project.json', {
|
|
255
|
+
file: projectJsonPath,
|
|
256
|
+
reason: err.message,
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
// Read package.json for npm scope
|
|
261
|
+
if (hasPackageJson) {
|
|
262
|
+
packageName = this.readPackageName(projectPath);
|
|
263
|
+
// If no project.json name, use package name without scope
|
|
264
|
+
if (!hasProjectJson && packageName) {
|
|
265
|
+
const shortName = packageName.replace(/^@[^/]+\//, '');
|
|
266
|
+
name = shortName;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
return { name, path: projectPath, packageName, type };
|
|
270
|
+
}
|
|
271
|
+
readPackageName(projectPath) {
|
|
272
|
+
const packageJsonPath = path.join(projectPath, 'package.json');
|
|
273
|
+
if (!fs.existsSync(packageJsonPath))
|
|
274
|
+
return null;
|
|
275
|
+
try {
|
|
276
|
+
const pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
|
|
277
|
+
return typeof pkg.name === 'string' ? pkg.name : null;
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
return null;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
inferProjectType(projectPath, name) {
|
|
284
|
+
// Check project.json
|
|
285
|
+
const projectJsonPath = path.join(projectPath, 'project.json');
|
|
286
|
+
if (fs.existsSync(projectJsonPath)) {
|
|
287
|
+
try {
|
|
288
|
+
const projectJson = JSON.parse(fs.readFileSync(projectJsonPath, 'utf-8'));
|
|
289
|
+
if (projectJson.projectType === 'application')
|
|
290
|
+
return 'app';
|
|
291
|
+
if (projectJson.projectType === 'library')
|
|
292
|
+
return 'lib';
|
|
293
|
+
}
|
|
294
|
+
catch { /* fall through */ }
|
|
295
|
+
}
|
|
296
|
+
// Infer from path
|
|
297
|
+
const rel = projectPath.replace(/\\/g, '/');
|
|
298
|
+
if (rel.includes('/apps/') || rel.includes('/app/'))
|
|
299
|
+
return 'app';
|
|
300
|
+
if (rel.includes('/libs/') || rel.includes('/lib/') || rel.includes('/packages/'))
|
|
301
|
+
return 'lib';
|
|
302
|
+
// Infer from name
|
|
303
|
+
if (name.includes('lib') || name.includes('shared') || name.includes('util'))
|
|
304
|
+
return 'lib';
|
|
305
|
+
return 'app';
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
exports.NxMonorepoResolver = NxMonorepoResolver;
|
|
@@ -1,11 +1,25 @@
|
|
|
1
1
|
import type { Config } from '../config';
|
|
2
|
-
import type { IEmbedder } from '../interfaces
|
|
2
|
+
import type { IEmbedder, InputType } from '../interfaces';
|
|
3
|
+
import type { ILogger } from '../interfaces';
|
|
4
|
+
import type { EmbedderDeps } from './OpenAICompatibleEmbedder';
|
|
3
5
|
export declare class OllamaEmbedder implements IEmbedder {
|
|
4
6
|
private config;
|
|
7
|
+
private logger;
|
|
8
|
+
readonly dimensions: number;
|
|
5
9
|
private client;
|
|
6
10
|
rerank?: (query: string, documents: string[], topN: number) => Promise<number[]>;
|
|
7
|
-
constructor(config: Config);
|
|
8
|
-
embed(text: string): Promise<number[]>;
|
|
9
|
-
|
|
10
|
-
|
|
11
|
+
constructor(config: Config, logger?: ILogger, deps?: EmbedderDeps);
|
|
12
|
+
embed(text: string, _inputType?: InputType): Promise<number[]>;
|
|
13
|
+
embedBatch(texts: string[], _inputType?: InputType): Promise<number[][]>;
|
|
14
|
+
listModels(): Promise<string[]>;
|
|
15
|
+
/**
|
|
16
|
+
* Pull the configured model if it is not already present locally, so a plain
|
|
17
|
+
* `rag-index` works after switching to (or back to) a local model that was
|
|
18
|
+
* never pulled — `rag-setup` is no longer the only place that fetches models.
|
|
19
|
+
* A tagless model name (`bge-m3`) matches its `:latest`/tagged install.
|
|
20
|
+
* Streams pull progress to the console: a live single-line bar on a TTY, or
|
|
21
|
+
* de-duplicated status lines otherwise.
|
|
22
|
+
*/
|
|
23
|
+
ensureModelAvailable(): Promise<void>;
|
|
24
|
+
private static formatPullProgress;
|
|
11
25
|
}
|
|
@@ -2,55 +2,110 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.OllamaEmbedder = void 0;
|
|
4
4
|
const ollama_1 = require("ollama");
|
|
5
|
+
const LoggerService_1 = require("./LoggerService");
|
|
6
|
+
const reranker_1 = require("./reranker");
|
|
7
|
+
const constants_1 = require("../constants");
|
|
5
8
|
class OllamaEmbedder {
|
|
6
9
|
config;
|
|
10
|
+
logger;
|
|
11
|
+
dimensions;
|
|
7
12
|
client;
|
|
8
13
|
rerank;
|
|
9
|
-
constructor(config) {
|
|
14
|
+
constructor(config, logger = LoggerService_1.LoggerService.silent(), deps = {}) {
|
|
10
15
|
this.config = config;
|
|
11
|
-
this.
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
}
|
|
16
|
-
else {
|
|
17
|
-
console.warn('[OllamaEmbedder] RERANKER_ENABLED=true but RERANKER_MODEL is not set — reranking disabled');
|
|
18
|
-
}
|
|
19
|
-
}
|
|
16
|
+
this.logger = logger;
|
|
17
|
+
this.client = new ollama_1.Ollama({ host: config.embedding.baseURL });
|
|
18
|
+
this.dimensions = config.embedding.dimensions ?? (0, constants_1.registryDimension)(config.embedding.model) ?? 0;
|
|
19
|
+
this.rerank = (0, reranker_1.makeReranker)(config, logger, deps);
|
|
20
20
|
}
|
|
21
|
-
|
|
21
|
+
// Ollama has no query/document asymmetry — inputType is accepted and ignored.
|
|
22
|
+
async embed(text, _inputType) {
|
|
22
23
|
const { embedding } = await this.client.embeddings({
|
|
23
|
-
model: this.config.
|
|
24
|
+
model: this.config.embedding.model,
|
|
24
25
|
prompt: text,
|
|
25
26
|
});
|
|
26
27
|
return embedding;
|
|
27
28
|
}
|
|
28
|
-
async
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
29
|
+
async embedBatch(texts, _inputType) {
|
|
30
|
+
if (texts.length === 0)
|
|
31
|
+
return [];
|
|
32
|
+
try {
|
|
33
|
+
// /api/embed accepts an array input and returns one vector per input.
|
|
34
|
+
const res = await this.client.embed({ model: this.config.embedding.model, input: texts });
|
|
35
|
+
if (res.embeddings?.length === texts.length)
|
|
36
|
+
return res.embeddings;
|
|
37
|
+
this.logger.warn('OllamaEmbedder: batch embed count mismatch, falling back to sequential', {
|
|
38
|
+
expected: texts.length, got: res.embeddings?.length ?? 0,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
this.logger.warn('OllamaEmbedder: batch embed failed, falling back to sequential', {
|
|
43
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
// Sequential fallback — the contract (one vector per input, in order) holds.
|
|
47
|
+
const out = [];
|
|
48
|
+
for (const t of texts)
|
|
49
|
+
out.push(await this.embed(t));
|
|
50
|
+
return out;
|
|
34
51
|
}
|
|
35
|
-
async
|
|
36
|
-
const res = await
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
52
|
+
async listModels() {
|
|
53
|
+
const res = await this.client.list();
|
|
54
|
+
return res.models.map(m => m.name).filter(Boolean).sort();
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Pull the configured model if it is not already present locally, so a plain
|
|
58
|
+
* `rag-index` works after switching to (or back to) a local model that was
|
|
59
|
+
* never pulled — `rag-setup` is no longer the only place that fetches models.
|
|
60
|
+
* A tagless model name (`bge-m3`) matches its `:latest`/tagged install.
|
|
61
|
+
* Streams pull progress to the console: a live single-line bar on a TTY, or
|
|
62
|
+
* de-duplicated status lines otherwise.
|
|
63
|
+
*/
|
|
64
|
+
async ensureModelAvailable() {
|
|
65
|
+
const model = this.config.embedding.model;
|
|
66
|
+
try {
|
|
67
|
+
const res = await this.client.list();
|
|
68
|
+
const present = res.models.some(m => m.name === model || m.name.startsWith(`${model}:`));
|
|
69
|
+
if (present)
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
catch (err) {
|
|
73
|
+
// Can't list (Ollama unreachable?) — try the pull; its error is clearer.
|
|
74
|
+
this.logger.warn('OllamaEmbedder: could not list models before pull', {
|
|
75
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
console.log(`Pulling Ollama model "${model}" (not found locally)...`);
|
|
79
|
+
this.logger.info('OllamaEmbedder: pulling model', { model });
|
|
80
|
+
const stream = await this.client.pull({ model, stream: true });
|
|
81
|
+
const isTty = Boolean(process.stdout.isTTY);
|
|
82
|
+
let lastLen = 0;
|
|
83
|
+
let lastLog = '';
|
|
84
|
+
for await (const part of stream) {
|
|
85
|
+
const out = ' ' + OllamaEmbedder.formatPullProgress(part);
|
|
86
|
+
if (isTty) {
|
|
87
|
+
process.stdout.write('\r' + out.padEnd(lastLen));
|
|
88
|
+
lastLen = out.length;
|
|
89
|
+
}
|
|
90
|
+
else if (out !== lastLog) {
|
|
91
|
+
console.log(out);
|
|
92
|
+
lastLog = out;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
if (isTty && lastLen > 0)
|
|
96
|
+
process.stdout.write('\r' + ' '.repeat(lastLen) + '\r');
|
|
97
|
+
console.log(` ✓ ${model} ready.`);
|
|
98
|
+
}
|
|
99
|
+
static formatPullProgress(p) {
|
|
100
|
+
const status = p.status ?? 'pulling';
|
|
101
|
+
if (p.total && p.total > 0) {
|
|
102
|
+
const pct = Math.min(100, Math.floor(((p.completed ?? 0) / p.total) * 100));
|
|
103
|
+
const width = 24;
|
|
104
|
+
const filled = Math.round((pct / 100) * width);
|
|
105
|
+
const bar = '[' + '█'.repeat(filled) + '░'.repeat(width - filled) + ']';
|
|
106
|
+
return `${status} ${bar} ${pct}% (${((p.completed ?? 0) / 1e6).toFixed(1)}/${(p.total / 1e6).toFixed(1)} MB)`;
|
|
51
107
|
}
|
|
52
|
-
|
|
53
|
-
return data.results.map(r => r.index);
|
|
108
|
+
return status;
|
|
54
109
|
}
|
|
55
110
|
}
|
|
56
111
|
exports.OllamaEmbedder = OllamaEmbedder;
|