@minko-ai/minko 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +577 -2
- package/assets/mascot-preview.png +0 -0
- package/assets/mascot-preview.txt +46 -0
- package/assets/mascot-source.png +0 -0
- package/assets/skills/README.md +21 -0
- package/assets/skills/brainstorming/LICENSE +21 -0
- package/assets/skills/brainstorming/SKILL.md +316 -0
- package/assets/skills/buddhist-method/LICENSE +21 -0
- package/assets/skills/buddhist-method/SKILL.md +137 -0
- package/assets/skills/buddhist-method/references/ariyasacca-debug.md +57 -0
- package/assets/skills/buddhist-method/references/extended-principles.md +100 -0
- package/assets/skills/code-humanizer/LICENSE +21 -0
- package/assets/skills/code-humanizer/SKILL.md +174 -0
- package/assets/skills/code-review/LICENSE +21 -0
- package/assets/skills/code-review/SKILL.md +232 -0
- package/assets/skills/code-review/reference/angular.md +768 -0
- package/assets/skills/code-review/reference/architecture-review-guide.md +472 -0
- package/assets/skills/code-review/reference/c.md +890 -0
- package/assets/skills/code-review/reference/code-quality-universal.md +488 -0
- package/assets/skills/code-review/reference/code-review-best-practices.md +136 -0
- package/assets/skills/code-review/reference/common-bugs-checklist.md +302 -0
- package/assets/skills/code-review/reference/cpp.md +893 -0
- package/assets/skills/code-review/reference/cross-cutting/async-concurrency-patterns.md +515 -0
- package/assets/skills/code-review/reference/cross-cutting/error-handling-principles.md +492 -0
- package/assets/skills/code-review/reference/cross-cutting/n-plus-one-queries.md +309 -0
- package/assets/skills/code-review/reference/cross-cutting/sql-injection-prevention.md +308 -0
- package/assets/skills/code-review/reference/cross-cutting/xss-prevention.md +264 -0
- package/assets/skills/code-review/reference/csharp.md +525 -0
- package/assets/skills/code-review/reference/css-less-sass.md +661 -0
- package/assets/skills/code-review/reference/dart.md +670 -0
- package/assets/skills/code-review/reference/django.md +985 -0
- package/assets/skills/code-review/reference/fastapi.md +580 -0
- package/assets/skills/code-review/reference/go.md +993 -0
- package/assets/skills/code-review/reference/java.md +409 -0
- package/assets/skills/code-review/reference/java8.md +586 -0
- package/assets/skills/code-review/reference/kotlin.md +1018 -0
- package/assets/skills/code-review/reference/nestjs.md +593 -0
- package/assets/skills/code-review/reference/performance-review-guide.md +816 -0
- package/assets/skills/code-review/reference/php.md +684 -0
- package/assets/skills/code-review/reference/python.md +1073 -0
- package/assets/skills/code-review/reference/qt.md +757 -0
- package/assets/skills/code-review/reference/react.md +871 -0
- package/assets/skills/code-review/reference/ruby.md +964 -0
- package/assets/skills/code-review/reference/rust.md +846 -0
- package/assets/skills/code-review/reference/security-review-guide.md +494 -0
- package/assets/skills/code-review/reference/svelte.md +1064 -0
- package/assets/skills/code-review/reference/swift.md +936 -0
- package/assets/skills/code-review/reference/typescript.md +1016 -0
- package/assets/skills/code-review/reference/vue.md +924 -0
- package/assets/skills/code-review/reference/zig.md +440 -0
- package/assets/skills/cyclomatic-complexity/LICENSE +201 -0
- package/assets/skills/cyclomatic-complexity/SKILL.md +69 -0
- package/assets/skills/five-whys/LICENSE +21 -0
- package/assets/skills/five-whys/SKILL.md +247 -0
- package/assets/skills/five-whys/examples/performance-regression.md +161 -0
- package/assets/skills/five-whys/examples/production-incident.md +134 -0
- package/assets/skills/five-whys/references/software-patterns.md +284 -0
- package/assets/skills/five-whys/references/toyota-origins.md +257 -0
- package/assets/skills/systematic-debugging/LICENSE +21 -0
- package/assets/skills/systematic-debugging/SKILL.md +283 -0
- package/assets/skills/test-driven-development/LICENSE +21 -0
- package/assets/skills/test-driven-development/SKILL.md +330 -0
- package/assets/skills/verification-before-completion/LICENSE +21 -0
- package/assets/skills/verification-before-completion/SKILL.md +120 -0
- package/assets/skills/vibe-sec/LICENSE +201 -0
- package/assets/skills/vibe-sec/SKILL.md +758 -0
- package/assets/skills/writing-plans/LICENSE +21 -0
- package/assets/skills/writing-plans/SKILL.md +204 -0
- package/bin/minko.js +2 -2
- package/dist/agent/approvalTokens.js +224 -0
- package/dist/agent/branchLock.js +315 -0
- package/dist/agent/compact.js +182 -0
- package/dist/agent/context.js +183 -0
- package/dist/agent/cron.js +326 -0
- package/dist/agent/hooks.js +211 -0
- package/dist/agent/lanes.js +511 -0
- package/dist/agent/loop.js +337 -0
- package/dist/agent/memory.js +159 -0
- package/dist/agent/permissions.js +71 -0
- package/dist/agent/plan.js +118 -0
- package/dist/agent/prompt.js +120 -0
- package/dist/agent/subagent.js +67 -0
- package/dist/agent/tasks.js +449 -0
- package/dist/agent/transcript.js +62 -0
- package/dist/agent/workerStore.js +172 -0
- package/dist/agent/workerTypes.js +1 -0
- package/dist/agent/workers.js +503 -0
- package/dist/cli/app.js +261 -0
- package/dist/cli/argv.js +206 -0
- package/dist/cli/commandsCommand.js +12 -0
- package/dist/cli/configCommand.js +170 -0
- package/dist/cli/doctorCommand.js +112 -0
- package/dist/cli/exportCommand.js +85 -0
- package/dist/cli/forkCommand.js +24 -0
- package/dist/cli/linkCommand.js +272 -0
- package/dist/cli/loginCommand.js +69 -0
- package/dist/cli/mcpLoginCommand.js +68 -0
- package/dist/cli/pluginCommand.js +113 -0
- package/dist/cli/powerCommand.js +174 -0
- package/dist/cli/promptFooter.js +74 -0
- package/dist/cli/relayCommand.js +38 -0
- package/dist/cli/sandboxCommand.js +31 -0
- package/dist/cli/serveCommand.js +104 -0
- package/dist/cli/statusCommand.js +120 -0
- package/dist/cli/tui.js +1104 -0
- package/dist/commands/registry.js +99 -0
- package/dist/config/paths.js +81 -0
- package/dist/config/store.js +605 -0
- package/dist/index.js +199 -0
- package/dist/link/protocol.js +159 -0
- package/dist/link/relay.js +184 -0
- package/dist/link/server.js +566 -0
- package/dist/link/tunnel.js +90 -0
- package/dist/lsp/client.js +340 -0
- package/dist/lsp/tool.js +177 -0
- package/dist/mcp/client.js +469 -0
- package/dist/mcp/httpClient.js +191 -0
- package/dist/mcp/server.js +206 -0
- package/dist/mcp/tools.js +173 -0
- package/dist/oauth/device.js +111 -0
- package/dist/oauth/store.js +56 -0
- package/dist/plugins/loader.js +291 -0
- package/dist/plugins/market.js +98 -0
- package/dist/providers/anthropic.js +279 -0
- package/dist/providers/factory.js +109 -0
- package/dist/providers/openai.js +182 -0
- package/dist/providers/retry.js +95 -0
- package/dist/providers/sse.js +67 -0
- package/dist/providers/types.js +10 -0
- package/dist/rag/embeddings.js +171 -0
- package/dist/rag/index.js +373 -0
- package/dist/rag/server.js +167 -0
- package/dist/rag/tool.js +148 -0
- package/dist/sandbox/index.js +55 -0
- package/dist/sdk/server.js +150 -0
- package/dist/sdk/session.js +104 -0
- package/dist/session/liveness.js +148 -0
- package/dist/session/recorder.js +49 -0
- package/dist/session/store.js +171 -0
- package/dist/skills/index.js +428 -0
- package/dist/theme.js +30 -0
- package/dist/tools/fs.js +195 -0
- package/dist/tools/notebook.js +109 -0
- package/dist/tools/registry.js +32 -0
- package/dist/tools/search.js +130 -0
- package/dist/tools/shell.js +200 -0
- package/dist/tools/structured.js +83 -0
- package/dist/tools/types.js +1 -0
- package/dist/ui/ansi.js +135 -0
- package/dist/ui/banner.js +103 -0
- package/dist/ui/input.js +49 -0
- package/dist/ui/inputModel.js +113 -0
- package/dist/ui/keyboard.js +142 -0
- package/dist/ui/mascot.js +34 -0
- package/dist/ui/mascotData.js +96 -0
- package/dist/ui/md.js +183 -0
- package/dist/ui/screen.js +49 -0
- package/dist/ui/spinner.js +52 -0
- package/dist/ui/transcript.js +287 -0
- package/dist/util/clipboard.js +53 -0
- package/dist/util/errors.js +50 -0
- package/dist/util/version.js +12 -0
- package/dist/web/fetch.js +168 -0
- package/dist/web/search.js +97 -0
- package/package.json +45 -36
- package/src/agent/loop.ts +0 -159
- package/src/agent/prompt.ts +0 -45
- package/src/cli/app.ts +0 -396
- package/src/cli/argv.ts +0 -139
- package/src/cli/configCommand.ts +0 -175
- package/src/config/paths.ts +0 -20
- package/src/config/store.ts +0 -149
- package/src/index.ts +0 -120
- package/src/providers/anthropic.ts +0 -215
- package/src/providers/openai.ts +0 -158
- package/src/providers/sse.ts +0 -57
- package/src/providers/types.ts +0 -59
- package/src/session/store.ts +0 -125
- package/src/theme.ts +0 -41
- package/src/tools/fs.ts +0 -143
- package/src/tools/registry.ts +0 -19
- package/src/tools/shell.ts +0 -120
- package/src/tools/types.ts +0 -26
- package/src/ui/ansi.ts +0 -110
- package/src/ui/banner.ts +0 -39
- package/src/ui/input.ts +0 -54
- package/src/ui/md.ts +0 -160
- package/src/ui/spinner.ts +0 -58
- package/src/util/errors.ts +0 -48
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
const IGNORED_DIRS = new Set([
|
|
5
|
+
'node_modules', '.git', '.minko', 'dist', 'build', 'out', '.next', '.nuxt',
|
|
6
|
+
'coverage', 'target', 'vendor', '__pycache__', '.venv', 'venv', '.idea',
|
|
7
|
+
'.vscode', '.cache', '.turbo',
|
|
8
|
+
]);
|
|
9
|
+
const IGNORED_FILES = new Set([
|
|
10
|
+
'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml', 'Cargo.lock', 'go.sum',
|
|
11
|
+
'composer.lock', 'poetry.lock',
|
|
12
|
+
]);
|
|
13
|
+
const IGNORED_EXT = /\.(png|jpe?g|gif|webp|ico|bmp|tiff?|pdf|zip|gz|tgz|rar|7z|xz|bz2|exe|dll|so|dylib|bin|o|a|class|jar|war|mp3|mp4|mov|avi|mkv|wav|flac|woff2?|ttf|otf|eot|sqlite|db)$/i;
|
|
14
|
+
/** Never feed credentials into the model context. */
|
|
15
|
+
const SENSITIVE = /^(\.env(\..*)?|\.npmrc|\.netrc|id_rsa|id_ed25519|.*\.(pem|key|pfx|p12))$/i;
|
|
16
|
+
/**
|
|
17
|
+
* A lightweight local retrieval index: line-based chunks scored with BM25.
|
|
18
|
+
* It is intentionally dependency-free and works offline.
|
|
19
|
+
*/
|
|
20
|
+
export class WorkspaceIndex {
|
|
21
|
+
root;
|
|
22
|
+
options;
|
|
23
|
+
chunks = [];
|
|
24
|
+
semanticError = null;
|
|
25
|
+
df = new Map();
|
|
26
|
+
avgLength = 0;
|
|
27
|
+
builtAt = 0;
|
|
28
|
+
building = null;
|
|
29
|
+
queryVectors = new Map();
|
|
30
|
+
constructor(root, options = {}) {
|
|
31
|
+
this.root = path.resolve(root);
|
|
32
|
+
this.options = {
|
|
33
|
+
maxFiles: 3_000,
|
|
34
|
+
maxFileBytes: 256 * 1024,
|
|
35
|
+
chunkLines: 40,
|
|
36
|
+
chunkOverlap: 8,
|
|
37
|
+
maxChunks: 12_000,
|
|
38
|
+
maxChunkChars: 4_000,
|
|
39
|
+
maxEmbeddingChunks: 1_000,
|
|
40
|
+
...options,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
get semanticActive() {
|
|
44
|
+
return Boolean(this.options.embedder) && this.chunks.some((chunk) => chunk.embedding);
|
|
45
|
+
}
|
|
46
|
+
stats() {
|
|
47
|
+
return {
|
|
48
|
+
chunks: this.chunks.length,
|
|
49
|
+
files: new Set(this.chunks.map((chunk) => chunk.path)).size,
|
|
50
|
+
semantic: this.semanticActive,
|
|
51
|
+
semanticError: this.semanticError,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
/** Rebuilds when the index is older than maxAgeMs. 0 forces a rebuild. */
|
|
55
|
+
async ensureFresh(maxAgeMs) {
|
|
56
|
+
if (this.builtAt > 0 && maxAgeMs > 0 && Date.now() - this.builtAt <= maxAgeMs) {
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
await this.rebuild();
|
|
60
|
+
}
|
|
61
|
+
async rebuild() {
|
|
62
|
+
if (this.building) {
|
|
63
|
+
await this.building;
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
this.building = this.doRebuild().finally(() => {
|
|
67
|
+
this.building = null;
|
|
68
|
+
});
|
|
69
|
+
await this.building;
|
|
70
|
+
}
|
|
71
|
+
search(query, topK = 6) {
|
|
72
|
+
const terms = [...new Set(tokenize(query))];
|
|
73
|
+
if (terms.length === 0 || this.chunks.length === 0) {
|
|
74
|
+
return [];
|
|
75
|
+
}
|
|
76
|
+
const phrase = query.trim().toLowerCase();
|
|
77
|
+
const hits = [];
|
|
78
|
+
for (const chunk of this.chunks) {
|
|
79
|
+
let score = this.scoreChunk(chunk, terms);
|
|
80
|
+
if (phrase.length >= 2 && chunk.text.toLowerCase().includes(phrase)) {
|
|
81
|
+
score += 3;
|
|
82
|
+
}
|
|
83
|
+
if (score > 0) {
|
|
84
|
+
hits.push(toHit(chunk, score));
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return rank(hits, topK);
|
|
88
|
+
}
|
|
89
|
+
/** BM25 + embeddings when configured; falls back to BM25 on any failure. */
|
|
90
|
+
async searchAsync(query, topK = 6) {
|
|
91
|
+
const embedder = this.options.embedder;
|
|
92
|
+
if (!embedder || !this.semanticActive) {
|
|
93
|
+
return this.search(query, topK);
|
|
94
|
+
}
|
|
95
|
+
const terms = [...new Set(tokenize(query))];
|
|
96
|
+
let queryVector = this.queryVectors.get(query);
|
|
97
|
+
if (!queryVector) {
|
|
98
|
+
try {
|
|
99
|
+
const [vector] = await embedder.embed([query]);
|
|
100
|
+
if (!vector) {
|
|
101
|
+
throw new Error('embedding 返回为空');
|
|
102
|
+
}
|
|
103
|
+
if (this.queryVectors.size >= 200) {
|
|
104
|
+
this.queryVectors.clear();
|
|
105
|
+
}
|
|
106
|
+
this.queryVectors.set(query, vector);
|
|
107
|
+
queryVector = vector;
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
this.semanticError = error instanceof Error ? error.message : String(error);
|
|
111
|
+
return this.search(query, topK);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
let maxBm25 = 1;
|
|
115
|
+
const scored = this.chunks.map((chunk) => {
|
|
116
|
+
const bm25 = this.scoreChunk(chunk, terms);
|
|
117
|
+
if (bm25 > maxBm25) {
|
|
118
|
+
maxBm25 = bm25;
|
|
119
|
+
}
|
|
120
|
+
return { chunk, bm25 };
|
|
121
|
+
});
|
|
122
|
+
const hits = [];
|
|
123
|
+
for (const { chunk, bm25 } of scored) {
|
|
124
|
+
const semantic = chunk.embedding ? cosine(queryVector, chunk.embedding) : 0;
|
|
125
|
+
const score = 0.45 * (bm25 / maxBm25) + 0.55 * Math.max(0, semantic);
|
|
126
|
+
if (score > 0.02) {
|
|
127
|
+
hits.push({ ...toHit(chunk, score), semantic });
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return rank(hits, topK);
|
|
131
|
+
}
|
|
132
|
+
scoreChunk(chunk, terms) {
|
|
133
|
+
const total = this.chunks.length;
|
|
134
|
+
const k1 = 1.2;
|
|
135
|
+
const b = 0.75;
|
|
136
|
+
let score = 0;
|
|
137
|
+
for (const term of terms) {
|
|
138
|
+
const tf = chunk.tokens.get(term) ?? 0;
|
|
139
|
+
if (tf === 0) {
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
const df = this.df.get(term) ?? 0;
|
|
143
|
+
const idf = Math.log(1 + (total - df + 0.5) / (df + 0.5));
|
|
144
|
+
const denominator = tf + k1 * (1 - b + b * (chunk.length / (this.avgLength || 1)));
|
|
145
|
+
score += idf * (tf * (k1 + 1)) / denominator;
|
|
146
|
+
}
|
|
147
|
+
return score;
|
|
148
|
+
}
|
|
149
|
+
async doRebuild() {
|
|
150
|
+
const files = await walkFiles(this.root, this.options);
|
|
151
|
+
const chunks = [];
|
|
152
|
+
const step = Math.max(1, this.options.chunkLines - this.options.chunkOverlap);
|
|
153
|
+
for (const file of files) {
|
|
154
|
+
if (chunks.length >= this.options.maxChunks) {
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
let text;
|
|
158
|
+
try {
|
|
159
|
+
const stat = await fs.promises.stat(file);
|
|
160
|
+
if (stat.size > this.options.maxFileBytes) {
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
text = await fs.promises.readFile(file, 'utf8');
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
if (text.includes('\u0000') || text.trim().length === 0) {
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
const relative = displayPath(this.root, file);
|
|
172
|
+
const lines = text.split(/\r?\n/);
|
|
173
|
+
for (let start = 0; start < lines.length; start += step) {
|
|
174
|
+
if (chunks.length >= this.options.maxChunks) {
|
|
175
|
+
break;
|
|
176
|
+
}
|
|
177
|
+
const end = Math.min(lines.length, start + this.options.chunkLines);
|
|
178
|
+
const block = lines.slice(start, end).join('\n');
|
|
179
|
+
if (block.trim().length === 0) {
|
|
180
|
+
if (end >= lines.length) {
|
|
181
|
+
break;
|
|
182
|
+
}
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
for (const piece of splitOversized(block, this.options.maxChunkChars)) {
|
|
186
|
+
if (chunks.length >= this.options.maxChunks) {
|
|
187
|
+
break;
|
|
188
|
+
}
|
|
189
|
+
chunks.push(makeChunk(relative, start + 1, end, piece));
|
|
190
|
+
}
|
|
191
|
+
if (end >= lines.length) {
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
this.semanticError = null;
|
|
197
|
+
this.queryVectors.clear();
|
|
198
|
+
if (this.options.embedder) {
|
|
199
|
+
try {
|
|
200
|
+
await this.attachEmbeddings(chunks);
|
|
201
|
+
}
|
|
202
|
+
catch (error) {
|
|
203
|
+
this.semanticError = error instanceof Error ? error.message : String(error);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
const df = new Map();
|
|
207
|
+
let totalLength = 0;
|
|
208
|
+
for (const chunk of chunks) {
|
|
209
|
+
totalLength += chunk.length;
|
|
210
|
+
for (const term of chunk.tokens.keys()) {
|
|
211
|
+
df.set(term, (df.get(term) ?? 0) + 1);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
this.chunks = chunks;
|
|
215
|
+
this.df = df;
|
|
216
|
+
this.avgLength = chunks.length > 0 ? totalLength / chunks.length : 0;
|
|
217
|
+
this.builtAt = Date.now();
|
|
218
|
+
}
|
|
219
|
+
async attachEmbeddings(chunks) {
|
|
220
|
+
const embedder = this.options.embedder;
|
|
221
|
+
if (!embedder) {
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
const target = chunks.slice(0, this.options.maxEmbeddingChunks);
|
|
225
|
+
const missing = [];
|
|
226
|
+
for (const chunk of target) {
|
|
227
|
+
const hash = crypto.createHash('sha1')
|
|
228
|
+
.update(embedder.model + '\n' + chunk.text)
|
|
229
|
+
.digest('hex');
|
|
230
|
+
const cached = this.options.embeddingCache?.get(hash);
|
|
231
|
+
if (cached) {
|
|
232
|
+
chunk.embedding = cached;
|
|
233
|
+
}
|
|
234
|
+
else {
|
|
235
|
+
missing.push({ chunk, hash });
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
if (missing.length === 0) {
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
const vectors = await embedder.embed(missing.map((entry) => entry.chunk.text));
|
|
242
|
+
if (vectors.length !== missing.length) {
|
|
243
|
+
throw new Error('embedding 返回数量不匹配:期望 ' + missing.length + ',实际 ' + vectors.length);
|
|
244
|
+
}
|
|
245
|
+
for (let index = 0; index < missing.length; index += 1) {
|
|
246
|
+
missing[index].chunk.embedding = vectors[index];
|
|
247
|
+
this.options.embeddingCache?.set(missing[index].hash, vectors[index]);
|
|
248
|
+
}
|
|
249
|
+
this.options.embeddingCache?.save();
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
function toHit(chunk, score) {
|
|
253
|
+
return {
|
|
254
|
+
path: chunk.path,
|
|
255
|
+
startLine: chunk.startLine,
|
|
256
|
+
endLine: chunk.endLine,
|
|
257
|
+
text: chunk.text,
|
|
258
|
+
score,
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
function rank(hits, topK) {
|
|
262
|
+
hits.sort((a, b) => b.score - a.score
|
|
263
|
+
|| a.path.localeCompare(b.path)
|
|
264
|
+
|| a.startLine - b.startLine);
|
|
265
|
+
return hits.slice(0, Math.max(1, Math.min(topK, 20)));
|
|
266
|
+
}
|
|
267
|
+
function cosine(a, b) {
|
|
268
|
+
const length = Math.min(a.length, b.length);
|
|
269
|
+
let dot = 0;
|
|
270
|
+
let normA = 0;
|
|
271
|
+
let normB = 0;
|
|
272
|
+
for (let index = 0; index < length; index += 1) {
|
|
273
|
+
dot += a[index] * b[index];
|
|
274
|
+
normA += a[index] * a[index];
|
|
275
|
+
normB += b[index] * b[index];
|
|
276
|
+
}
|
|
277
|
+
if (normA === 0 || normB === 0) {
|
|
278
|
+
return 0;
|
|
279
|
+
}
|
|
280
|
+
return dot / Math.sqrt(normA * normB);
|
|
281
|
+
}
|
|
282
|
+
/** Latin words plus CJK bigrams, which work without a dictionary. */
|
|
283
|
+
export function tokenize(text) {
|
|
284
|
+
const lower = text.toLowerCase();
|
|
285
|
+
const tokens = lower.match(/[a-z0-9_]{2,}/g) ?? [];
|
|
286
|
+
const cjk = lower.match(/[㐀-鿿]+/g) ?? [];
|
|
287
|
+
for (const sequence of cjk) {
|
|
288
|
+
if (sequence.length === 1) {
|
|
289
|
+
tokens.push(sequence);
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
for (let index = 0; index < sequence.length - 1; index += 1) {
|
|
293
|
+
tokens.push(sequence.slice(index, index + 2));
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
return tokens;
|
|
297
|
+
}
|
|
298
|
+
function makeChunk(filePath, startLine, endLine, text) {
|
|
299
|
+
const tokens = new Map();
|
|
300
|
+
let length = 0;
|
|
301
|
+
for (const term of tokenize(text)) {
|
|
302
|
+
tokens.set(term, (tokens.get(term) ?? 0) + 1);
|
|
303
|
+
length += 1;
|
|
304
|
+
}
|
|
305
|
+
// File names are strong signals ("auth.ts" for an auth question).
|
|
306
|
+
for (const term of tokenize(filePath)) {
|
|
307
|
+
tokens.set(term, (tokens.get(term) ?? 0) + 1);
|
|
308
|
+
length += 1;
|
|
309
|
+
}
|
|
310
|
+
return { path: filePath, startLine, endLine, text, tokens, length };
|
|
311
|
+
}
|
|
312
|
+
function splitOversized(text, max) {
|
|
313
|
+
if (text.length <= max) {
|
|
314
|
+
return [text];
|
|
315
|
+
}
|
|
316
|
+
const pieces = [];
|
|
317
|
+
const overlap = Math.min(200, Math.floor(max / 10));
|
|
318
|
+
const step = Math.max(1, max - overlap);
|
|
319
|
+
for (let index = 0; index < text.length; index += step) {
|
|
320
|
+
pieces.push(text.slice(index, index + max));
|
|
321
|
+
if (index + max >= text.length) {
|
|
322
|
+
break;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
return pieces;
|
|
326
|
+
}
|
|
327
|
+
async function walkFiles(root, options) {
|
|
328
|
+
const files = [];
|
|
329
|
+
const stack = [root];
|
|
330
|
+
while (stack.length > 0 && files.length < options.maxFiles) {
|
|
331
|
+
const dir = stack.pop();
|
|
332
|
+
if (!dir) {
|
|
333
|
+
break;
|
|
334
|
+
}
|
|
335
|
+
let entries;
|
|
336
|
+
try {
|
|
337
|
+
entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
|
338
|
+
}
|
|
339
|
+
catch {
|
|
340
|
+
continue;
|
|
341
|
+
}
|
|
342
|
+
for (const entry of entries) {
|
|
343
|
+
if (files.length >= options.maxFiles) {
|
|
344
|
+
break;
|
|
345
|
+
}
|
|
346
|
+
if (entry.isSymbolicLink()) {
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
if (entry.isDirectory()) {
|
|
350
|
+
if (IGNORED_DIRS.has(entry.name)) {
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
stack.push(path.join(dir, entry.name));
|
|
354
|
+
continue;
|
|
355
|
+
}
|
|
356
|
+
if (!entry.isFile()) {
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
if (IGNORED_DIRS.has(entry.name) || IGNORED_FILES.has(entry.name)) {
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
if (IGNORED_EXT.test(entry.name) || SENSITIVE.test(entry.name)) {
|
|
363
|
+
continue;
|
|
364
|
+
}
|
|
365
|
+
files.push(path.join(dir, entry.name));
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
return files.sort();
|
|
369
|
+
}
|
|
370
|
+
function displayPath(root, file) {
|
|
371
|
+
const relative = path.relative(root, file);
|
|
372
|
+
return (relative.length > 0 ? relative : file).split(path.sep).join('/');
|
|
373
|
+
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import http from 'node:http';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { loadConfig } from "../config/store.js";
|
|
4
|
+
import { readVersion } from "../util/version.js";
|
|
5
|
+
import { workspaceIndex } from "./tool.js";
|
|
6
|
+
/** HTTP API compatible with the remote retrieve_context client. */
|
|
7
|
+
export class RagService {
|
|
8
|
+
config;
|
|
9
|
+
root;
|
|
10
|
+
constructor(options = {}) {
|
|
11
|
+
this.config = options.config ?? loadConfig();
|
|
12
|
+
this.root = path.resolve(options.cwd ?? process.cwd());
|
|
13
|
+
}
|
|
14
|
+
async handle(method, rawUrl, body = {}) {
|
|
15
|
+
const url = new URL(rawUrl, 'http://localhost');
|
|
16
|
+
if (method === 'GET' && url.pathname === '/health') {
|
|
17
|
+
return {
|
|
18
|
+
status: 200,
|
|
19
|
+
body: { ok: true, service: 'minko-rag', version: readVersion() },
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
if (method === 'GET' && url.pathname === '/') {
|
|
23
|
+
return { status: 200, body: { ok: true }, html: INDEX_HTML };
|
|
24
|
+
}
|
|
25
|
+
if (method === 'GET' && url.pathname === '/v1/stats') {
|
|
26
|
+
const root = this.resolveRoot(url.searchParams.get('root'));
|
|
27
|
+
if (!root) {
|
|
28
|
+
return { status: 400, body: { error: 'root 不在服务工作目录内' } };
|
|
29
|
+
}
|
|
30
|
+
const index = workspaceIndex(root, this.config);
|
|
31
|
+
await index.ensureFresh(30_000);
|
|
32
|
+
return { status: 200, body: { root, ...index.stats() } };
|
|
33
|
+
}
|
|
34
|
+
if (method === 'POST' && url.pathname === '/v1/query') {
|
|
35
|
+
const query = typeof body.query === 'string' ? body.query.trim() : '';
|
|
36
|
+
if (query.length === 0) {
|
|
37
|
+
return { status: 400, body: { error: 'query is required' } };
|
|
38
|
+
}
|
|
39
|
+
const root = this.resolveRoot(typeof body.root === 'string' ? body.root : null);
|
|
40
|
+
if (!root) {
|
|
41
|
+
return { status: 400, body: { error: 'root 不在服务工作目录内' } };
|
|
42
|
+
}
|
|
43
|
+
const topK = typeof body.top_k === 'number' && Number.isFinite(body.top_k)
|
|
44
|
+
? Math.min(20, Math.max(1, Math.floor(body.top_k)))
|
|
45
|
+
: 6;
|
|
46
|
+
const index = workspaceIndex(root, this.config);
|
|
47
|
+
await index.ensureFresh(30_000);
|
|
48
|
+
const hits = await index.searchAsync(query, topK);
|
|
49
|
+
return {
|
|
50
|
+
status: 200,
|
|
51
|
+
body: {
|
|
52
|
+
root,
|
|
53
|
+
results: hits.map((hit) => ({
|
|
54
|
+
path: hit.path,
|
|
55
|
+
start_line: hit.startLine,
|
|
56
|
+
end_line: hit.endLine,
|
|
57
|
+
text: hit.text,
|
|
58
|
+
score: hit.score,
|
|
59
|
+
})),
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
if (method === 'POST' && url.pathname === '/v1/rebuild') {
|
|
64
|
+
const root = this.resolveRoot(typeof body.root === 'string' ? body.root : null);
|
|
65
|
+
if (!root) {
|
|
66
|
+
return { status: 400, body: { error: 'root 不在服务工作目录内' } };
|
|
67
|
+
}
|
|
68
|
+
const index = workspaceIndex(root, this.config);
|
|
69
|
+
await index.ensureFresh(0);
|
|
70
|
+
return { status: 200, body: { root, ...index.stats() } };
|
|
71
|
+
}
|
|
72
|
+
return { status: 404, body: { error: 'not found' } };
|
|
73
|
+
}
|
|
74
|
+
resolveRoot(raw) {
|
|
75
|
+
if (!raw || raw.trim().length === 0) {
|
|
76
|
+
return this.root;
|
|
77
|
+
}
|
|
78
|
+
const candidate = path.resolve(this.root, raw.trim());
|
|
79
|
+
const relative = path.relative(this.root, candidate);
|
|
80
|
+
if (relative.startsWith('..') || path.isAbsolute(relative)) {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
return candidate;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
export async function runRagServer(options = {}) {
|
|
87
|
+
const host = options.host ?? '127.0.0.1';
|
|
88
|
+
const port = options.port ?? 8787;
|
|
89
|
+
const service = new RagService();
|
|
90
|
+
const server = http.createServer((request, response) => {
|
|
91
|
+
void handleRequest(service, request, response).catch((error) => {
|
|
92
|
+
writeJson(response, 500, { error: error instanceof Error ? error.message : String(error) });
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
await new Promise((resolve, reject) => {
|
|
96
|
+
server.once('error', reject);
|
|
97
|
+
server.listen(port, host, () => resolve());
|
|
98
|
+
});
|
|
99
|
+
const address = server.address();
|
|
100
|
+
const actualPort = typeof address === 'object' && address ? address.port : port;
|
|
101
|
+
process.stdout.write('RAG 服务已启动:http://' + host + ':' + actualPort + '\n');
|
|
102
|
+
process.stdout.write(' GET /health GET /v1/stats POST /v1/query POST /v1/rebuild\n');
|
|
103
|
+
const shutdown = () => {
|
|
104
|
+
server.close(() => process.exit(0));
|
|
105
|
+
};
|
|
106
|
+
process.on('SIGINT', shutdown);
|
|
107
|
+
process.on('SIGTERM', shutdown);
|
|
108
|
+
await new Promise(() => { });
|
|
109
|
+
return 0;
|
|
110
|
+
}
|
|
111
|
+
async function handleRequest(service, request, response) {
|
|
112
|
+
const body = await readJson(request);
|
|
113
|
+
const result = await service.handle(request.method ?? 'GET', request.url ?? '/', body);
|
|
114
|
+
if (result.html) {
|
|
115
|
+
response.writeHead(result.status, { 'content-type': 'text/html; charset=utf-8' });
|
|
116
|
+
response.end(result.html);
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
writeJson(response, result.status, result.body);
|
|
120
|
+
}
|
|
121
|
+
async function readJson(request) {
|
|
122
|
+
const chunks = [];
|
|
123
|
+
let size = 0;
|
|
124
|
+
for await (const chunk of request) {
|
|
125
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
126
|
+
size += buffer.length;
|
|
127
|
+
if (size > 1024 * 1024) {
|
|
128
|
+
throw new Error('请求体过大');
|
|
129
|
+
}
|
|
130
|
+
chunks.push(buffer);
|
|
131
|
+
}
|
|
132
|
+
if (chunks.length === 0) {
|
|
133
|
+
return {};
|
|
134
|
+
}
|
|
135
|
+
try {
|
|
136
|
+
return JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
return {};
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
function writeJson(response, status, body) {
|
|
143
|
+
if (response.headersSent) {
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
response.writeHead(status, { 'content-type': 'application/json; charset=utf-8' });
|
|
147
|
+
response.end(JSON.stringify(body));
|
|
148
|
+
}
|
|
149
|
+
const INDEX_HTML = `<!doctype html>
|
|
150
|
+
<html lang="zh"><head><meta charset="utf-8"><title>Minko RAG</title>
|
|
151
|
+
<style>body{font-family:system-ui,sans-serif;max-width:760px;margin:40px auto;padding:0 16px}
|
|
152
|
+
input,button{font-size:16px;padding:8px}pre{background:#f4f4f5;padding:16px;white-space:pre-wrap}</style>
|
|
153
|
+
</head><body><h1>Minko RAG</h1>
|
|
154
|
+
<p>输入查询,调用本机 <code>/v1/query</code>。</p>
|
|
155
|
+
<form id="f"><input id="q" placeholder="例如:登录逻辑在哪" size="50"><button>查询</button></form>
|
|
156
|
+
<pre id="out"></pre>
|
|
157
|
+
<script>
|
|
158
|
+
document.getElementById('f').addEventListener('submit', async (event) => {
|
|
159
|
+
event.preventDefault();
|
|
160
|
+
const query = document.getElementById('q').value;
|
|
161
|
+
const out = document.getElementById('out');
|
|
162
|
+
out.textContent = '查询中…';
|
|
163
|
+
const response = await fetch('/v1/query', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ query, top_k: 6 }) });
|
|
164
|
+
const data = await response.json();
|
|
165
|
+
out.textContent = (data.results || []).map((hit, index) => (index + 1) + '. ' + hit.path + ':' + hit.start_line + '-' + hit.end_line + '\\n' + hit.text).join('\\n\\n') || data.error || '没有结果';
|
|
166
|
+
});
|
|
167
|
+
</script></body></html>`;
|
package/dist/rag/tool.js
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { WorkspaceIndex } from "./index.js";
|
|
3
|
+
import { resolveEmbedding } from "./embeddings.js";
|
|
4
|
+
const indexes = new Map();
|
|
5
|
+
const MAX_SNIPPET = 800;
|
|
6
|
+
const DEFAULT_REMOTE_TIMEOUT_MS = 10_000;
|
|
7
|
+
export function workspaceIndex(cwd, config) {
|
|
8
|
+
const root = path.resolve(cwd);
|
|
9
|
+
const resolved = resolveEmbedding(config, root);
|
|
10
|
+
const key = root + '|' + (resolved?.embedder.model ?? 'bm25');
|
|
11
|
+
let index = indexes.get(key);
|
|
12
|
+
if (!index) {
|
|
13
|
+
index = new WorkspaceIndex(root, {
|
|
14
|
+
embedder: resolved?.embedder,
|
|
15
|
+
embeddingCache: resolved?.cache,
|
|
16
|
+
maxEmbeddingChunks: resolved?.maxChunks,
|
|
17
|
+
});
|
|
18
|
+
indexes.set(key, index);
|
|
19
|
+
}
|
|
20
|
+
return index;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Ranks workspace chunks by relevance and returns file:line snippets.
|
|
24
|
+
* Prefers a configured remote RAG service, then falls back to the local
|
|
25
|
+
* BM25/embedding index.
|
|
26
|
+
*/
|
|
27
|
+
export const retrieveContextTool = {
|
|
28
|
+
readOnly: true,
|
|
29
|
+
spec: {
|
|
30
|
+
name: 'retrieve_context',
|
|
31
|
+
description: '在项目里检索与问题最相关的代码和文档片段,返回 file:line 与片段。适合问“这个功能在哪实现/怎么用”。',
|
|
32
|
+
parameters: {
|
|
33
|
+
type: 'object',
|
|
34
|
+
properties: {
|
|
35
|
+
query: { type: 'string', description: '自然语言或关键词' },
|
|
36
|
+
top_k: { type: 'number', description: '返回条数,默认 6,最多 20' },
|
|
37
|
+
refresh: { type: 'boolean', description: '为 true 时强制重建索引' },
|
|
38
|
+
},
|
|
39
|
+
required: ['query'],
|
|
40
|
+
additionalProperties: false,
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
async run(args, ctx) {
|
|
44
|
+
const query = typeof args.query === 'string' ? args.query.trim() : '';
|
|
45
|
+
if (query.length === 0) {
|
|
46
|
+
throw new Error('缺少 query 参数');
|
|
47
|
+
}
|
|
48
|
+
const topK = typeof args.top_k === 'number' && Number.isFinite(args.top_k)
|
|
49
|
+
? Math.floor(args.top_k)
|
|
50
|
+
: 6;
|
|
51
|
+
const remote = ctx.config?.rag?.remote;
|
|
52
|
+
let hits = [];
|
|
53
|
+
let note = '';
|
|
54
|
+
if (remote?.baseUrl) {
|
|
55
|
+
try {
|
|
56
|
+
hits = await remoteSearch(remote, query, topK);
|
|
57
|
+
note = '(远程 RAG)\n\n';
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
61
|
+
note = '(远程 RAG 不可用,已回退本地检索:' + reason + ')\n\n';
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (!remote?.baseUrl || note.startsWith('(远程 RAG 不可用')) {
|
|
65
|
+
const index = workspaceIndex(ctx.cwd, ctx.config);
|
|
66
|
+
await index.ensureFresh(args.refresh === true ? 0 : 30_000);
|
|
67
|
+
hits = await index.searchAsync(query, topK);
|
|
68
|
+
note += index.semanticError
|
|
69
|
+
? '(语义检索不可用,已回退关键词检索:' + index.semanticError + ')\n\n'
|
|
70
|
+
: index.semanticActive
|
|
71
|
+
? '(语义 + 关键词混合检索)\n\n'
|
|
72
|
+
: '';
|
|
73
|
+
}
|
|
74
|
+
if (hits.length === 0) {
|
|
75
|
+
return note + '没有找到相关内容。';
|
|
76
|
+
}
|
|
77
|
+
return note + hits.map((hit, position) => {
|
|
78
|
+
const body = hit.text.length > MAX_SNIPPET
|
|
79
|
+
? hit.text.slice(0, MAX_SNIPPET) + '\n…'
|
|
80
|
+
: hit.text;
|
|
81
|
+
const semantic = hit.semantic === undefined
|
|
82
|
+
? ''
|
|
83
|
+
: ' · 语义 ' + hit.semantic.toFixed(2);
|
|
84
|
+
return (position + 1) + '. ' + hit.path + ':' + hit.startLine + '-' + hit.endLine
|
|
85
|
+
+ '(相关度 ' + hit.score.toFixed(2) + semantic + ')\n```\n' + body + '\n```';
|
|
86
|
+
}).join('\n\n');
|
|
87
|
+
},
|
|
88
|
+
};
|
|
89
|
+
async function remoteSearch(options, query, fallbackTopK) {
|
|
90
|
+
const headers = { 'content-type': 'application/json' };
|
|
91
|
+
if (options.apiKey) {
|
|
92
|
+
headers.authorization = 'Bearer ' + options.apiKey;
|
|
93
|
+
}
|
|
94
|
+
const response = await fetch(remoteQueryUrl(options.baseUrl ?? ''), {
|
|
95
|
+
method: 'POST',
|
|
96
|
+
headers,
|
|
97
|
+
body: JSON.stringify({ query, top_k: options.topK ?? fallbackTopK }),
|
|
98
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_REMOTE_TIMEOUT_MS),
|
|
99
|
+
});
|
|
100
|
+
if (!response.ok) {
|
|
101
|
+
const body = (await response.text()).replace(/\s+/g, ' ').trim().slice(0, 200);
|
|
102
|
+
throw new Error('HTTP ' + response.status + (body.length > 0 ? ':' + body : ''));
|
|
103
|
+
}
|
|
104
|
+
const payload = await response.json();
|
|
105
|
+
const raw = Array.isArray(payload.results)
|
|
106
|
+
? payload.results
|
|
107
|
+
: Array.isArray(payload.hits)
|
|
108
|
+
? payload.hits
|
|
109
|
+
: [];
|
|
110
|
+
const hits = [];
|
|
111
|
+
for (const item of raw) {
|
|
112
|
+
if (item === null || typeof item !== 'object') {
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
const record = item;
|
|
116
|
+
const filePath = typeof record.path === 'string'
|
|
117
|
+
? record.path
|
|
118
|
+
: typeof record.file === 'string' ? record.file : '';
|
|
119
|
+
const text = typeof record.text === 'string'
|
|
120
|
+
? record.text
|
|
121
|
+
: typeof record.content === 'string'
|
|
122
|
+
? record.content
|
|
123
|
+
: typeof record.snippet === 'string' ? record.snippet : '';
|
|
124
|
+
if (filePath.length === 0 || text.length === 0) {
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
const startLine = toNumber(record.start_line ?? record.startLine, 1);
|
|
128
|
+
const endLine = toNumber(record.end_line ?? record.endLine, startLine);
|
|
129
|
+
const score = typeof record.score === 'number' && Number.isFinite(record.score)
|
|
130
|
+
? record.score
|
|
131
|
+
: 0;
|
|
132
|
+
hits.push({ path: filePath, startLine, endLine, text, score });
|
|
133
|
+
}
|
|
134
|
+
hits.sort((a, b) => b.score - a.score);
|
|
135
|
+
return hits.slice(0, 20);
|
|
136
|
+
}
|
|
137
|
+
function remoteQueryUrl(baseUrl) {
|
|
138
|
+
const base = baseUrl.replace(/\/+$/, '');
|
|
139
|
+
if (/\/v1$/i.test(base)) {
|
|
140
|
+
return base + '/query';
|
|
141
|
+
}
|
|
142
|
+
return base + '/v1/query';
|
|
143
|
+
}
|
|
144
|
+
function toNumber(value, fallback) {
|
|
145
|
+
return typeof value === 'number' && Number.isFinite(value)
|
|
146
|
+
? Math.max(1, Math.floor(value))
|
|
147
|
+
: fallback;
|
|
148
|
+
}
|