@takara-ai/miru-code 0.4.12 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +6 -1
- package/src/agents/claude.md +0 -27
- package/src/agents/copilot.md +0 -12
- package/src/agents/cursor.md +0 -26
- package/src/agents/gemini.md +0 -14
- package/src/agents/kiro.md +0 -6
- package/src/agents/opencode.md +0 -6
- package/src/agents.ts +37 -1
- package/src/cli-ui.ts +4 -1
- package/src/cli.ts +1 -9
- package/src/concurrency.ts +1 -5
- package/src/embeddings/openai.ts +60 -30
- package/src/git.ts +1 -1
- package/src/index/create.ts +24 -4
- package/src/index/entry-chunks.ts +71 -0
- package/src/index/vector-storage.ts +1 -1
- package/src/installer/hooks/search-guard.ts +3 -4
- package/src/installer/prompt.ts +118 -18
- package/src/installer/search-policy.ts +89 -29
- package/src/mcp/index-cache.ts +57 -24
- package/src/mcp/server.ts +31 -27
- package/src/ranking/boosting.ts +11 -0
- package/src/ranking/features.ts +11 -0
- package/src/ranking/location.ts +237 -0
- package/src/search.ts +4 -1
- package/src/setup.ts +1 -1
- package/src/snippet.ts +154 -0
- package/src/spinner.ts +1 -1
- package/src/utils.ts +63 -6
package/README.md
CHANGED
|
@@ -6,6 +6,12 @@ Find code by meaning, not grep. Miru returns the best **chunks** (path, lines, s
|
|
|
6
6
|
|
|
7
7
|
**Requires:** [Bun](https://bun.sh) 1.1+ · [Takara API key](https://takara.ai)
|
|
8
8
|
|
|
9
|
+
## Privacy and API usage
|
|
10
|
+
|
|
11
|
+
Miru sends **file contents** to the [Takara inference API](https://takara.ai) when building an index and when embedding search queries. Chunks from your repo are transmitted over HTTPS to generate embeddings. API usage may incur cost depending on your Takara plan.
|
|
12
|
+
|
|
13
|
+
If you index proprietary code, make sure that sending snippets to Takara's endpoint fits your security and compliance requirements. Use `MIRU_WORKSPACE_ROOT` to restrict MCP indexing to a single workspace directory.
|
|
14
|
+
|
|
9
15
|
## Install
|
|
10
16
|
|
|
11
17
|
```bash
|
|
@@ -143,6 +149,9 @@ const results = await index.search({ query: "BM25 tokenize", topK: 10 });
|
|
|
143
149
|
|----------|-------|
|
|
144
150
|
| `TAKARA_API_KEY` | Required |
|
|
145
151
|
| `MIRU_OPENAI_BASE_URL` | Default `https://infer.takara.ai/v1` |
|
|
152
|
+
| `MIRU_WORKSPACE_ROOT` | Restrict MCP local `repo` paths to this directory |
|
|
153
|
+
| `MIRU_MAX_INDEX_FILES` | Cap files indexed per operation |
|
|
154
|
+
| `MIRU_ALLOW_HTTP_GIT` | Set `1` to allow plain `http://` git clones |
|
|
146
155
|
| `MIRU_MCP_WATCH` | Set `0` to disable MCP file watch |
|
|
147
156
|
| `NO_COLOR` | Disable CLI colors |
|
|
148
157
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@takara-ai/miru-code",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Miru (見る) — hybrid code search for agents. Bun, TypeScript, Takara embeddings.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -50,6 +50,11 @@
|
|
|
50
50
|
"lint": "biome check .",
|
|
51
51
|
"lint:fix": "biome check --write .",
|
|
52
52
|
"prepublishOnly": "bun run typecheck && bun test",
|
|
53
|
+
"benchmark:ab": "bun run scripts/search-ab-local.ts",
|
|
54
|
+
"benchmark:ab-multi": "bun run scripts/search-ab-multi.ts",
|
|
55
|
+
"benchmark:recall": "bun run scripts/benchmark-recall-tokens.ts",
|
|
56
|
+
"benchmark:vs-grep": "bun run scripts/benchmark-miru-vs-grep.ts",
|
|
57
|
+
"benchmark:workflow": "bun run scripts/benchmark-workflow-tokens.ts",
|
|
53
58
|
"benchmark:quality": "bun run scripts/benchmark-quality.ts",
|
|
54
59
|
"benchmark:quantization": "bun run scripts/benchmark-quantization.ts"
|
|
55
60
|
},
|
package/src/agents/claude.md
CHANGED
|
@@ -3,30 +3,3 @@ name: miru-code
|
|
|
3
3
|
description: Code search via Miru MCP. Use search, expand, and find_related. DO NOT use Grep, Glob, or Read for exploration when Miru MCP is available.
|
|
4
4
|
tools: Bash, Read
|
|
5
5
|
---
|
|
6
|
-
|
|
7
|
-
When Miru MCP is available, use MCP `search`, `expand`, and `find_related` — not Grep, Glob, or Read-for-exploration.
|
|
8
|
-
|
|
9
|
-
| Task | Tool |
|
|
10
|
-
|------|------|
|
|
11
|
-
| How/where/what handles X? | Miru `search` (once) |
|
|
12
|
-
| Same file, more context | Miru `expand` |
|
|
13
|
-
| Similar implementations | Miru `find_related` |
|
|
14
|
-
| Exact literal confirmation | Grep |
|
|
15
|
-
| Edit a located file | Read |
|
|
16
|
-
|
|
17
|
-
### MCP workflow
|
|
18
|
-
|
|
19
|
-
1. Call `search` with `repo` set to the project root.
|
|
20
|
-
2. If context is cut off, call `expand` with the hit's `file_path` and `line`.
|
|
21
|
-
3. Use `find_related` for similar code in other files — not for same-file context.
|
|
22
|
-
4. Read files only to edit or when `expand` still lacks context.
|
|
23
|
-
|
|
24
|
-
### CLI fallback
|
|
25
|
-
|
|
26
|
-
```bash
|
|
27
|
-
miru search "authentication flow" .
|
|
28
|
-
miru expand src/auth.ts 42 .
|
|
29
|
-
miru find-related src/auth.ts 42 .
|
|
30
|
-
```
|
|
31
|
-
|
|
32
|
-
If `miru` is not on `$PATH`, use `bunx @takara-ai/miru-code`.
|
package/src/agents/copilot.md
CHANGED
|
@@ -2,15 +2,3 @@
|
|
|
2
2
|
name: miru-code
|
|
3
3
|
description: Code search via Miru MCP. Use search, expand, and find_related instead of grep_search or codebase_search when Miru is connected.
|
|
4
4
|
---
|
|
5
|
-
|
|
6
|
-
When Miru MCP is available, use `search`, `expand`, and `find_related` instead of `grep_search`, `codebase_search`, or broad file reads.
|
|
7
|
-
|
|
8
|
-
| Task | Tool |
|
|
9
|
-
|------|------|
|
|
10
|
-
| How/where/what handles X? | Miru `search` (once) |
|
|
11
|
-
| Same file, more context | Miru `expand` |
|
|
12
|
-
| Similar implementations | Miru `find_related` |
|
|
13
|
-
| Exact literal confirmation | grep_search |
|
|
14
|
-
| Edit a located file | read_file |
|
|
15
|
-
|
|
16
|
-
Pass project root as `repo` on every MCP call.
|
package/src/agents/cursor.md
CHANGED
|
@@ -2,29 +2,3 @@
|
|
|
2
2
|
name: miru-code
|
|
3
3
|
description: Code search via Miru MCP. Use search, expand, and find_related. DO NOT use Grep, Glob, or SemanticSearch when Miru MCP is available.
|
|
4
4
|
---
|
|
5
|
-
|
|
6
|
-
Use Miru MCP tools when connected:
|
|
7
|
-
|
|
8
|
-
- `search` — one call per question; always pass project root as `repo`
|
|
9
|
-
- `expand` — adjacent chunks in the same file when a hit looks truncated (`file_path` + `line`)
|
|
10
|
-
- `find_related` — similar code in other files after a search hit
|
|
11
|
-
|
|
12
|
-
Do not re-search with paraphrases after a good first hit. Do not read whole files for exploration.
|
|
13
|
-
|
|
14
|
-
| Task | Tool |
|
|
15
|
-
|------|------|
|
|
16
|
-
| How/where/what handles X? | Miru `search` (once) |
|
|
17
|
-
| Same file, more context | Miru `expand` |
|
|
18
|
-
| Similar implementations | Miru `find_related` |
|
|
19
|
-
| Exact literal confirmation | Grep |
|
|
20
|
-
| Edit a located file | Read |
|
|
21
|
-
|
|
22
|
-
CLI fallback when MCP is unavailable:
|
|
23
|
-
|
|
24
|
-
```bash
|
|
25
|
-
miru search "authentication flow" .
|
|
26
|
-
miru expand src/auth.ts 42 .
|
|
27
|
-
miru find-related src/auth.ts 42 .
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
If `miru` is not on `$PATH`, use `bunx @takara-ai/miru-code`.
|
package/src/agents/gemini.md
CHANGED
|
@@ -2,17 +2,3 @@
|
|
|
2
2
|
name: miru-code
|
|
3
3
|
description: Code search via Miru MCP. Use search, expand, and find_related instead of grep_search or read_file for exploration.
|
|
4
4
|
---
|
|
5
|
-
|
|
6
|
-
When Miru MCP is available, use `search`, `expand`, and `find_related` instead of `grep_search`, `glob`, or `read_file` for codebase exploration.
|
|
7
|
-
|
|
8
|
-
| Task | Tool |
|
|
9
|
-
|------|------|
|
|
10
|
-
| How/where/what handles X? | Miru `search` (once) |
|
|
11
|
-
| Same file, more context | Miru `expand` |
|
|
12
|
-
| Similar implementations | Miru `find_related` |
|
|
13
|
-
| Exact literal confirmation | grep_search |
|
|
14
|
-
| Edit a located file | read_file |
|
|
15
|
-
|
|
16
|
-
Pass project root as `repo` on every MCP call. Do not re-search with paraphrases after a good first hit.
|
|
17
|
-
|
|
18
|
-
CLI fallback: `miru search "query" .` or `bunx @takara-ai/miru-code search "query" .`
|
package/src/agents/kiro.md
CHANGED
|
@@ -2,9 +2,3 @@
|
|
|
2
2
|
name: miru-code
|
|
3
3
|
description: Code search via Miru MCP. Use search, expand, and find_related for codebase exploration instead of built-in grep or file search.
|
|
4
4
|
---
|
|
5
|
-
|
|
6
|
-
When Miru MCP is available, use `search`, `expand`, and `find_related` before any built-in search or broad file reads.
|
|
7
|
-
|
|
8
|
-
Pass project root as `repo`. Use `expand` for truncated hits in the same file. Use Grep only for exact literal confirmation.
|
|
9
|
-
|
|
10
|
-
CLI fallback: `miru search "query" .`
|
package/src/agents/opencode.md
CHANGED
|
@@ -3,9 +3,3 @@ name: miru-code
|
|
|
3
3
|
description: Code search via Miru MCP. Use search, expand, and find_related instead of grep or glob for exploration.
|
|
4
4
|
mode: subagent
|
|
5
5
|
---
|
|
6
|
-
|
|
7
|
-
When Miru MCP is available, use `search`, `expand`, and `find_related` instead of grep, glob, or read-for-exploration.
|
|
8
|
-
|
|
9
|
-
Pass project root as `repo` on every MCP call.
|
|
10
|
-
|
|
11
|
-
CLI fallback: `miru search "query" .`
|
package/src/agents.ts
CHANGED
|
@@ -1,17 +1,53 @@
|
|
|
1
1
|
import { mkdir, readFile } from "node:fs/promises";
|
|
2
2
|
import { dirname, join } from "node:path";
|
|
3
|
+
import { buildSubagentBody, type NativeToolNames } from "./installer/search-policy.ts";
|
|
3
4
|
|
|
4
5
|
export type AgentId = "claude" | "copilot" | "cursor" | "gemini" | "kiro" | "opencode";
|
|
5
6
|
|
|
6
7
|
const AGENTS_DIR = join(import.meta.dir, "agents");
|
|
7
8
|
|
|
9
|
+
const AGENT_NATIVE_TOOLS: Record<AgentId, NativeToolNames> = {
|
|
10
|
+
claude: {
|
|
11
|
+
explorationDenied: "Grep, Glob, SemanticSearch, or Read",
|
|
12
|
+
grep: "Grep",
|
|
13
|
+
read: "Read",
|
|
14
|
+
},
|
|
15
|
+
cursor: {
|
|
16
|
+
explorationDenied: "Grep, Glob, SemanticSearch, or Read",
|
|
17
|
+
grep: "Grep",
|
|
18
|
+
read: "Read",
|
|
19
|
+
},
|
|
20
|
+
copilot: {
|
|
21
|
+
explorationDenied: "grep_search, codebase_search, glob, or read_file",
|
|
22
|
+
grep: "grep_search",
|
|
23
|
+
read: "read_file",
|
|
24
|
+
},
|
|
25
|
+
gemini: {
|
|
26
|
+
explorationDenied: "grep_search, glob, or read_file",
|
|
27
|
+
grep: "grep_search",
|
|
28
|
+
read: "read_file",
|
|
29
|
+
},
|
|
30
|
+
kiro: {
|
|
31
|
+
explorationDenied: "built-in grep, file search, or broad file reads",
|
|
32
|
+
grep: "Grep",
|
|
33
|
+
read: "Read",
|
|
34
|
+
},
|
|
35
|
+
opencode: {
|
|
36
|
+
explorationDenied: "grep, glob, or read-for-exploration",
|
|
37
|
+
grep: "grep",
|
|
38
|
+
read: "read",
|
|
39
|
+
},
|
|
40
|
+
};
|
|
41
|
+
|
|
8
42
|
export function agentDestination(agent: AgentId): string {
|
|
9
43
|
const baseDir = agent === "copilot" ? ".github" : `.${agent}`;
|
|
10
44
|
return join(baseDir, "agents", "miru-code.md");
|
|
11
45
|
}
|
|
12
46
|
|
|
13
47
|
export async function loadAgentTemplate(agent: AgentId): Promise<string> {
|
|
14
|
-
|
|
48
|
+
const frontmatter = (await readFile(join(AGENTS_DIR, `${agent}.md`), "utf-8")).trim();
|
|
49
|
+
const body = buildSubagentBody(AGENT_NATIVE_TOOLS[agent]);
|
|
50
|
+
return `${frontmatter}\n\n${body}\n`;
|
|
15
51
|
}
|
|
16
52
|
|
|
17
53
|
export async function writeAgentFile(
|
package/src/cli-ui.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { SearchResult } from "./types.ts";
|
|
2
|
+
import { formatRelevanceScore } from "./utils.ts";
|
|
2
3
|
|
|
3
4
|
const ANSI = {
|
|
4
5
|
reset: "\x1b[0m",
|
|
@@ -145,6 +146,8 @@ export function formatSearchResultsPretty(query: string, results: SearchResult[]
|
|
|
145
146
|
lines.push(bold(label) + dim(` for `) + cyan(`"${query}"`));
|
|
146
147
|
lines.push("");
|
|
147
148
|
|
|
149
|
+
const maxScore = results.reduce((max, result) => Math.max(max, result?.score ?? 0), 0);
|
|
150
|
+
|
|
148
151
|
for (let i = 0; i < results.length; i++) {
|
|
149
152
|
const result = results[i];
|
|
150
153
|
if (!result) {
|
|
@@ -154,7 +157,7 @@ export function formatSearchResultsPretty(query: string, results: SearchResult[]
|
|
|
154
157
|
const location = `${chunk.file_path}:${chunk.start_line}-${chunk.end_line}`;
|
|
155
158
|
const lang = chunk.language ? dim(` ${chunk.language}`) : "";
|
|
156
159
|
const rank = dim(`[${i + 1}]`);
|
|
157
|
-
const scoreText = magenta(score
|
|
160
|
+
const scoreText = magenta(formatRelevanceScore(score, maxScore));
|
|
158
161
|
|
|
159
162
|
lines.push(`${rank} ${scoreText} ${bold(location)}${lang}`);
|
|
160
163
|
lines.push(dim("─".repeat(Math.min(PREVIEW_WIDTH, 52))));
|
package/src/cli.ts
CHANGED
|
@@ -397,15 +397,7 @@ async function runCli(argv: string[]): Promise<void> {
|
|
|
397
397
|
}
|
|
398
398
|
const line = Number(lineRaw);
|
|
399
399
|
const path = resolveSearchPath(sizedRest[2] ?? process.cwd());
|
|
400
|
-
|
|
401
|
-
await runFindRelated(path, filePath, line, topK, content, jsonFlag);
|
|
402
|
-
} catch (err) {
|
|
403
|
-
if (err instanceof RelatedChunkNotFoundError) {
|
|
404
|
-
fail(err.message);
|
|
405
|
-
process.exit(1);
|
|
406
|
-
}
|
|
407
|
-
throw err;
|
|
408
|
-
}
|
|
400
|
+
await runFindRelated(path, filePath, line, topK, content, jsonFlag);
|
|
409
401
|
return;
|
|
410
402
|
}
|
|
411
403
|
|
package/src/concurrency.ts
CHANGED
|
@@ -11,11 +11,7 @@ export function resolveCpuCount(): number {
|
|
|
11
11
|
|
|
12
12
|
/** Worker pool size: logical CPUs minus 2, minimum 1. Override with MIRU_CONCURRENCY. */
|
|
13
13
|
export function resolveWorkerConcurrency(): number {
|
|
14
|
-
const raw =
|
|
15
|
-
process.env.MIRU_CONCURRENCY ??
|
|
16
|
-
process.env.SEMBLE_CONCURRENCY ??
|
|
17
|
-
process.env.MIRU_WORKERS ??
|
|
18
|
-
process.env.SEMBLE_WORKERS;
|
|
14
|
+
const raw = process.env.MIRU_CONCURRENCY ?? process.env.MIRU_WORKERS;
|
|
19
15
|
if (raw) {
|
|
20
16
|
const n = Number(raw);
|
|
21
17
|
if (Number.isFinite(n) && n >= 1) {
|
package/src/embeddings/openai.ts
CHANGED
|
@@ -32,14 +32,14 @@ export interface EmbeddingBackend {
|
|
|
32
32
|
|
|
33
33
|
export function resolveEmbeddingModel(): string {
|
|
34
34
|
return envFirstString(
|
|
35
|
-
["MIRU_OPENAI_EMBEDDING_MODEL", "
|
|
35
|
+
["MIRU_OPENAI_EMBEDDING_MODEL", "OPENAI_EMBEDDING_MODEL"],
|
|
36
36
|
DEFAULT_MODEL,
|
|
37
37
|
);
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
export function resolveMaxEmbedChars(): number {
|
|
41
41
|
return (
|
|
42
|
-
envOptionalInt(["MIRU_MAX_EMBED_CHARS"
|
|
42
|
+
envOptionalInt(["MIRU_MAX_EMBED_CHARS"], 256) ??
|
|
43
43
|
DEFAULT_MAX_EMBED_CHARS
|
|
44
44
|
);
|
|
45
45
|
}
|
|
@@ -73,13 +73,26 @@ function stripLoneSurrogates(text: string): string {
|
|
|
73
73
|
|
|
74
74
|
export function sanitizeEmbeddingInput(text: string): string {
|
|
75
75
|
const out = stripLoneSurrogates(text);
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
const mode = process.env.MIRU_EMBED_ESCAPE_MODE ?? "preserve";
|
|
77
|
+
if (mode === "quad") {
|
|
78
|
+
return out.replaceAll("\\", "\\\\\\\\");
|
|
79
|
+
}
|
|
79
80
|
if (mode === "strip") {
|
|
80
81
|
return out.replaceAll("\\", "/");
|
|
81
82
|
}
|
|
82
|
-
return out
|
|
83
|
+
return out;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const TRANSIENT_EMBED_STATUSES = new Set([429, 500, 502, 503, 504]);
|
|
87
|
+
const MAX_TRANSIENT_EMBED_RETRIES = 4;
|
|
88
|
+
|
|
89
|
+
function isTransientEmbeddingError(err: unknown): boolean {
|
|
90
|
+
const status = (err as { status?: number }).status;
|
|
91
|
+
return status !== undefined && TRANSIENT_EMBED_STATUSES.has(status);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function transientEmbedBackoffMs(attempt: number): number {
|
|
95
|
+
return Math.min(250 * 2 ** attempt, 4000);
|
|
83
96
|
}
|
|
84
97
|
|
|
85
98
|
function isPayloadTooLargeError(err: unknown): boolean {
|
|
@@ -95,11 +108,7 @@ function isPayloadTooLargeError(err: unknown): boolean {
|
|
|
95
108
|
}
|
|
96
109
|
|
|
97
110
|
export function resolveEmbeddingDimensions(model?: string): number | undefined {
|
|
98
|
-
const fromEnv = envOptionalInt([
|
|
99
|
-
"MIRU_EMBEDDING_DIMENSIONS",
|
|
100
|
-
"SEMBLE_EMBEDDING_DIMENSIONS",
|
|
101
|
-
"OPENAI_EMBEDDING_DIMENSIONS",
|
|
102
|
-
]);
|
|
111
|
+
const fromEnv = envOptionalInt(["MIRU_EMBEDDING_DIMENSIONS", "OPENAI_EMBEDDING_DIMENSIONS"]);
|
|
103
112
|
if (fromEnv != null) {
|
|
104
113
|
return fromEnv;
|
|
105
114
|
}
|
|
@@ -108,17 +117,14 @@ export function resolveEmbeddingDimensions(model?: string): number | undefined {
|
|
|
108
117
|
|
|
109
118
|
export function resolveEmbeddingBatchSize(): number {
|
|
110
119
|
return (
|
|
111
|
-
envOptionalInt([
|
|
112
|
-
|
|
113
|
-
"SEMBLE_EMBEDDING_BATCH_SIZE",
|
|
114
|
-
"OPENAI_EMBEDDING_BATCH_SIZE",
|
|
115
|
-
]) ?? DEFAULT_BATCH_SIZE
|
|
120
|
+
envOptionalInt(["MIRU_EMBEDDING_BATCH_SIZE", "OPENAI_EMBEDDING_BATCH_SIZE"]) ??
|
|
121
|
+
DEFAULT_BATCH_SIZE
|
|
116
122
|
);
|
|
117
123
|
}
|
|
118
124
|
|
|
119
125
|
export function resolveEmbeddingBaseUrl(): string {
|
|
120
126
|
return envFirstString(
|
|
121
|
-
["MIRU_OPENAI_BASE_URL", "
|
|
127
|
+
["MIRU_OPENAI_BASE_URL", "OPENAI_BASE_URL"],
|
|
122
128
|
DEFAULT_BASE_URL,
|
|
123
129
|
).replace(/\/$/, "");
|
|
124
130
|
}
|
|
@@ -282,23 +288,42 @@ export class OpenAIEmbeddingBackend implements EmbeddingBackend {
|
|
|
282
288
|
return out;
|
|
283
289
|
}
|
|
284
290
|
|
|
291
|
+
private async requestEmbeddings(texts: string[]): Promise<EmbeddingResponse> {
|
|
292
|
+
let lastErr: unknown;
|
|
293
|
+
for (let attempt = 0; attempt <= MAX_TRANSIENT_EMBED_RETRIES; attempt++) {
|
|
294
|
+
try {
|
|
295
|
+
const started = performance.now();
|
|
296
|
+
const response = await this.client.createEmbeddings(
|
|
297
|
+
texts,
|
|
298
|
+
this.model,
|
|
299
|
+
this.requestedDimensions,
|
|
300
|
+
);
|
|
301
|
+
const elapsed = performance.now() - started;
|
|
302
|
+
this.stats.requests += 1;
|
|
303
|
+
this.stats.inputItems += texts.length;
|
|
304
|
+
this.stats.inputChars += texts.reduce((acc, text) => acc + text.length, 0);
|
|
305
|
+
this.stats.totalRttMs += elapsed;
|
|
306
|
+
this.stats.maxRttMs = Math.max(this.stats.maxRttMs, elapsed);
|
|
307
|
+
return response;
|
|
308
|
+
} catch (err: unknown) {
|
|
309
|
+
lastErr = err;
|
|
310
|
+
if (isTransientEmbeddingError(err) && attempt < MAX_TRANSIENT_EMBED_RETRIES) {
|
|
311
|
+
this.stats.retries += 1;
|
|
312
|
+
await Bun.sleep(transientEmbedBackoffMs(attempt));
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
throw err;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
throw lastErr;
|
|
319
|
+
}
|
|
320
|
+
|
|
285
321
|
private async embedBatchRawWithRetry(texts: string[]): Promise<Float32Array[]> {
|
|
286
322
|
if (texts.length === 0) {
|
|
287
323
|
return [];
|
|
288
324
|
}
|
|
289
325
|
try {
|
|
290
|
-
const
|
|
291
|
-
const response = await this.client.createEmbeddings(
|
|
292
|
-
texts,
|
|
293
|
-
this.model,
|
|
294
|
-
this.requestedDimensions,
|
|
295
|
-
);
|
|
296
|
-
const elapsed = performance.now() - started;
|
|
297
|
-
this.stats.requests += 1;
|
|
298
|
-
this.stats.inputItems += texts.length;
|
|
299
|
-
this.stats.inputChars += texts.reduce((acc, text) => acc + text.length, 0);
|
|
300
|
-
this.stats.totalRttMs += elapsed;
|
|
301
|
-
this.stats.maxRttMs = Math.max(this.stats.maxRttMs, elapsed);
|
|
326
|
+
const response = await this.requestEmbeddings(texts);
|
|
302
327
|
const vectors = vectorsFromResponse(response.data, texts.length);
|
|
303
328
|
for (const vec of vectors) {
|
|
304
329
|
if (this.requestedDimensions != null && vec.length !== this.requestedDimensions) {
|
|
@@ -341,7 +366,12 @@ export class OpenAIEmbeddingBackend implements EmbeddingBackend {
|
|
|
341
366
|
const rightText = text.slice(mid);
|
|
342
367
|
const left = await this.embedBatchRawWithRetry([leftText]);
|
|
343
368
|
const right = await this.embedBatchRawWithRetry([rightText]);
|
|
344
|
-
|
|
369
|
+
const parts = [left[0], right[0]].filter((vec): vec is Float32Array => vec !== undefined);
|
|
370
|
+
if (parts.length === 0) {
|
|
371
|
+
this.stats.errors += 1;
|
|
372
|
+
throw err;
|
|
373
|
+
}
|
|
374
|
+
return [poolWindowVectors(parts)];
|
|
345
375
|
}
|
|
346
376
|
this.stats.errors += 1;
|
|
347
377
|
throw err;
|
package/src/git.ts
CHANGED
|
@@ -7,7 +7,7 @@ const DEFAULT_CLONE_TIMEOUT_SEC = 60;
|
|
|
7
7
|
|
|
8
8
|
export async function cloneGitRepository(url: string, ref?: string | null): Promise<string> {
|
|
9
9
|
const timeoutSec =
|
|
10
|
-
envOptionalInt(["MIRU_CLONE_TIMEOUT"
|
|
10
|
+
envOptionalInt(["MIRU_CLONE_TIMEOUT"], 1) ?? DEFAULT_CLONE_TIMEOUT_SEC;
|
|
11
11
|
const dir = await mkdtemp(join(tmpdir(), "miru-git-"));
|
|
12
12
|
const args = ["clone", "--depth", "1"];
|
|
13
13
|
if (ref) {
|
package/src/index/create.ts
CHANGED
|
@@ -3,9 +3,11 @@ import { chunkSource } from "../chunking/chunking.ts";
|
|
|
3
3
|
import { resolveWorkerConcurrency } from "../concurrency.ts";
|
|
4
4
|
import type { EmbeddingBackend } from "../embeddings/openai.ts";
|
|
5
5
|
import { envOptionalInt } from "../env.ts";
|
|
6
|
+
import { searchImprovementsEnabled } from "../ranking/features.ts";
|
|
6
7
|
import { tokenize } from "../tokens.ts";
|
|
7
8
|
import type { Chunk, ContentType } from "../types.ts";
|
|
8
9
|
import { BM25Index } from "./bm25.ts";
|
|
10
|
+
import { loadRootEntryChunks } from "./entry-chunks.ts";
|
|
9
11
|
import { walkFiles } from "./file-walker.ts";
|
|
10
12
|
import { detectLanguage, getExtensions, getFileStatus, readFileText } from "./files.ts";
|
|
11
13
|
import type { SemanticIndex } from "./semantic-index.ts";
|
|
@@ -17,31 +19,36 @@ const DEFAULT_PIPELINE_EMBED_INFLIGHT = 8;
|
|
|
17
19
|
|
|
18
20
|
function resolvePipelineEmbedBatch(): number {
|
|
19
21
|
return (
|
|
20
|
-
envOptionalInt(["MIRU_PIPELINE_EMBED_BATCH"
|
|
22
|
+
envOptionalInt(["MIRU_PIPELINE_EMBED_BATCH"], 1) ??
|
|
21
23
|
DEFAULT_PIPELINE_EMBED_BATCH
|
|
22
24
|
);
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
function resolvePipelineEmbedInflight(): number {
|
|
26
28
|
return (
|
|
27
|
-
envOptionalInt(["MIRU_PIPELINE_EMBED_INFLIGHT"
|
|
29
|
+
envOptionalInt(["MIRU_PIPELINE_EMBED_INFLIGHT"], 1) ??
|
|
28
30
|
DEFAULT_PIPELINE_EMBED_INFLIGHT
|
|
29
31
|
);
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
function resolveMaxIndexFiles(): number | undefined {
|
|
35
|
+
return envOptionalInt(["MIRU_MAX_INDEX_FILES"], 1);
|
|
36
|
+
}
|
|
37
|
+
|
|
32
38
|
export async function createIndexFromPath(
|
|
33
39
|
path: string,
|
|
34
40
|
embeddings: EmbeddingBackend,
|
|
35
41
|
content: ContentType[] = ["code"],
|
|
36
42
|
displayRoot?: string,
|
|
37
43
|
): Promise<{ bm25: BM25Index; semantic: SemanticIndex; chunks: Chunk[] }> {
|
|
38
|
-
const profile = process.env.MIRU_PROFILE === "1"
|
|
44
|
+
const profile = process.env.MIRU_PROFILE === "1";
|
|
39
45
|
const started = performance.now();
|
|
40
46
|
let fileProcessMs = 0;
|
|
41
47
|
let embedBackpressureWaitMs = 0;
|
|
42
48
|
let fileEnumerated = 0;
|
|
43
49
|
let fileTasks = 0;
|
|
44
50
|
let emptyFileTasks = 0;
|
|
51
|
+
let fileErrors = 0;
|
|
45
52
|
|
|
46
53
|
if (profile && "resetStats" in embeddings && typeof embeddings.resetStats === "function") {
|
|
47
54
|
embeddings.resetStats();
|
|
@@ -53,6 +60,7 @@ export async function createIndexFromPath(
|
|
|
53
60
|
const fileConcurrency = resolveWorkerConcurrency();
|
|
54
61
|
const embedBatchSize = resolvePipelineEmbedBatch();
|
|
55
62
|
const maxEmbedInflight = resolvePipelineEmbedInflight();
|
|
63
|
+
const maxIndexFiles = resolveMaxIndexFiles();
|
|
56
64
|
const readyBySeq = new Map<number, Chunk[]>();
|
|
57
65
|
const pendingEmbedChunks: Chunk[] = [];
|
|
58
66
|
const emittedChunkBatches: Chunk[][] = [];
|
|
@@ -74,7 +82,7 @@ export async function createIndexFromPath(
|
|
|
74
82
|
const chunkPath = displayRoot ? relative(root, filePath).replace(/\\/g, "/") : filePath;
|
|
75
83
|
return chunkSource(source, chunkPath, language);
|
|
76
84
|
} catch {
|
|
77
|
-
|
|
85
|
+
fileErrors++;
|
|
78
86
|
return [];
|
|
79
87
|
} finally {
|
|
80
88
|
fileProcessMs += performance.now() - t;
|
|
@@ -134,6 +142,9 @@ export async function createIndexFromPath(
|
|
|
134
142
|
|
|
135
143
|
for await (const filePath of walkFiles(resolved, extensions)) {
|
|
136
144
|
fileEnumerated++;
|
|
145
|
+
if (maxIndexFiles != null && fileEnumerated > maxIndexFiles) {
|
|
146
|
+
throw new Error(`Index file budget exceeded: max ${maxIndexFiles} files per operation`);
|
|
147
|
+
}
|
|
137
148
|
while (runningFiles.size >= fileConcurrency) {
|
|
138
149
|
await Promise.race(runningFiles);
|
|
139
150
|
}
|
|
@@ -145,6 +156,14 @@ export async function createIndexFromPath(
|
|
|
145
156
|
await Promise.race(runningFiles);
|
|
146
157
|
}
|
|
147
158
|
|
|
159
|
+
if (searchImprovementsEnabled()) {
|
|
160
|
+
const entryChunks = await loadRootEntryChunks(resolved, displayRoot ? root : undefined);
|
|
161
|
+
if (entryChunks.length > 0) {
|
|
162
|
+
readyBySeq.set(fileSeq++, entryChunks);
|
|
163
|
+
flushReadySequentialChunks();
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
148
167
|
flushReadySequentialChunks();
|
|
149
168
|
await maybeScheduleEmbed(true);
|
|
150
169
|
await Promise.all(embedPromises);
|
|
@@ -174,6 +193,7 @@ export async function createIndexFromPath(
|
|
|
174
193
|
file_enumerated: fileEnumerated,
|
|
175
194
|
file_tasks: fileTasks,
|
|
176
195
|
empty_or_skipped_files: emptyFileTasks,
|
|
196
|
+
file_errors: fileErrors,
|
|
177
197
|
chunks: chunks.length,
|
|
178
198
|
vectors: vectors.length,
|
|
179
199
|
pipeline: {
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { join, relative } from "node:path";
|
|
2
|
+
import type { Chunk } from "../types.ts";
|
|
3
|
+
|
|
4
|
+
function indexedPath(absPath: string, repoRoot: string, displayRoot?: string): string {
|
|
5
|
+
if (displayRoot) {
|
|
6
|
+
return relative(displayRoot, absPath).replace(/\\/g, "/");
|
|
7
|
+
}
|
|
8
|
+
return absPath.replace(/\\/g, "/");
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
async function packageJsonEntryChunks(
|
|
12
|
+
absPath: string,
|
|
13
|
+
repoRoot: string,
|
|
14
|
+
displayRoot?: string,
|
|
15
|
+
): Promise<Chunk[]> {
|
|
16
|
+
let pkg: Record<string, unknown>;
|
|
17
|
+
try {
|
|
18
|
+
pkg = JSON.parse(await Bun.file(absPath).text()) as Record<string, unknown>;
|
|
19
|
+
} catch {
|
|
20
|
+
return [];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const entryLines = ["[package entry]"];
|
|
24
|
+
|
|
25
|
+
const bin = pkg.bin;
|
|
26
|
+
if (typeof bin === "string") {
|
|
27
|
+
entryLines.push(`bin: ${bin}`);
|
|
28
|
+
} else if (bin && typeof bin === "object") {
|
|
29
|
+
for (const [name, target] of Object.entries(bin)) {
|
|
30
|
+
entryLines.push(`bin ${name}: ${String(target)}`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if (typeof pkg.main === "string") {
|
|
35
|
+
entryLines.push(`main: ${pkg.main}`);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const types = pkg.types ?? pkg.typings;
|
|
39
|
+
if (typeof types === "string") {
|
|
40
|
+
entryLines.push(`types: ${types}`);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (entryLines.length <= 1) {
|
|
44
|
+
return [];
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const filePath = indexedPath(absPath, repoRoot, displayRoot);
|
|
48
|
+
const content = `${entryLines.join("\n")}\n`;
|
|
49
|
+
|
|
50
|
+
return [
|
|
51
|
+
{
|
|
52
|
+
content,
|
|
53
|
+
file_path: filePath,
|
|
54
|
+
start_line: 1,
|
|
55
|
+
end_line: entryLines.length,
|
|
56
|
+
language: "json",
|
|
57
|
+
},
|
|
58
|
+
];
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Synthetic chunks for root config entry points (bin/main) when indexing code repos. */
|
|
62
|
+
export async function loadRootEntryChunks(
|
|
63
|
+
repoRoot: string,
|
|
64
|
+
displayRoot?: string,
|
|
65
|
+
): Promise<Chunk[]> {
|
|
66
|
+
const pkgPath = join(repoRoot, "package.json");
|
|
67
|
+
if (!(await Bun.file(pkgPath).exists())) {
|
|
68
|
+
return [];
|
|
69
|
+
}
|
|
70
|
+
return packageJsonEntryChunks(pkgPath, repoRoot, displayRoot);
|
|
71
|
+
}
|
|
@@ -6,7 +6,7 @@ export type SemanticStorage = "int8" | "float32";
|
|
|
6
6
|
|
|
7
7
|
/** Default: int8 quantized vectors (~4x less RAM). Set MIRU_FLOAT_VECTORS=1 for float32. */
|
|
8
8
|
export function resolveSemanticStorage(): SemanticStorage {
|
|
9
|
-
return process.env.MIRU_FLOAT_VECTORS === "1"
|
|
9
|
+
return process.env.MIRU_FLOAT_VECTORS === "1"
|
|
10
10
|
? "float32"
|
|
11
11
|
: "int8";
|
|
12
12
|
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { SEARCH_GUARD_EXPAND_HINT } from "../search-policy.ts";
|
|
2
|
+
|
|
1
3
|
const REDIRECT_PREFIX =
|
|
2
4
|
"Use Miru MCP instead of built-in search tools (pass project root as `repo`). " +
|
|
3
5
|
"Grep/Glob/Shell are only for exact literal confirmation or non-code tasks.";
|
|
@@ -119,10 +121,7 @@ export function searchGuardBlockReason(
|
|
|
119
121
|
toolInput: Record<string, unknown>,
|
|
120
122
|
): string {
|
|
121
123
|
const query = suggestedMiruQuery(toolName, toolInput);
|
|
122
|
-
return
|
|
123
|
-
`${REDIRECT_PREFIX} Try Miru \`search\` with query "${query}". ` +
|
|
124
|
-
"If the hit looks truncated, call `expand` on its file_path + line — do not re-search or read the whole file."
|
|
125
|
-
);
|
|
124
|
+
return `${REDIRECT_PREFIX} Try Miru \`search\` with query "${query}". ${SEARCH_GUARD_EXPAND_HINT}`;
|
|
126
125
|
}
|
|
127
126
|
|
|
128
127
|
/** Map host-specific stdin JSON into a common evaluation shape. */
|