botholomew 0.11.6 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/chat/agent.ts +3 -4
- package/src/commands/context.ts +62 -28
- package/src/commands/tools.ts +0 -12
- package/src/tools/registry.ts +2 -4
- package/src/tools/search/fuse.ts +117 -0
- package/src/tools/search/index.ts +134 -0
- package/src/tools/search/regexp.ts +70 -0
- package/src/tools/search/semantic.ts +74 -62
- package/src/worker/prompt.ts +2 -2
- package/src/tools/search/grep.ts +0 -128
package/package.json
CHANGED
package/src/chat/agent.ts
CHANGED
|
@@ -42,8 +42,7 @@ const CHAT_TOOL_NAMES = new Set([
|
|
|
42
42
|
"context_read",
|
|
43
43
|
"context_write",
|
|
44
44
|
"context_edit",
|
|
45
|
-
"
|
|
46
|
-
"search_semantic",
|
|
45
|
+
"search",
|
|
47
46
|
"list_threads",
|
|
48
47
|
"view_thread",
|
|
49
48
|
"create_schedule",
|
|
@@ -134,14 +133,14 @@ Format your responses using Markdown. Use headings, bold, italic, lists, and cod
|
|
|
134
133
|
|
|
135
134
|
Workflow for any "look up / find / read" intent:
|
|
136
135
|
|
|
137
|
-
1. \`
|
|
136
|
+
1. \`search\` (hybrid regexp + semantic) or \`context_search\` (keyword), then \`context_read\` / \`context_tree\` to drill in.
|
|
138
137
|
2. If freshness matters, call \`context_info\` and check \`indexed_at\`. To re-pull a single stale item, use \`context_refresh\` rather than going to MCP for the whole document.
|
|
139
138
|
3. Only call \`mcp_exec\` for reads when the data is genuinely missing locally **or** must be real-time (e.g., "what's on my calendar right now").
|
|
140
139
|
|
|
141
140
|
Writes always go through MCP — sending an email, creating an issue, posting to Slack. Don't search context first for those.
|
|
142
141
|
|
|
143
142
|
Examples:
|
|
144
|
-
- "What does doc X say?" → \`
|
|
143
|
+
- "What does doc X say?" → \`search\` first.
|
|
145
144
|
- "Any new emails from Y?" → check the \`gmail\` drive first; only hit Gmail MCP if the freshest indexed item is too old for the question.
|
|
146
145
|
- "Send an email to Y" → MCP write directly; no context lookup.
|
|
147
146
|
|
package/src/commands/context.ts
CHANGED
|
@@ -14,7 +14,6 @@ import {
|
|
|
14
14
|
formatDriveRef,
|
|
15
15
|
parseDriveRef,
|
|
16
16
|
} from "../context/drives.ts";
|
|
17
|
-
import { embedSingle } from "../context/embedder.ts";
|
|
18
17
|
import { FetchFailureError, fetchUrl } from "../context/fetcher.ts";
|
|
19
18
|
import {
|
|
20
19
|
type PreparedIngestion,
|
|
@@ -36,14 +35,13 @@ import {
|
|
|
36
35
|
resolveContextItem,
|
|
37
36
|
upsertContextItem,
|
|
38
37
|
} from "../db/context.ts";
|
|
39
|
-
import { getEmbeddingsForItem
|
|
38
|
+
import { getEmbeddingsForItem } from "../db/embeddings.ts";
|
|
40
39
|
import { reembedMissingVectors } from "../db/reembed.ts";
|
|
41
40
|
import { createMcpxClient } from "../mcpx/client.ts";
|
|
41
|
+
import { searchTool } from "../tools/search/index.ts";
|
|
42
|
+
import type { ToolContext } from "../tools/tool.ts";
|
|
42
43
|
import { logger } from "../utils/logger.ts";
|
|
43
|
-
import {
|
|
44
|
-
registerContextToolSubcommands,
|
|
45
|
-
registerSearchToolSubcommands,
|
|
46
|
-
} from "./tools.ts";
|
|
44
|
+
import { registerContextToolSubcommands } from "./tools.ts";
|
|
47
45
|
import { withDb } from "./with-db.ts";
|
|
48
46
|
|
|
49
47
|
function fmtDate(d: Date): string {
|
|
@@ -513,46 +511,82 @@ export function registerContextCommand(program: Command) {
|
|
|
513
511
|
|
|
514
512
|
const search = ctx
|
|
515
513
|
.command("search")
|
|
516
|
-
.description("Search context entries")
|
|
517
|
-
.argument(
|
|
518
|
-
|
|
514
|
+
.description("Search context entries (hybrid regexp + semantic)")
|
|
515
|
+
.argument(
|
|
516
|
+
"[query]",
|
|
517
|
+
"natural-language query (semantic + BM25). Combine with --pattern for fused regexp + semantic ranking.",
|
|
518
|
+
)
|
|
519
|
+
.option("-k, --top-k <n>", "max results", Number.parseInt, 20)
|
|
520
|
+
.option(
|
|
521
|
+
"--pattern <regex>",
|
|
522
|
+
"regex pattern (regexp side). May be combined with [query] to fuse signals.",
|
|
523
|
+
)
|
|
524
|
+
.option("--drive <drive>", "restrict to a single drive")
|
|
525
|
+
.option("--path <path>", "directory prefix within drive (requires --drive)")
|
|
526
|
+
.option("--glob <glob>", "filter results to files whose basename matches")
|
|
527
|
+
.option("--ignore-case", "case-insensitive regex")
|
|
528
|
+
.option(
|
|
529
|
+
"--context <n>",
|
|
530
|
+
"context lines around each regexp hit",
|
|
531
|
+
Number.parseInt,
|
|
532
|
+
)
|
|
519
533
|
.action((query, opts) =>
|
|
520
534
|
withDb(program, async (conn, dir) => {
|
|
521
|
-
if (!query) {
|
|
535
|
+
if (!query && !opts.pattern) {
|
|
522
536
|
search.help();
|
|
523
537
|
return;
|
|
524
538
|
}
|
|
525
539
|
const config = await loadConfig(dir);
|
|
526
|
-
const
|
|
527
|
-
|
|
540
|
+
const toolCtx: ToolContext = {
|
|
541
|
+
conn,
|
|
542
|
+
dbPath: getDbPath(dir),
|
|
543
|
+
projectDir: dir,
|
|
544
|
+
config,
|
|
545
|
+
mcpxClient: null,
|
|
546
|
+
};
|
|
547
|
+
const result = await searchTool.execute(
|
|
548
|
+
{
|
|
549
|
+
query,
|
|
550
|
+
pattern: opts.pattern,
|
|
551
|
+
drive: opts.drive,
|
|
552
|
+
path: opts.path,
|
|
553
|
+
glob: opts.glob,
|
|
554
|
+
ignore_case: opts.ignoreCase,
|
|
555
|
+
context: opts.context,
|
|
556
|
+
max_results: opts.topK,
|
|
557
|
+
},
|
|
558
|
+
toolCtx,
|
|
559
|
+
);
|
|
528
560
|
|
|
529
|
-
if (
|
|
561
|
+
if (result.is_error) {
|
|
562
|
+
logger.error(result.message ?? "Search failed");
|
|
563
|
+
process.exit(1);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
if (result.matches.length === 0) {
|
|
530
567
|
logger.dim("No results found.");
|
|
531
568
|
return;
|
|
532
569
|
}
|
|
533
570
|
|
|
534
|
-
for (const [i,
|
|
535
|
-
const
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
571
|
+
for (const [i, m] of result.matches.entries()) {
|
|
572
|
+
const tagColor =
|
|
573
|
+
m.match_type === "both"
|
|
574
|
+
? ansis.green
|
|
575
|
+
: m.match_type === "regexp"
|
|
576
|
+
? ansis.yellow
|
|
577
|
+
: ansis.cyan;
|
|
578
|
+
const tag = tagColor(`[${m.match_type}]`);
|
|
579
|
+
const location = m.line != null ? `${m.ref}:${m.line}` : m.ref;
|
|
543
580
|
console.log(
|
|
544
|
-
|
|
581
|
+
`${ansis.bold(`${i + 1}.`)} ${tag} ${ansis.cyan(location)} ${ansis.dim(`score=${m.score.toFixed(4)}`)}`,
|
|
545
582
|
);
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
console.log(` ${snippet}...`);
|
|
549
|
-
}
|
|
583
|
+
const snippet = m.content.slice(0, 200).replace(/\n/g, " ");
|
|
584
|
+
if (snippet) console.log(` ${snippet}`);
|
|
550
585
|
console.log("");
|
|
551
586
|
}
|
|
552
587
|
}),
|
|
553
588
|
);
|
|
554
589
|
|
|
555
|
-
registerSearchToolSubcommands(search);
|
|
556
590
|
ctx
|
|
557
591
|
.command("delete <ref>")
|
|
558
592
|
.description("Delete a context entry (UUID or drive:/path)")
|
package/src/commands/tools.ts
CHANGED
|
@@ -37,16 +37,6 @@ export function registerContextToolSubcommands(parent: Command) {
|
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
/**
|
|
41
|
-
* Register search tool subcommands (grep, semantic) onto an
|
|
42
|
-
* existing Commander command (e.g. the "context search" group).
|
|
43
|
-
*/
|
|
44
|
-
export function registerSearchToolSubcommands(parent: Command) {
|
|
45
|
-
for (const tool of getToolsByGroup("search")) {
|
|
46
|
-
registerToolAsCLI(parent, tool);
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
|
|
50
40
|
/** Derive CLI subcommand name from tool name: "context_read" → "read", "context_create_dir" → "create-dir" */
|
|
51
41
|
function deriveSubName(toolName: string): string {
|
|
52
42
|
return toolName.replace(/^[^_]+_/, "").replace(/_/g, "-");
|
|
@@ -341,8 +331,6 @@ function isPositionalArg(key: string, toolName: string): boolean {
|
|
|
341
331
|
context_exists: ["path"],
|
|
342
332
|
context_count_lines: ["path"],
|
|
343
333
|
context_search: ["query"],
|
|
344
|
-
search_grep: ["pattern"],
|
|
345
|
-
search_semantic: ["query"],
|
|
346
334
|
};
|
|
347
335
|
return positionalKeys[toolName]?.includes(key) ?? false;
|
|
348
336
|
}
|
package/src/tools/registry.ts
CHANGED
|
@@ -31,8 +31,7 @@ import { mcpSearchTool } from "./mcp/search.ts";
|
|
|
31
31
|
import { createScheduleTool } from "./schedule/create.ts";
|
|
32
32
|
import { listSchedulesTool } from "./schedule/list.ts";
|
|
33
33
|
// Search tools
|
|
34
|
-
import {
|
|
35
|
-
import { searchSemanticTool } from "./search/semantic.ts";
|
|
34
|
+
import { searchTool } from "./search/index.ts";
|
|
36
35
|
// Skill tools
|
|
37
36
|
import { skillDeleteTool } from "./skill/delete.ts";
|
|
38
37
|
import { skillEditTool } from "./skill/edit.ts";
|
|
@@ -96,8 +95,7 @@ export function registerAllTools(): void {
|
|
|
96
95
|
registerTool(listSchedulesTool);
|
|
97
96
|
|
|
98
97
|
// Search
|
|
99
|
-
registerTool(
|
|
100
|
-
registerTool(searchSemanticTool);
|
|
98
|
+
registerTool(searchTool);
|
|
101
99
|
|
|
102
100
|
// Skill
|
|
103
101
|
registerTool(skillListTool);
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import type { RegexpHit } from "./regexp.ts";
|
|
2
|
+
import type { SemanticHit } from "./semantic.ts";
|
|
3
|
+
|
|
4
|
+
export interface FusedMatch {
|
|
5
|
+
ref: string;
|
|
6
|
+
drive: string;
|
|
7
|
+
path: string;
|
|
8
|
+
line: number | null;
|
|
9
|
+
content: string;
|
|
10
|
+
context_lines: string[];
|
|
11
|
+
match_type: "regexp" | "semantic" | "both";
|
|
12
|
+
semantic_score: number | null;
|
|
13
|
+
score: number;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const SNIPPET_MAX = 300;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Reciprocal rank fusion of regexp line hits and semantic chunk hits.
|
|
20
|
+
*
|
|
21
|
+
* Each regexp hit becomes its own row. If the file (drive + path) also has a
|
|
22
|
+
* semantic hit, the regexp row picks up that semantic side's RRF contribution
|
|
23
|
+
* and is tagged `match_type: "both"` — exact-line + semantic agreement is
|
|
24
|
+
* the strongest signal.
|
|
25
|
+
*
|
|
26
|
+
* Semantic hits are emitted as their own rows only for paths with no regexp
|
|
27
|
+
* hit; otherwise the regexp row already represents that file (and is more
|
|
28
|
+
* locatable). This keeps the result list focused without losing pure
|
|
29
|
+
* semantic matches in files the regexp didn't touch.
|
|
30
|
+
*/
|
|
31
|
+
export function fuseRRF(
|
|
32
|
+
regexpHits: RegexpHit[],
|
|
33
|
+
semanticHits: SemanticHit[],
|
|
34
|
+
options: { k?: number; limit: number },
|
|
35
|
+
): FusedMatch[] {
|
|
36
|
+
const k = options.k ?? 60;
|
|
37
|
+
|
|
38
|
+
const bestSemByPath = new Map<
|
|
39
|
+
string,
|
|
40
|
+
{ rank: number; score: number; hit: SemanticHit }
|
|
41
|
+
>();
|
|
42
|
+
for (let i = 0; i < semanticHits.length; i++) {
|
|
43
|
+
const hit = semanticHits[i];
|
|
44
|
+
if (!hit) continue;
|
|
45
|
+
const key = pathKey(hit.drive, hit.path);
|
|
46
|
+
if (key == null) continue;
|
|
47
|
+
const existing = bestSemByPath.get(key);
|
|
48
|
+
if (!existing || i < existing.rank) {
|
|
49
|
+
bestSemByPath.set(key, { rank: i, score: hit.score, hit });
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const regexpPaths = new Set<string>();
|
|
54
|
+
for (const hit of regexpHits) {
|
|
55
|
+
regexpPaths.add(pathKey(hit.drive, hit.path) ?? "");
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const fused: FusedMatch[] = [];
|
|
59
|
+
|
|
60
|
+
for (let i = 0; i < regexpHits.length; i++) {
|
|
61
|
+
const rx = regexpHits[i];
|
|
62
|
+
if (!rx) continue;
|
|
63
|
+
const key = pathKey(rx.drive, rx.path) ?? "";
|
|
64
|
+
const sem = bestSemByPath.get(key);
|
|
65
|
+
let score = 1 / (k + i + 1);
|
|
66
|
+
let matchType: FusedMatch["match_type"] = "regexp";
|
|
67
|
+
let semanticScore: number | null = null;
|
|
68
|
+
if (sem) {
|
|
69
|
+
score += 1 / (k + sem.rank + 1);
|
|
70
|
+
matchType = "both";
|
|
71
|
+
semanticScore = round(sem.score);
|
|
72
|
+
}
|
|
73
|
+
fused.push({
|
|
74
|
+
ref: rx.ref,
|
|
75
|
+
drive: rx.drive,
|
|
76
|
+
path: rx.path,
|
|
77
|
+
line: rx.line,
|
|
78
|
+
content: rx.content,
|
|
79
|
+
context_lines: rx.context_lines,
|
|
80
|
+
match_type: matchType,
|
|
81
|
+
semantic_score: semanticScore,
|
|
82
|
+
score: round(score),
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
for (let i = 0; i < semanticHits.length; i++) {
|
|
87
|
+
const sem = semanticHits[i];
|
|
88
|
+
if (!sem) continue;
|
|
89
|
+
const key = pathKey(sem.drive, sem.path);
|
|
90
|
+
if (key == null) continue;
|
|
91
|
+
if (regexpPaths.has(key)) continue;
|
|
92
|
+
const score = 1 / (k + i + 1);
|
|
93
|
+
fused.push({
|
|
94
|
+
ref: sem.ref,
|
|
95
|
+
drive: sem.drive ?? "",
|
|
96
|
+
path: sem.path ?? "",
|
|
97
|
+
line: null,
|
|
98
|
+
content: sem.chunk_content.slice(0, SNIPPET_MAX),
|
|
99
|
+
context_lines: [],
|
|
100
|
+
match_type: "semantic",
|
|
101
|
+
semantic_score: round(sem.score),
|
|
102
|
+
score: round(score),
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
fused.sort((a, b) => b.score - a.score);
|
|
107
|
+
return fused.slice(0, options.limit);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function pathKey(drive: string | null, path: string | null): string | null {
|
|
111
|
+
if (!drive || !path) return null;
|
|
112
|
+
return `${drive}:${path}`;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function round(n: number): number {
|
|
116
|
+
return Math.round(n * 10000) / 10000;
|
|
117
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import {
|
|
3
|
+
listContextItems,
|
|
4
|
+
listContextItemsByPrefix,
|
|
5
|
+
} from "../../db/context.ts";
|
|
6
|
+
import type { ToolDefinition } from "../tool.ts";
|
|
7
|
+
import { fuseRRF } from "./fuse.ts";
|
|
8
|
+
import { runRegexp } from "./regexp.ts";
|
|
9
|
+
import { runSemantic } from "./semantic.ts";
|
|
10
|
+
|
|
11
|
+
const MatchSchema = z.object({
|
|
12
|
+
ref: z.string(),
|
|
13
|
+
drive: z.string(),
|
|
14
|
+
path: z.string(),
|
|
15
|
+
line: z.number().nullable(),
|
|
16
|
+
content: z.string(),
|
|
17
|
+
context_lines: z.array(z.string()),
|
|
18
|
+
match_type: z.enum(["regexp", "semantic", "both"]),
|
|
19
|
+
semantic_score: z.number().nullable(),
|
|
20
|
+
score: z.number(),
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
const inputSchema = z.object({
|
|
24
|
+
query: z
|
|
25
|
+
.string()
|
|
26
|
+
.optional()
|
|
27
|
+
.describe(
|
|
28
|
+
"Natural-language query for semantic + keyword (BM25) hybrid search. Provide alongside `pattern` for the strongest signal — chunks matched by both methods are boosted via reciprocal rank fusion.",
|
|
29
|
+
),
|
|
30
|
+
pattern: z
|
|
31
|
+
.string()
|
|
32
|
+
.optional()
|
|
33
|
+
.describe("Regex pattern for exact text search across context contents."),
|
|
34
|
+
drive: z
|
|
35
|
+
.string()
|
|
36
|
+
.optional()
|
|
37
|
+
.describe(
|
|
38
|
+
"Restrict to a single drive (applies to both `query` and `pattern`).",
|
|
39
|
+
),
|
|
40
|
+
path: z
|
|
41
|
+
.string()
|
|
42
|
+
.optional()
|
|
43
|
+
.describe("Directory prefix within the drive. Requires `drive`."),
|
|
44
|
+
glob: z
|
|
45
|
+
.string()
|
|
46
|
+
.optional()
|
|
47
|
+
.describe("Filter results to files whose basename matches this glob."),
|
|
48
|
+
ignore_case: z
|
|
49
|
+
.boolean()
|
|
50
|
+
.optional()
|
|
51
|
+
.describe("Case-insensitive regex (only affects `pattern`)."),
|
|
52
|
+
context: z
|
|
53
|
+
.number()
|
|
54
|
+
.optional()
|
|
55
|
+
.describe(
|
|
56
|
+
"Lines of surrounding context to include for each regex hit (only affects `pattern`).",
|
|
57
|
+
),
|
|
58
|
+
max_results: z
|
|
59
|
+
.number()
|
|
60
|
+
.optional()
|
|
61
|
+
.describe("Maximum number of fused results to return (default 20)."),
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
const outputSchema = z.object({
|
|
65
|
+
matches: z.array(MatchSchema),
|
|
66
|
+
is_error: z.boolean(),
|
|
67
|
+
error_type: z.string().optional(),
|
|
68
|
+
message: z.string().optional(),
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
export const searchTool = {
|
|
72
|
+
name: "search",
|
|
73
|
+
description:
|
|
74
|
+
"[[ bash equivalent command: grep -r ]] Hybrid search over indexed context. At least one of `query` (natural language → semantic + BM25) or `pattern` (regex over file contents) is required. Pass both for the strongest signal: results matched by both methods float to the top via reciprocal rank fusion. Scoping (`drive`, `path`, `glob`) applies to both sides.",
|
|
75
|
+
group: "search",
|
|
76
|
+
inputSchema,
|
|
77
|
+
outputSchema,
|
|
78
|
+
execute: async (input, ctx) => {
|
|
79
|
+
if (!input.query && !input.pattern) {
|
|
80
|
+
return {
|
|
81
|
+
matches: [],
|
|
82
|
+
is_error: true,
|
|
83
|
+
error_type: "invalid_arguments",
|
|
84
|
+
message:
|
|
85
|
+
"Provide at least one of `query` (natural language) or `pattern` (regex). Pass both to fuse semantic and exact-match signals.",
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
if (input.path && !input.drive) {
|
|
89
|
+
return {
|
|
90
|
+
matches: [],
|
|
91
|
+
is_error: true,
|
|
92
|
+
error_type: "invalid_arguments",
|
|
93
|
+
message:
|
|
94
|
+
"`path` requires `drive` — call context_list_drives to see which drives exist, then pass `drive` alongside `path`.",
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const limit = input.max_results ?? 20;
|
|
99
|
+
|
|
100
|
+
const regexpHits = input.pattern
|
|
101
|
+
? runRegexp(
|
|
102
|
+
input.drive
|
|
103
|
+
? await listContextItemsByPrefix(
|
|
104
|
+
ctx.conn,
|
|
105
|
+
input.drive,
|
|
106
|
+
input.path ?? "/",
|
|
107
|
+
{ recursive: true },
|
|
108
|
+
)
|
|
109
|
+
: await listContextItems(ctx.conn),
|
|
110
|
+
{
|
|
111
|
+
pattern: input.pattern,
|
|
112
|
+
glob: input.glob,
|
|
113
|
+
ignore_case: input.ignore_case,
|
|
114
|
+
context: input.context,
|
|
115
|
+
max_results: 100,
|
|
116
|
+
},
|
|
117
|
+
)
|
|
118
|
+
: [];
|
|
119
|
+
|
|
120
|
+
const semanticHits = input.query
|
|
121
|
+
? await runSemantic(ctx, {
|
|
122
|
+
query: input.query,
|
|
123
|
+
drive: input.drive,
|
|
124
|
+
path: input.path,
|
|
125
|
+
glob: input.glob,
|
|
126
|
+
limit: 100,
|
|
127
|
+
})
|
|
128
|
+
: [];
|
|
129
|
+
|
|
130
|
+
const matches = fuseRRF(regexpHits, semanticHits, { limit });
|
|
131
|
+
|
|
132
|
+
return { matches, is_error: false };
|
|
133
|
+
},
|
|
134
|
+
} satisfies ToolDefinition<typeof inputSchema, typeof outputSchema>;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { formatDriveRef } from "../../context/drives.ts";
|
|
2
|
+
import type { ContextItem } from "../../db/context.ts";
|
|
3
|
+
|
|
4
|
+
export interface RegexpHit {
|
|
5
|
+
ref: string;
|
|
6
|
+
drive: string;
|
|
7
|
+
path: string;
|
|
8
|
+
line: number;
|
|
9
|
+
content: string;
|
|
10
|
+
context_lines: string[];
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface RegexpOptions {
|
|
14
|
+
pattern: string;
|
|
15
|
+
glob?: string;
|
|
16
|
+
ignore_case?: boolean;
|
|
17
|
+
context?: number;
|
|
18
|
+
max_results?: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function runRegexp(
|
|
22
|
+
items: ContextItem[],
|
|
23
|
+
options: RegexpOptions,
|
|
24
|
+
): RegexpHit[] {
|
|
25
|
+
const flags = options.ignore_case ? "gi" : "g";
|
|
26
|
+
const regex = new RegExp(options.pattern, flags);
|
|
27
|
+
const globRegex = options.glob ? globToRegex(options.glob) : null;
|
|
28
|
+
const contextLines = options.context ?? 0;
|
|
29
|
+
const maxResults = options.max_results ?? 100;
|
|
30
|
+
|
|
31
|
+
const hits: RegexpHit[] = [];
|
|
32
|
+
|
|
33
|
+
for (const item of items) {
|
|
34
|
+
if (item.content == null) continue;
|
|
35
|
+
|
|
36
|
+
if (globRegex) {
|
|
37
|
+
const filename = item.path.split("/").pop() ?? "";
|
|
38
|
+
if (!globRegex.test(filename)) continue;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const lines = item.content.split("\n");
|
|
42
|
+
for (let i = 0; i < lines.length; i++) {
|
|
43
|
+
regex.lastIndex = 0;
|
|
44
|
+
const line = lines[i];
|
|
45
|
+
if (line !== undefined && regex.test(line)) {
|
|
46
|
+
const start = Math.max(0, i - contextLines);
|
|
47
|
+
const end = Math.min(lines.length, i + contextLines + 1);
|
|
48
|
+
hits.push({
|
|
49
|
+
ref: formatDriveRef(item),
|
|
50
|
+
drive: item.drive,
|
|
51
|
+
path: item.path,
|
|
52
|
+
line: i + 1,
|
|
53
|
+
content: line,
|
|
54
|
+
context_lines: lines.slice(start, end),
|
|
55
|
+
});
|
|
56
|
+
if (hits.length >= maxResults) return hits;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return hits;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function globToRegex(glob: string): RegExp {
|
|
65
|
+
const escaped = glob
|
|
66
|
+
.replace(/[.+^${}()|[\]\\]/g, "\\$&")
|
|
67
|
+
.replace(/\*/g, ".*")
|
|
68
|
+
.replace(/\?/g, ".");
|
|
69
|
+
return new RegExp(`^${escaped}$`, "i");
|
|
70
|
+
}
|
|
@@ -1,69 +1,81 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
1
|
import { formatDriveRef } from "../../context/drives.ts";
|
|
3
2
|
import { embedSingle } from "../../context/embedder.ts";
|
|
4
|
-
import { hybridSearch } from "../../db/embeddings.ts";
|
|
5
|
-
import type {
|
|
3
|
+
import { type HybridSearchResult, hybridSearch } from "../../db/embeddings.ts";
|
|
4
|
+
import type { ToolContext } from "../tool.ts";
|
|
5
|
+
import { globToRegex } from "./regexp.ts";
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
.describe("Minimum similarity score (0-1) to include in results"),
|
|
18
|
-
});
|
|
7
|
+
export interface SemanticHit {
|
|
8
|
+
ref: string;
|
|
9
|
+
drive: string | null;
|
|
10
|
+
path: string | null;
|
|
11
|
+
context_item_id: string;
|
|
12
|
+
chunk_index: number;
|
|
13
|
+
title: string;
|
|
14
|
+
chunk_content: string;
|
|
15
|
+
score: number;
|
|
16
|
+
}
|
|
19
17
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
}),
|
|
28
|
-
),
|
|
29
|
-
is_error: z.boolean(),
|
|
30
|
-
});
|
|
18
|
+
export interface SemanticOptions {
|
|
19
|
+
query: string;
|
|
20
|
+
drive?: string;
|
|
21
|
+
path?: string;
|
|
22
|
+
glob?: string;
|
|
23
|
+
limit?: number;
|
|
24
|
+
}
|
|
31
25
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
26
|
+
/**
|
|
27
|
+
* Run the embedding + hybrid-search pipeline. Scoping (`drive` / `path` /
|
|
28
|
+
* `glob`) is applied as a *post-filter* on results so the caller gets
|
|
29
|
+
* consistent behavior whether they used the regex side, the semantic side,
|
|
30
|
+
* or both.
|
|
31
|
+
*/
|
|
32
|
+
export async function runSemantic(
|
|
33
|
+
ctx: ToolContext,
|
|
34
|
+
options: SemanticOptions,
|
|
35
|
+
): Promise<SemanticHit[]> {
|
|
36
|
+
const queryVec = await embedSingle(options.query, ctx.config);
|
|
37
|
+
const results = await hybridSearch(
|
|
38
|
+
ctx.conn,
|
|
39
|
+
options.query,
|
|
40
|
+
queryVec,
|
|
41
|
+
options.limit ?? 100,
|
|
42
|
+
);
|
|
47
43
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
threshold !== undefined
|
|
51
|
-
? results.filter((r) => r.score >= threshold)
|
|
52
|
-
: results;
|
|
44
|
+
return results.filter((r) => matchesScope(r, options)).map(toHit);
|
|
45
|
+
}
|
|
53
46
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
47
|
+
function matchesScope(
|
|
48
|
+
result: HybridSearchResult,
|
|
49
|
+
options: SemanticOptions,
|
|
50
|
+
): boolean {
|
|
51
|
+
if (options.drive && result.drive !== options.drive) return false;
|
|
52
|
+
if (options.path && result.path) {
|
|
53
|
+
const prefix = options.path.endsWith("/")
|
|
54
|
+
? options.path
|
|
55
|
+
: `${options.path}/`;
|
|
56
|
+
if (result.path !== options.path && !result.path.startsWith(prefix)) {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (options.glob && result.path) {
|
|
61
|
+
const filename = result.path.split("/").pop() ?? "";
|
|
62
|
+
if (!globToRegex(options.glob).test(filename)) return false;
|
|
63
|
+
}
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function toHit(r: HybridSearchResult): SemanticHit {
|
|
68
|
+
return {
|
|
69
|
+
ref:
|
|
70
|
+
r.drive && r.path
|
|
71
|
+
? formatDriveRef({ drive: r.drive, path: r.path })
|
|
72
|
+
: r.context_item_id,
|
|
73
|
+
drive: r.drive,
|
|
74
|
+
path: r.path,
|
|
75
|
+
context_item_id: r.context_item_id,
|
|
76
|
+
chunk_index: r.chunk_index,
|
|
77
|
+
title: r.title,
|
|
78
|
+
chunk_content: r.chunk_content ?? "",
|
|
79
|
+
score: r.score,
|
|
80
|
+
};
|
|
81
|
+
}
|
package/src/worker/prompt.ts
CHANGED
|
@@ -145,14 +145,14 @@ When calling complete_task, write a summary that captures your key findings, dec
|
|
|
145
145
|
|
|
146
146
|
Workflow for any "look up / find / read" intent:
|
|
147
147
|
|
|
148
|
-
1. \`
|
|
148
|
+
1. \`search\` (hybrid regexp + semantic) or \`context_search\` (keyword), then \`context_read\` / \`context_tree\` to drill in.
|
|
149
149
|
2. If freshness matters, call \`context_info\` and check \`indexed_at\`. To re-pull a single stale item, use \`context_refresh\` rather than going to MCP for the whole document.
|
|
150
150
|
3. Only call \`mcp_exec\` for reads when the data is genuinely missing locally **or** must be real-time (e.g., "what's on my calendar right now").
|
|
151
151
|
|
|
152
152
|
Writes always go through MCP — sending an email, creating an issue, posting to Slack. Don't search context first for those.
|
|
153
153
|
|
|
154
154
|
Examples:
|
|
155
|
-
- "What does doc X say?" → \`
|
|
155
|
+
- "What does doc X say?" → \`search\` first.
|
|
156
156
|
- "Any new emails from Y?" → check the \`gmail\` drive first; only hit Gmail MCP if the freshest indexed item is too old for the question.
|
|
157
157
|
- "Send an email to Y" → MCP write directly; no context lookup.
|
|
158
158
|
|
package/src/tools/search/grep.ts
DELETED
|
@@ -1,128 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
import { formatDriveRef } from "../../context/drives.ts";
|
|
3
|
-
import {
|
|
4
|
-
listContextItems,
|
|
5
|
-
listContextItemsByPrefix,
|
|
6
|
-
} from "../../db/context.ts";
|
|
7
|
-
import type { ToolDefinition } from "../tool.ts";
|
|
8
|
-
|
|
9
|
-
const GrepMatchSchema = z.object({
|
|
10
|
-
ref: z.string(),
|
|
11
|
-
drive: z.string(),
|
|
12
|
-
path: z.string(),
|
|
13
|
-
line: z.number(),
|
|
14
|
-
content: z.string(),
|
|
15
|
-
context_lines: z.array(z.string()),
|
|
16
|
-
});
|
|
17
|
-
|
|
18
|
-
const inputSchema = z.object({
|
|
19
|
-
pattern: z.string().describe("Regex pattern to search for"),
|
|
20
|
-
drive: z
|
|
21
|
-
.string()
|
|
22
|
-
.optional()
|
|
23
|
-
.describe("Restrict search to a single drive (defaults to all drives)"),
|
|
24
|
-
path: z
|
|
25
|
-
.string()
|
|
26
|
-
.optional()
|
|
27
|
-
.describe(
|
|
28
|
-
"Directory to search under within the drive (defaults to /). Requires `drive`.",
|
|
29
|
-
),
|
|
30
|
-
glob: z
|
|
31
|
-
.string()
|
|
32
|
-
.optional()
|
|
33
|
-
.describe("Only search files whose basename matches this glob pattern"),
|
|
34
|
-
ignore_case: z.boolean().optional().describe("Case-insensitive search"),
|
|
35
|
-
context: z
|
|
36
|
-
.number()
|
|
37
|
-
.optional()
|
|
38
|
-
.describe("Number of context lines before and after each match"),
|
|
39
|
-
max_results: z
|
|
40
|
-
.number()
|
|
41
|
-
.optional()
|
|
42
|
-
.describe("Maximum number of matches to return"),
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
const outputSchema = z.object({
|
|
46
|
-
matches: z.array(GrepMatchSchema),
|
|
47
|
-
is_error: z.boolean(),
|
|
48
|
-
error_type: z.string().optional(),
|
|
49
|
-
message: z.string().optional(),
|
|
50
|
-
});
|
|
51
|
-
|
|
52
|
-
export const searchGrepTool = {
|
|
53
|
-
name: "search_grep",
|
|
54
|
-
description: "Search file contents by regex pattern across context drives.",
|
|
55
|
-
group: "search",
|
|
56
|
-
inputSchema,
|
|
57
|
-
outputSchema,
|
|
58
|
-
execute: async (input, ctx) => {
|
|
59
|
-
// `path` scopes to a directory within a single drive; requiring `drive`
|
|
60
|
-
// alongside prevents a silent full-DB scan when only `path` is passed.
|
|
61
|
-
if (input.path && !input.drive) {
|
|
62
|
-
return {
|
|
63
|
-
matches: [],
|
|
64
|
-
is_error: true,
|
|
65
|
-
error_type: "invalid_arguments",
|
|
66
|
-
message:
|
|
67
|
-
"`path` requires `drive` — use context_list_drives to see which drives exist, then pass `drive` alongside `path`.",
|
|
68
|
-
};
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
const items = input.drive
|
|
72
|
-
? await listContextItemsByPrefix(
|
|
73
|
-
ctx.conn,
|
|
74
|
-
input.drive,
|
|
75
|
-
input.path ?? "/",
|
|
76
|
-
{
|
|
77
|
-
recursive: true,
|
|
78
|
-
},
|
|
79
|
-
)
|
|
80
|
-
: await listContextItems(ctx.conn);
|
|
81
|
-
|
|
82
|
-
const flags = input.ignore_case ? "gi" : "g";
|
|
83
|
-
const regex = new RegExp(input.pattern, flags);
|
|
84
|
-
const globRegex = input.glob ? globToRegex(input.glob) : null;
|
|
85
|
-
const contextLines = input.context ?? 0;
|
|
86
|
-
const maxResults = input.max_results ?? 100;
|
|
87
|
-
|
|
88
|
-
const matches: z.infer<typeof GrepMatchSchema>[] = [];
|
|
89
|
-
|
|
90
|
-
for (const item of items) {
|
|
91
|
-
if (item.content == null) continue;
|
|
92
|
-
|
|
93
|
-
if (globRegex) {
|
|
94
|
-
const filename = item.path.split("/").pop() ?? "";
|
|
95
|
-
if (!globRegex.test(filename)) continue;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
const lines = item.content.split("\n");
|
|
99
|
-
for (let i = 0; i < lines.length; i++) {
|
|
100
|
-
regex.lastIndex = 0;
|
|
101
|
-
const line = lines[i];
|
|
102
|
-
if (line !== undefined && regex.test(line)) {
|
|
103
|
-
const start = Math.max(0, i - contextLines);
|
|
104
|
-
const end = Math.min(lines.length, i + contextLines + 1);
|
|
105
|
-
matches.push({
|
|
106
|
-
ref: formatDriveRef(item),
|
|
107
|
-
drive: item.drive,
|
|
108
|
-
path: item.path,
|
|
109
|
-
line: i + 1,
|
|
110
|
-
content: line,
|
|
111
|
-
context_lines: lines.slice(start, end),
|
|
112
|
-
});
|
|
113
|
-
if (matches.length >= maxResults) return { matches, is_error: false };
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
return { matches, is_error: false };
|
|
119
|
-
},
|
|
120
|
-
} satisfies ToolDefinition<typeof inputSchema, typeof outputSchema>;
|
|
121
|
-
|
|
122
|
-
function globToRegex(glob: string): RegExp {
|
|
123
|
-
const escaped = glob
|
|
124
|
-
.replace(/[.+^${}()|[\]\\]/g, "\\$&")
|
|
125
|
-
.replace(/\*/g, ".*")
|
|
126
|
-
.replace(/\?/g, ".");
|
|
127
|
-
return new RegExp(`^${escaped}$`, "i");
|
|
128
|
-
}
|