@a-t-h-i/bot-lobby 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +177 -799
- package/package.json +1 -1
- package/prompts/master.md +17 -0
- package/prompts/panel.md +3 -1
- package/prompts/planner.md +14 -0
- package/prompts/quickfix.md +3 -1
- package/prompts/scout.md +3 -0
- package/prompts/worker.md +3 -0
- package/src/classifier/answers.ts +110 -0
- package/src/classifier/classifier.ts +171 -0
- package/src/classifier/client.ts +239 -0
- package/src/classifier/effort.ts +143 -0
- package/src/classifier/files.ts +427 -0
- package/src/classifier/hosts.ts +157 -0
- package/src/classifier/instance.ts +98 -0
- package/src/classifier/limits.ts +37 -0
- package/src/classifier/seats.ts +102 -0
- package/src/classifier/tools.ts +58 -0
- package/src/classifier/triage.ts +203 -0
- package/src/execution/agent-runner.ts +5 -0
- package/src/index.ts +6 -0
- package/src/lobby/keys.ts +1 -1
- package/src/lobby/planner.ts +305 -27
- package/src/lobby/quickfix.ts +120 -8
- package/src/lobby/runtime.ts +136 -5
- package/src/lobby/session-files.ts +12 -0
- package/src/lobby/sessions.ts +28 -2
- package/src/lobby/tabs/home.ts +54 -26
- package/src/lobby/tabs/metrics.ts +28 -1
- package/src/lobby/tabs/plan.ts +27 -4
- package/src/lobby/tabs/quickfix.ts +5 -1
- package/src/lobby/tabs/tasks.ts +30 -14
- package/src/lobby/view.ts +334 -41
- package/src/master/master.ts +95 -29
- package/src/pi/commands.ts +24 -4
- package/src/pi/events.ts +11 -2
- package/src/pi/owner.ts +57 -8
- package/src/pi/run-summary.ts +3 -1
- package/src/pi/settings-ui.ts +138 -2
- package/src/pi/start-task.ts +4 -0
- package/src/pi/tools.ts +7 -2
- package/src/schemas/configuration.ts +125 -1
- package/src/schemas/findings.ts +4 -0
- package/src/schemas/task.ts +24 -0
- package/src/state/archive.ts +99 -0
- package/src/state/inbox.ts +49 -15
- package/src/state/metrics.ts +74 -1
- package/src/state/presence.ts +99 -0
- package/src/workflow/workflow.ts +49 -1
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Effort routing: a step the classifier judges simple runs one thinking
|
|
3
|
+
* level lower, and one it judges trivial runs on the cheaper model set in
|
|
4
|
+
* settings (`classifier.effort.cheapModel`), so large models and high
|
|
5
|
+
* thinking levels are spent where the work needs them. The configured model
|
|
6
|
+
* and thinking stay the ceiling: a route only ever goes down. A routed run
|
|
7
|
+
* that falls short (fails, stalls, times out, wraps up early or returns an
|
|
8
|
+
* unusable report) is run again on the configured profile, and review is
|
|
9
|
+
* unchanged. The Master, the oracle, the QA gate and the researcher are never
|
|
10
|
+
* routed.
|
|
11
|
+
*/
|
|
12
|
+
import { THINKING_LEVELS, type ClassifierConfig, type ThinkingLevelName } from "../schemas/configuration.ts";
|
|
13
|
+
import type { Classifier } from "./classifier.ts";
|
|
14
|
+
import { score, scoreOf } from "./client.ts";
|
|
15
|
+
import { clip } from "./limits.ts";
|
|
16
|
+
|
|
17
|
+
export const EFFORT_LEVELS = ["trivial", "simple", "moderate", "complex"] as const;
|
|
18
|
+
export type EffortLevel = (typeof EFFORT_LEVELS)[number];
|
|
19
|
+
|
|
20
|
+
const LEVELS = [
|
|
21
|
+
"trivial: a mechanical edit or lookup — a rename, a copy change, a one-line fix, reading one known file",
|
|
22
|
+
"simple: a small change that follows an existing pattern in one or two files",
|
|
23
|
+
"moderate: new logic or several files, with some judgment about the design",
|
|
24
|
+
"complex: design decisions, cross-cutting or concurrent changes, security, or an unclear root cause",
|
|
25
|
+
];
|
|
26
|
+
|
|
27
|
+
export interface RunProfileLike {
|
|
28
|
+
model?: string;
|
|
29
|
+
thinking: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface EffortRoute extends RunProfileLike {
|
|
33
|
+
level: "trivial" | "simple";
|
|
34
|
+
confidence: number;
|
|
35
|
+
/** The configured profile the route came down from. */
|
|
36
|
+
from: RunProfileLike;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Floor for every route: thinking never goes below `low` because of the classifier. */
|
|
40
|
+
const FLOOR: ThinkingLevelName = "low";
|
|
41
|
+
|
|
42
|
+
function rank(level: string): number {
|
|
43
|
+
return (THINKING_LEVELS as readonly string[]).indexOf(level);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** One thinking level lower, never below `low`; a level already at or below it stays. */
|
|
47
|
+
export function stepDown(thinking: string): string {
|
|
48
|
+
const index = rank(thinking);
|
|
49
|
+
if (index < 0 || index <= rank(FLOOR)) return thinking;
|
|
50
|
+
return THINKING_LEVELS[index - 1]!;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** The lower of two thinking levels. */
|
|
54
|
+
function lower(a: string, b: string): string {
|
|
55
|
+
return rank(a) >= 0 && rank(a) < rank(b) ? a : b;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface RouteOptions {
|
|
59
|
+
/** The run's thinking is fixed (scouts): only the model may change. */
|
|
60
|
+
thinkingFixed?: boolean;
|
|
61
|
+
/** Clamp a thinking level to what a model supports. */
|
|
62
|
+
clamp?: (model: string, thinking: string) => string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Where a scored step goes; undefined keeps the configured profile. */
|
|
66
|
+
export function planRoute(level: EffortLevel, confidence: number, profile: RunProfileLike, config: Pick<ClassifierConfig, "thresholds" | "effort">, options: RouteOptions = {}): EffortRoute | undefined {
|
|
67
|
+
const { simpleAt, trivialAt } = config.thresholds;
|
|
68
|
+
const cheap = config.effort.cheapModel && config.effort.cheapModel !== "inherit" ? config.effort.cheapModel : undefined;
|
|
69
|
+
let next: RunProfileLike | undefined;
|
|
70
|
+
let routed: EffortRoute["level"] | undefined;
|
|
71
|
+
if (level === "trivial" && confidence >= trivialAt && cheap && cheap !== profile.model) {
|
|
72
|
+
const thinking = options.thinkingFixed ? profile.thinking : lower(profile.thinking, FLOOR);
|
|
73
|
+
next = { model: cheap, thinking: options.clamp ? options.clamp(cheap, thinking) : thinking };
|
|
74
|
+
routed = "trivial";
|
|
75
|
+
} else if ((level === "trivial" || level === "simple") && confidence >= simpleAt && !options.thinkingFixed) {
|
|
76
|
+
next = { ...(profile.model ? { model: profile.model } : {}), thinking: stepDown(profile.thinking) };
|
|
77
|
+
routed = level;
|
|
78
|
+
}
|
|
79
|
+
if (!next || !routed || (next.model === profile.model && next.thinking === profile.thinking)) return undefined;
|
|
80
|
+
return { ...next, level: routed, confidence, from: { ...(profile.model ? { model: profile.model } : {}), thinking: profile.thinking } };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** How hard a step is, or undefined when the classifier is off or fails. */
|
|
84
|
+
export async function scoreEffort(classifier: Classifier, instruction: string, context: string | undefined, signal?: AbortSignal): Promise<{ level: EffortLevel; confidence: number } | undefined> {
|
|
85
|
+
if (!classifier.enabled("effort") || !instruction.trim()) return undefined;
|
|
86
|
+
const result = await classifier.ask("effort", {
|
|
87
|
+
state: { step: clip(instruction, 6000), ...(context ? { task: clip(context, 3000) } : {}) },
|
|
88
|
+
questions: { effort: score("How much reasoning does an experienced engineer need to do `step` well, given `task`?", LEVELS) },
|
|
89
|
+
}, signal ? { signal } : {});
|
|
90
|
+
const scored = scoreOf(result?.answers, "effort");
|
|
91
|
+
if (!scored) return undefined;
|
|
92
|
+
return { level: EFFORT_LEVELS[Math.max(0, Math.min(EFFORT_LEVELS.length - 1, scored.level))]!, confidence: scored.confidence };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** `p/big · medium`, the profile a route came down from. */
|
|
96
|
+
export function profileLabel(profile: RunProfileLike): string {
|
|
97
|
+
return `${profile.model ?? "session model"} · ${profile.thinking}`;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** `trivial 0.88: p/big · medium → p/cheap · low`, for receipts and the activity log. */
|
|
101
|
+
export function routeLabel(route: EffortRoute): string {
|
|
102
|
+
return `${route.level} ${route.confidence.toFixed(2)}: ${profileLabel(route.from)} → ${profileLabel(route)}`;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export interface EffortScore {
|
|
106
|
+
level: EffortLevel;
|
|
107
|
+
confidence: number;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Lowers the model or thinking of a step the classifier judges simple or trivial. */
|
|
111
|
+
export interface EffortRouter {
|
|
112
|
+
/** Score a step and plan its route in one go. */
|
|
113
|
+
route(instruction: string, profile: RunProfileLike, options?: { thinkingFixed?: boolean; context?: string; signal?: AbortSignal }): Promise<EffortRoute | undefined>;
|
|
114
|
+
/** Score a step once (several agents may share it)… */
|
|
115
|
+
score(instruction: string, options?: { context?: string; signal?: AbortSignal }): Promise<EffortScore | undefined>;
|
|
116
|
+
/** …then plan each agent's route from its own profile. */
|
|
117
|
+
plan(scored: EffortScore, profile: RunProfileLike, options?: { thinkingFixed?: boolean }): EffortRoute | undefined;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export function effortRouter(classifier: Classifier, deps: { clamp?: RouteOptions["clamp"]; log?: (text: string) => void } = {}): EffortRouter {
|
|
121
|
+
const router: EffortRouter = {
|
|
122
|
+
async score(instruction, options = {}) {
|
|
123
|
+
if (!classifier.enabled("effort")) return undefined;
|
|
124
|
+
return scoreEffort(classifier, instruction, options.context, options.signal);
|
|
125
|
+
},
|
|
126
|
+
plan(scored, profile, options = {}) {
|
|
127
|
+
const route = planRoute(scored.level, scored.confidence, profile, classifier.config, { ...(options.thinkingFixed ? { thinkingFixed: true } : {}), ...(deps.clamp ? { clamp: deps.clamp } : {}) });
|
|
128
|
+
if (route) deps.log?.(`routed ${routeLabel(route)}`);
|
|
129
|
+
return route;
|
|
130
|
+
},
|
|
131
|
+
async route(instruction, profile, options = {}) {
|
|
132
|
+
const scored = await router.score(instruction, { ...(options.context ? { context: options.context } : {}), ...(options.signal ? { signal: options.signal } : {}) });
|
|
133
|
+
return scored ? router.plan(scored, profile, options.thinkingFixed ? { thinkingFixed: true } : {}) : undefined;
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
return router;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Whether a routed run fell short and must run again on the configured profile. */
|
|
140
|
+
export function fellShort(run: { status: string; stalled?: boolean; wrappedUp?: boolean }, issues: readonly string[] = []): boolean {
|
|
141
|
+
if (run.status === "cancelled") return false;
|
|
142
|
+
return run.status !== "success" || Boolean(run.stalled) || Boolean(run.wrappedUp) || issues.length > 0;
|
|
143
|
+
}
|
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Likely files: instead of an agent spending its first turns on `find` and
|
|
3
|
+
* `grep`, the engine ranks the repository's files against the step and hands
|
|
4
|
+
* the agent a short list up front (and, inside subagents, a
|
|
5
|
+
* `find_relevant_files` tool for later lookups).
|
|
6
|
+
*
|
|
7
|
+
* Every file gets a short excerpt — its leading comment, imports and
|
|
8
|
+
* signatures, never the whole file — kept in a cache that is refreshed by
|
|
9
|
+
* size and modification time. A lexical prefilter narrows a large repository
|
|
10
|
+
* to the most promising candidates; the classifier then judges each one
|
|
11
|
+
* independently (one yes/no per file, in parallel batches), so batches merge
|
|
12
|
+
* by sorting. Secrets and anything matching `classifier.exclude` are never
|
|
13
|
+
* indexed.
|
|
14
|
+
*/
|
|
15
|
+
import { execFile } from "node:child_process";
|
|
16
|
+
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
17
|
+
import { open, readdir, stat } from "node:fs/promises";
|
|
18
|
+
import { dirname, join } from "node:path";
|
|
19
|
+
import { promisify } from "node:util";
|
|
20
|
+
import { dataRoot } from "../state/project.ts";
|
|
21
|
+
import type { Classifier } from "./classifier.ts";
|
|
22
|
+
import { noul, yesOf, type SystemOneRequest } from "./client.ts";
|
|
23
|
+
import { clip, LIMITS } from "./limits.ts";
|
|
24
|
+
|
|
25
|
+
const run = promisify(execFile);
|
|
26
|
+
|
|
27
|
+
export const FIND_FILES_TOOL = "find_relevant_files";
|
|
28
|
+
|
|
29
|
+
/** Where the index lives and which tree it describes. */
|
|
30
|
+
export interface FileScope {
|
|
31
|
+
cwd: string;
|
|
32
|
+
root: string;
|
|
33
|
+
configDir: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface IndexedFile {
|
|
37
|
+
path: string;
|
|
38
|
+
excerpt: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface LikelyFile {
|
|
42
|
+
path: string;
|
|
43
|
+
relevance: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface LikelyFiles {
|
|
47
|
+
files: LikelyFile[];
|
|
48
|
+
/** How likely any candidate answers the query at all. */
|
|
49
|
+
anyRelevant: number;
|
|
50
|
+
/** Files the classifier judged. */
|
|
51
|
+
judged: number;
|
|
52
|
+
ms: number;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Never sent anywhere: keys, certificates, env files and the like. */
|
|
56
|
+
export const SECRET_EXCLUDES: readonly string[] = [
|
|
57
|
+
".env", ".env.*", "*.env", "*.pem", "*.key", "*.p12", "*.pfx", "*.keystore", "*.jks", "*.kdbx",
|
|
58
|
+
"id_rsa*", "id_dsa*", "id_ecdsa*", "id_ed25519*", "**/secrets/**", "**/.ssh/**", "**/.aws/**",
|
|
59
|
+
"*.secret", "*secrets.json", "*secrets.yaml", "*secrets.yml", "*secrets.toml", "credentials", "credentials.json",
|
|
60
|
+
".npmrc", ".pypirc", ".netrc", ".git-credentials",
|
|
61
|
+
];
|
|
62
|
+
|
|
63
|
+
/** Directories never walked (when git is not there to list files). */
|
|
64
|
+
const SKIP_DIRS = new Set([".git", "node_modules", "dist", "build", "out", "coverage", ".next", ".nuxt", ".cache", "vendor", "target", ".venv", "venv", "__pycache__", ".pi", ".idea", ".vscode"]);
|
|
65
|
+
|
|
66
|
+
/** Files that carry no signal for "which file does this step need". */
|
|
67
|
+
const SKIP_EXTENSIONS = new Set([
|
|
68
|
+
"png", "jpg", "jpeg", "gif", "webp", "ico", "bmp", "tiff", "svgz", "pdf", "zip", "gz", "tgz", "bz2", "xz", "7z", "rar", "jar", "war",
|
|
69
|
+
"woff", "woff2", "ttf", "otf", "eot", "mp3", "mp4", "mov", "avi", "webm", "wav", "ogg", "flac", "exe", "dll", "so", "dylib", "bin",
|
|
70
|
+
"class", "o", "a", "pyc", "wasm", "map", "lock", "sqlite", "db",
|
|
71
|
+
]);
|
|
72
|
+
const SKIP_NAMES = new Set(["package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lockb", "Cargo.lock", "poetry.lock", "composer.lock", "Gemfile.lock", "go.sum"]);
|
|
73
|
+
|
|
74
|
+
export const MAX_FILES = 20_000;
|
|
75
|
+
export const MAX_FILE_BYTES = 256 * 1024;
|
|
76
|
+
const HEAD_BYTES = 8 * 1024;
|
|
77
|
+
export const EXCERPT_CHARS = 400;
|
|
78
|
+
/** Candidates per classifier call; each costs its excerpt plus one short question. */
|
|
79
|
+
export const BATCH_SIZE = 150;
|
|
80
|
+
const BATCH_CHARS = 90_000;
|
|
81
|
+
/** Below this, no list is shown: pointing an agent at the least irrelevant files would mislead it. */
|
|
82
|
+
export const ANY_RELEVANT_FLOOR = 0.3;
|
|
83
|
+
|
|
84
|
+
/** A path glob as a regular expression: `**` spans directories, `*` and `?` stay within one. */
|
|
85
|
+
export function globToRegExp(glob: string): RegExp {
|
|
86
|
+
let pattern = "";
|
|
87
|
+
for (let i = 0; i < glob.length; i++) {
|
|
88
|
+
const char = glob[i]!;
|
|
89
|
+
if (char === "*" && glob[i + 1] === "*") {
|
|
90
|
+
const slash = glob[i + 2] === "/";
|
|
91
|
+
pattern += slash ? "(?:.*/)?" : ".*";
|
|
92
|
+
i += slash ? 2 : 1;
|
|
93
|
+
} else if (char === "*") pattern += "[^/]*";
|
|
94
|
+
else if (char === "?") pattern += "[^/]";
|
|
95
|
+
else pattern += char.replace(/[.+^${}()|[\]\\]/g, "\\$&");
|
|
96
|
+
}
|
|
97
|
+
return new RegExp(`^${pattern}$`, "i");
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** A path is excluded when a glob matches it, or its file name for a glob without a slash. */
|
|
101
|
+
export function excludedBy(globs: readonly string[]): (path: string) => boolean {
|
|
102
|
+
const tests = globs.map((glob) => ({ regex: globToRegExp(glob), basename: !glob.includes("/") }));
|
|
103
|
+
return (path) => {
|
|
104
|
+
const name = path.slice(path.lastIndexOf("/") + 1);
|
|
105
|
+
return tests.some(({ regex, basename }) => regex.test(path) || (basename && regex.test(name)));
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function skippable(path: string): boolean {
|
|
110
|
+
const name = path.slice(path.lastIndexOf("/") + 1);
|
|
111
|
+
if (SKIP_NAMES.has(name) || /\.min\.(js|css)$/i.test(name)) return true;
|
|
112
|
+
const dot = name.lastIndexOf(".");
|
|
113
|
+
return dot > 0 && SKIP_EXTENSIONS.has(name.slice(dot + 1).toLowerCase());
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const SIGNATURE = /^(export\s|(?:pub(?:\(\w+\))?\s+)?(?:async\s+)?(?:fn|func|def|class|function|interface|type|enum|struct|trait|impl|module|namespace|protocol|record)\s|const\s+\w+\s*=\s*(?:async\s*)?(?:\(|function)|(?:public|private|protected|internal)\s+(?:static\s+)?[\w<>[\],\s]+\s+\w+\s*\(|@(?:app|router)\.\w+\(|describe\(|test\(|it\()/;
|
|
117
|
+
const IMPORT = /^(import\s|from\s+\S+\s+import\s|(?:const|let|var)\s+\w+\s*=\s*require\(|use\s+[\w:]+|#include\s|using\s+[\w.]+;|package\s+[\w.]+)/;
|
|
118
|
+
const COMMENT = /^(\/\/|\/\*|\*|#(?![!#])|"""|'''|<!--|--\s|;)/;
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* What a file is about in a few hundred characters: its leading comment, its
|
|
122
|
+
* signatures (headings for Markdown), then its imports; the first lines when
|
|
123
|
+
* none of those exist.
|
|
124
|
+
*/
|
|
125
|
+
export function excerptOf(path: string, text: string, max = EXCERPT_CHARS): string {
|
|
126
|
+
const lines = text.split(/\r?\n/).slice(0, 600);
|
|
127
|
+
const markdown = /\.(md|mdx|markdown|rst|txt)$/i.test(path);
|
|
128
|
+
const lead: string[] = [];
|
|
129
|
+
for (const line of lines) {
|
|
130
|
+
const trimmed = line.trim();
|
|
131
|
+
if (!trimmed) {
|
|
132
|
+
if (lead.length > 0) break;
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
if (trimmed.startsWith("#!")) continue;
|
|
136
|
+
if (markdown || !COMMENT.test(trimmed)) break;
|
|
137
|
+
const words = trimmed.replace(/^(\/\*+|\*+\/?|\/\/+|#+|"""|'''|<!--|--|;+)\s*/, "").replace(/\*\/$|-->$/, "").trim();
|
|
138
|
+
if (words) lead.push(words);
|
|
139
|
+
if (lead.length >= 4) break;
|
|
140
|
+
}
|
|
141
|
+
const signatures: string[] = [];
|
|
142
|
+
const imports: string[] = [];
|
|
143
|
+
for (const line of lines) {
|
|
144
|
+
const trimmed = line.trim();
|
|
145
|
+
if (!trimmed) continue;
|
|
146
|
+
if (markdown ? /^#{1,3}\s+\S/.test(trimmed) : SIGNATURE.test(trimmed)) {
|
|
147
|
+
if (signatures.length < 14) signatures.push(trimmed.replace(/\s*\{\s*\}?\s*$/, "").replace(/\s+/g, " ").slice(0, 120));
|
|
148
|
+
} else if (!markdown && IMPORT.test(trimmed) && imports.length < 6) {
|
|
149
|
+
imports.push(trimmed.replace(/\s+/g, " ").slice(0, 80));
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
// Half the room at most for the comment, so the signatures always show.
|
|
153
|
+
const parts = [clip(lead.join(" "), Math.floor(max / 2)), signatures.join("\n"), imports.length > 0 ? imports.join("; ") : ""].filter(Boolean);
|
|
154
|
+
const body = parts.length > 0 ? parts.join("\n") : lines.map((line) => line.trim()).filter(Boolean).slice(0, 6).join("\n");
|
|
155
|
+
return clip(body, max);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
interface CacheEntry {
|
|
159
|
+
mtimeMs: number;
|
|
160
|
+
size: number;
|
|
161
|
+
excerpt: string;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
interface CacheFile {
|
|
165
|
+
version: 1;
|
|
166
|
+
cwd: string;
|
|
167
|
+
files: Record<string, CacheEntry>;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export function fileCachePath(scope: FileScope): string {
|
|
171
|
+
return join(dataRoot(scope.root, scope.configDir), "cache", "files.json");
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function readCache(scope: FileScope): Record<string, CacheEntry> {
|
|
175
|
+
try {
|
|
176
|
+
const parsed = JSON.parse(readFileSync(fileCachePath(scope), "utf8")) as Partial<CacheFile>;
|
|
177
|
+
return parsed.version === 1 && parsed.cwd === scope.cwd && parsed.files && typeof parsed.files === "object" ? parsed.files : {};
|
|
178
|
+
} catch {
|
|
179
|
+
return {};
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function writeCache(scope: FileScope, files: Record<string, CacheEntry>): void {
|
|
184
|
+
const path = fileCachePath(scope);
|
|
185
|
+
try {
|
|
186
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
187
|
+
writeFileSync(`${path}.tmp`, JSON.stringify({ version: 1, cwd: scope.cwd, files } satisfies CacheFile));
|
|
188
|
+
renameSync(`${path}.tmp`, path);
|
|
189
|
+
} catch {
|
|
190
|
+
// The cache is an optimisation; a read-only tree just re-reads next time.
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Files git tracks or would track (untracked but not ignored); undefined outside a git work tree. */
|
|
195
|
+
async function gitFiles(cwd: string): Promise<string[] | undefined> {
|
|
196
|
+
try {
|
|
197
|
+
const { stdout } = await run("git", ["ls-files", "-co", "--exclude-standard", "-z"], { cwd, maxBuffer: 64 * 1024 * 1024, timeout: 10_000 });
|
|
198
|
+
return stdout.split("\0").filter(Boolean).slice(0, MAX_FILES);
|
|
199
|
+
} catch {
|
|
200
|
+
return undefined;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
async function walkFiles(cwd: string): Promise<string[]> {
|
|
205
|
+
const found: string[] = [];
|
|
206
|
+
const queue = [""];
|
|
207
|
+
while (queue.length > 0 && found.length < MAX_FILES) {
|
|
208
|
+
const dir = queue.shift()!;
|
|
209
|
+
let entries;
|
|
210
|
+
try {
|
|
211
|
+
entries = await readdir(join(cwd, dir), { withFileTypes: true });
|
|
212
|
+
} catch {
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
for (const entry of entries) {
|
|
216
|
+
const path = dir ? `${dir}/${entry.name}` : entry.name;
|
|
217
|
+
if (entry.isDirectory()) {
|
|
218
|
+
if (!SKIP_DIRS.has(entry.name)) queue.push(path);
|
|
219
|
+
} else if (entry.isFile()) found.push(path);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return found.slice(0, MAX_FILES);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
async function readHead(path: string): Promise<string | undefined> {
|
|
226
|
+
const handle = await open(path, "r");
|
|
227
|
+
try {
|
|
228
|
+
const buffer = Buffer.alloc(HEAD_BYTES);
|
|
229
|
+
const { bytesRead } = await handle.read(buffer, 0, HEAD_BYTES, 0);
|
|
230
|
+
const head = buffer.subarray(0, bytesRead);
|
|
231
|
+
return head.includes(0) ? undefined : head.toString("utf8");
|
|
232
|
+
} finally {
|
|
233
|
+
await handle.close();
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* The repository's files with their excerpts. Unchanged files come from the
|
|
239
|
+
* cache (same size and modification time); new or changed ones are read
|
|
240
|
+
* (their first 8 KB) and the cache is rewritten.
|
|
241
|
+
*/
|
|
242
|
+
export async function indexFiles(scope: FileScope, exclude: readonly string[] = []): Promise<IndexedFile[]> {
|
|
243
|
+
const listed = (await gitFiles(scope.cwd)) ?? (await walkFiles(scope.cwd));
|
|
244
|
+
const excluded = excludedBy([...SECRET_EXCLUDES, ...exclude]);
|
|
245
|
+
const paths = listed.filter((path) => !path.startsWith(`${scope.configDir}/`) && !skippable(path) && !excluded(path));
|
|
246
|
+
const cached = readCache(scope);
|
|
247
|
+
const next: Record<string, CacheEntry> = {};
|
|
248
|
+
let changed = Object.keys(cached).length !== paths.length;
|
|
249
|
+
const indexed: IndexedFile[] = [];
|
|
250
|
+
const CONCURRENCY = 64;
|
|
251
|
+
for (let start = 0; start < paths.length; start += CONCURRENCY) {
|
|
252
|
+
const slice = paths.slice(start, start + CONCURRENCY);
|
|
253
|
+
const entries = await Promise.all(slice.map(async (path): Promise<[string, CacheEntry] | undefined> => {
|
|
254
|
+
try {
|
|
255
|
+
const info = await stat(join(scope.cwd, path));
|
|
256
|
+
if (!info.isFile() || info.size > MAX_FILE_BYTES) return undefined;
|
|
257
|
+
const hit = cached[path];
|
|
258
|
+
if (hit && hit.mtimeMs === info.mtimeMs && hit.size === info.size) return [path, hit];
|
|
259
|
+
const head = await readHead(join(scope.cwd, path));
|
|
260
|
+
if (head === undefined) return undefined;
|
|
261
|
+
changed = true;
|
|
262
|
+
return [path, { mtimeMs: info.mtimeMs, size: info.size, excerpt: excerptOf(path, head) }];
|
|
263
|
+
} catch {
|
|
264
|
+
return undefined;
|
|
265
|
+
}
|
|
266
|
+
}));
|
|
267
|
+
for (const entry of entries) {
|
|
268
|
+
if (!entry) continue;
|
|
269
|
+
next[entry[0]] = entry[1];
|
|
270
|
+
indexed.push({ path: entry[0], excerpt: entry[1].excerpt });
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
if (changed) writeCache(scope, next);
|
|
274
|
+
return indexed;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
const STOPWORDS = new Set(["the", "and", "for", "with", "that", "this", "from", "into", "when", "then", "than", "should", "must", "will", "not", "are", "was", "has", "have", "its", "you", "our", "use", "add", "make", "file", "files", "code", "step", "task", "all", "any", "new", "can", "get", "set"]);
|
|
278
|
+
|
|
279
|
+
/** Words of three letters or more, camelCase and snake_case split, stopwords dropped. */
|
|
280
|
+
export function terms(text: string): string[] {
|
|
281
|
+
return [...new Set(text.replace(/([a-z0-9])([A-Z])/g, "$1 $2").toLowerCase().split(/[^a-z0-9]+/).filter((word) => word.length >= 3 && !STOPWORDS.has(word)))];
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* The most promising candidates for a query, at most `max`: files whose path
|
|
286
|
+
* (weighted three times) or excerpt share its words first, then the
|
|
287
|
+
* shallowest files, so a small repository is judged whole.
|
|
288
|
+
*/
|
|
289
|
+
export function prefilter(query: string, files: readonly IndexedFile[], max: number): IndexedFile[] {
|
|
290
|
+
if (files.length <= max) return [...files];
|
|
291
|
+
const words = terms(query);
|
|
292
|
+
const scored = files.map((file, index) => {
|
|
293
|
+
const path = file.path.toLowerCase();
|
|
294
|
+
const excerpt = file.excerpt.toLowerCase();
|
|
295
|
+
const score = words.reduce((total, word) => total + (path.includes(word) ? 3 : 0) + (excerpt.includes(word) ? 1 : 0), 0);
|
|
296
|
+
return { file, index, score, depth: file.path.split("/").length };
|
|
297
|
+
});
|
|
298
|
+
scored.sort((a, b) => b.score - a.score || a.depth - b.depth || a.index - b.index);
|
|
299
|
+
return scored.slice(0, max).map((entry) => entry.file);
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const ANY_KEY = "any_relevant";
|
|
303
|
+
|
|
304
|
+
/** Split candidates into requests that fit one call each. */
|
|
305
|
+
export function rankRequests(query: string, candidates: readonly IndexedFile[], context?: string): Array<{ request: SystemOneRequest; keys: Map<string, string> }> {
|
|
306
|
+
const batches: Array<{ request: SystemOneRequest; keys: Map<string, string> }> = [];
|
|
307
|
+
let entries: Record<string, string> = {};
|
|
308
|
+
let questions: SystemOneRequest["questions"] = {};
|
|
309
|
+
let keys = new Map<string, string>();
|
|
310
|
+
let size = 0;
|
|
311
|
+
const flush = () => {
|
|
312
|
+
if (keys.size === 0) return;
|
|
313
|
+
questions[ANY_KEY] = noul("Does at least one entry in `candidates` hold what `query` needs?");
|
|
314
|
+
batches.push({ request: { state: { query: clip(query, 3000), ...(context ? { context: clip(context, 2000) } : {}), candidates: entries }, questions }, keys });
|
|
315
|
+
entries = {};
|
|
316
|
+
questions = {};
|
|
317
|
+
keys = new Map();
|
|
318
|
+
size = 0;
|
|
319
|
+
};
|
|
320
|
+
candidates.forEach((file, index) => {
|
|
321
|
+
const key = `c${index + 1}`;
|
|
322
|
+
const text = `${file.path}\n${file.excerpt}`;
|
|
323
|
+
const question = noul(`Would an agent doing \`query\` need to read or change \`candidates.${key}\` (a file path, then its opening lines and signatures)? Yes if it holds the code, config, tests or docs the task touches; no if it is about something else or only shares words with it.`);
|
|
324
|
+
const cost = text.length + JSON.stringify(question).length + 16;
|
|
325
|
+
if (keys.size >= BATCH_SIZE || (keys.size > 0 && size + cost > BATCH_CHARS)) flush();
|
|
326
|
+
entries[key] = text;
|
|
327
|
+
questions[key] = question;
|
|
328
|
+
keys.set(key, file.path);
|
|
329
|
+
size += cost;
|
|
330
|
+
});
|
|
331
|
+
flush();
|
|
332
|
+
return batches.filter((batch) => JSON.stringify(batch.request).length <= LIMITS.requestChars);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
export interface LikelyOptions {
|
|
336
|
+
topK?: number;
|
|
337
|
+
signal?: AbortSignal;
|
|
338
|
+
/** Stop waiting after this long and go without (the index keeps building for next time). */
|
|
339
|
+
budgetMs?: number;
|
|
340
|
+
/** Shared context every file is judged against (the task, not only this step). */
|
|
341
|
+
context?: string;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/** Rank the repository's files for a query; undefined when the classifier is off, fails or runs out of time. */
|
|
345
|
+
export async function likelyFiles(classifier: Classifier, scope: FileScope, query: string, options: LikelyOptions = {}): Promise<LikelyFiles | undefined> {
|
|
346
|
+
if (!classifier.enabled("files") || !query.trim()) return undefined;
|
|
347
|
+
const settings = classifier.config;
|
|
348
|
+
const controller = new AbortController();
|
|
349
|
+
const onAbort = () => controller.abort();
|
|
350
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
351
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
352
|
+
const budget = options.budgetMs && options.budgetMs > 0
|
|
353
|
+
? new Promise<undefined>((resolve) => {
|
|
354
|
+
timer = setTimeout(() => {
|
|
355
|
+
controller.abort();
|
|
356
|
+
resolve(undefined);
|
|
357
|
+
}, options.budgetMs);
|
|
358
|
+
})
|
|
359
|
+
: undefined;
|
|
360
|
+
const work = (async (): Promise<LikelyFiles | undefined> => {
|
|
361
|
+
const started = Date.now();
|
|
362
|
+
const files = await indexFiles(scope, settings.exclude);
|
|
363
|
+
if (files.length === 0 || controller.signal.aborted) return undefined;
|
|
364
|
+
const candidates = prefilter(query, files, settings.fileHints.maxCandidates);
|
|
365
|
+
const batches = rankRequests(query, candidates, options.context);
|
|
366
|
+
const results = await Promise.all(batches.map((batch) => classifier.ask("files", batch.request, { signal: controller.signal })));
|
|
367
|
+
if (results.every((result) => !result) || controller.signal.aborted) return undefined;
|
|
368
|
+
const ranked: LikelyFile[] = [];
|
|
369
|
+
let anyRelevant = 0;
|
|
370
|
+
let judged = 0;
|
|
371
|
+
results.forEach((result, index) => {
|
|
372
|
+
if (!result) return;
|
|
373
|
+
anyRelevant = Math.max(anyRelevant, yesOf(result.answers, ANY_KEY) ?? 0);
|
|
374
|
+
for (const [key, path] of batches[index]!.keys) {
|
|
375
|
+
const relevance = yesOf(result.answers, key);
|
|
376
|
+
if (relevance === undefined) continue;
|
|
377
|
+
judged += 1;
|
|
378
|
+
ranked.push({ path, relevance });
|
|
379
|
+
}
|
|
380
|
+
});
|
|
381
|
+
ranked.sort((a, b) => b.relevance - a.relevance);
|
|
382
|
+
const topK = options.topK ?? settings.fileHints.topK;
|
|
383
|
+
const files_ = ranked.filter((entry) => entry.relevance >= settings.thresholds.fileRelevantAt).slice(0, topK);
|
|
384
|
+
return { files: files_, anyRelevant, judged, ms: Date.now() - started };
|
|
385
|
+
})();
|
|
386
|
+
try {
|
|
387
|
+
return await (budget ? Promise.race([work, budget]) : work);
|
|
388
|
+
} catch {
|
|
389
|
+
return undefined;
|
|
390
|
+
} finally {
|
|
391
|
+
if (timer) clearTimeout(timer);
|
|
392
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/** The block an agent's context gets; empty when nothing stands out. */
|
|
397
|
+
export function likelyFilesBlock(result: LikelyFiles | undefined): string {
|
|
398
|
+
if (!result || result.files.length === 0 || result.anyRelevant < ANY_RELEVANT_FLOOR) return "";
|
|
399
|
+
return [
|
|
400
|
+
"## Likely files",
|
|
401
|
+
"",
|
|
402
|
+
"The classifier ranked these as the files this step most likely needs (hints, not facts). Open them first; search with grep or find only for what they do not answer.",
|
|
403
|
+
"",
|
|
404
|
+
...result.files.map((file) => `- \`${file.path}\` (${file.relevance.toFixed(2)})`),
|
|
405
|
+
].join("\n");
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** Hands likely files to agents: a block for their context and the lookup tool for their allowlist. */
|
|
409
|
+
export interface FileHinter {
|
|
410
|
+
/** The Likely files block for a step, or "" (off, nothing stands out, out of time). */
|
|
411
|
+
block(query: string, signal?: AbortSignal, context?: string): Promise<string>;
|
|
412
|
+
/** Tools to add to a subagent's allowlist: `find_relevant_files` while file hints are on. */
|
|
413
|
+
tools(): readonly string[];
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
export function fileHinter(classifier: Classifier, scope: FileScope, log?: (text: string) => void): FileHinter {
|
|
417
|
+
return {
|
|
418
|
+
async block(query, signal, context) {
|
|
419
|
+
if (!classifier.enabled("files")) return "";
|
|
420
|
+
const result = await likelyFiles(classifier, scope, query, { ...(signal ? { signal } : {}), budgetMs: classifier.config.fileHints.budgetMs, ...(context ? { context } : {}) });
|
|
421
|
+
const block = likelyFilesBlock(result);
|
|
422
|
+
if (result) log?.(block ? `likely files: ${result.files.map((file) => `${file.path} (${file.relevance.toFixed(2)})`).join(", ")} · ${result.ms} ms` : `no file stands out (${result.anyRelevant.toFixed(2)}) · ${result.ms} ms`);
|
|
423
|
+
return block;
|
|
424
|
+
},
|
|
425
|
+
tools: () => (classifier.enabled("files") ? [FIND_FILES_TOOL] : []),
|
|
426
|
+
};
|
|
427
|
+
}
|