jeopi-cli 16.3.0 → 16.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/cli.js +2731 -2707
- package/dist/types/config/settings-schema.d.ts +20 -0
- package/dist/types/memories/index.d.ts +9 -0
- package/dist/types/memory-backend/types.d.ts +11 -0
- package/dist/types/slash-commands/helpers/skill-verify.d.ts +5 -0
- package/dist/types/task/model-tier-classifier.d.ts +8 -0
- package/dist/types/tools/builtin-names.d.ts +1 -1
- package/dist/types/tools/index.d.ts +1 -0
- package/dist/types/tools/learn.d.ts +4 -0
- package/dist/types/tools/path-write-lock.d.ts +10 -0
- package/dist/types/tools/vision-verify.d.ts +34 -0
- package/package.json +12 -12
- package/src/autolearn/controller.ts +21 -5
- package/src/config/settings-schema.ts +24 -0
- package/src/edit/index.ts +2 -1
- package/src/internal-urls/docs-index.generated.txt +2 -2
- package/src/memories/index.ts +10 -2
- package/src/memory-backend/local-backend.ts +34 -0
- package/src/memory-backend/types.ts +11 -0
- package/src/prompts/agents/designer.md +1 -0
- package/src/prompts/tools/learn.md +2 -0
- package/src/prompts/tools/vision-verify-system.md +16 -0
- package/src/prompts/tools/vision-verify.md +20 -0
- package/src/slash-commands/builtin-registry.ts +9 -0
- package/src/slash-commands/helpers/skill-verify.ts +171 -0
- package/src/task/index.ts +26 -1
- package/src/task/model-tier-classifier.ts +43 -0
- package/src/tools/builtin-names.ts +1 -0
- package/src/tools/index.ts +4 -0
- package/src/tools/learn.ts +16 -1
- package/src/tools/path-write-lock.ts +44 -0
- package/src/tools/vision-verify.ts +347 -0
- package/src/tools/write.ts +12 -9
package/src/memories/index.ts
CHANGED
|
@@ -1342,7 +1342,7 @@ export function getMemoryRoot(agentDir: string, cwd: string): string {
|
|
|
1342
1342
|
* consolidation artifacts (`MEMORY.md`, `memory_summary.md`, `skills/`) so a
|
|
1343
1343
|
* consolidation pass never clobbers manually captured lessons.
|
|
1344
1344
|
*/
|
|
1345
|
-
const LEARNED_LESSONS_FILE = "learned.md";
|
|
1345
|
+
export const LEARNED_LESSONS_FILE = "learned.md";
|
|
1346
1346
|
/** Newest-first cap on retained lessons, bounding file growth by entry count. */
|
|
1347
1347
|
const MAX_LEARNED_LESSONS = 100;
|
|
1348
1348
|
/** Per-field char caps so a single huge capture can't bloat learned.md. */
|
|
@@ -1414,7 +1414,15 @@ export async function saveLearnedLesson(
|
|
|
1414
1414
|
return { backend: "local", stored: 0, message: "Empty lesson; nothing stored." };
|
|
1415
1415
|
}
|
|
1416
1416
|
const context = input.context ? normalizeLearnedText(input.context, MAX_LEARNED_CONTEXT_CHARS) : "";
|
|
1417
|
-
const
|
|
1417
|
+
const contextSuffix = context ? ` _(context: ${context})_` : "";
|
|
1418
|
+
let line: string;
|
|
1419
|
+
if (input.verified === true) {
|
|
1420
|
+
const evidence = input.evidence ? normalizeLearnedText(input.evidence, MAX_LEARNED_CONTEXT_CHARS) : "";
|
|
1421
|
+
const evidenceSuffix = evidence ? ` _(evidence: ${evidence})_` : "";
|
|
1422
|
+
line = `- [VERIFIED] ${content}${evidenceSuffix}${contextSuffix}`;
|
|
1423
|
+
} else {
|
|
1424
|
+
line = `- [UNVERIFIED] ${content}${contextSuffix}`;
|
|
1425
|
+
}
|
|
1418
1426
|
const filePath = path.join(getMemoryRoot(agentDir, cwd), LEARNED_LESSONS_FILE);
|
|
1419
1427
|
|
|
1420
1428
|
// Serialize the read-modify-write per file: parallel `learn` calls (sibling
|
|
@@ -1,8 +1,12 @@
|
|
|
1
|
+
import * as path from "node:path";
|
|
2
|
+
import { isEnoent } from "jeopi-utils";
|
|
1
3
|
import {
|
|
2
4
|
buildMemoryToolDeveloperInstructions,
|
|
3
5
|
clearMemoryData,
|
|
4
6
|
clearMemoryToolDeveloperInstructionsCache,
|
|
5
7
|
enqueueMemoryConsolidation,
|
|
8
|
+
getMemoryRoot,
|
|
9
|
+
LEARNED_LESSONS_FILE,
|
|
6
10
|
saveLearnedLesson,
|
|
7
11
|
startMemoryStartupTask,
|
|
8
12
|
} from "../memories";
|
|
@@ -44,4 +48,34 @@ export const localBackend: MemoryBackend = {
|
|
|
44
48
|
"Local rollout-summary memory is active; lessons from the `learn` tool are saved to learned.md. Structured search is not available.",
|
|
45
49
|
};
|
|
46
50
|
},
|
|
51
|
+
async stats(agentDir, cwd) {
|
|
52
|
+
const filePath = path.join(getMemoryRoot(agentDir, cwd), LEARNED_LESSONS_FILE);
|
|
53
|
+
let raw = "";
|
|
54
|
+
try {
|
|
55
|
+
raw = await Bun.file(filePath).text();
|
|
56
|
+
} catch (err) {
|
|
57
|
+
if (!isEnoent(err)) throw err;
|
|
58
|
+
}
|
|
59
|
+
let total = 0;
|
|
60
|
+
let verified = 0;
|
|
61
|
+
for (const rawLine of raw.split("\n")) {
|
|
62
|
+
const line = rawLine.trim();
|
|
63
|
+
if (!line.startsWith("- ")) continue;
|
|
64
|
+
total++;
|
|
65
|
+
if (line.startsWith(VERIFIED_PREFIX)) verified++;
|
|
66
|
+
}
|
|
67
|
+
const unverified = total - verified;
|
|
68
|
+
const coverage = total > 0 ? `${((verified / total) * 100).toFixed(1)}%` : "N/A";
|
|
69
|
+
return [
|
|
70
|
+
"# Local Memory Stats",
|
|
71
|
+
"",
|
|
72
|
+
`- Lessons: ${total}`,
|
|
73
|
+
`- Verified: ${verified}`,
|
|
74
|
+
`- Unverified: ${unverified}`,
|
|
75
|
+
`- Verification coverage: ${coverage}`,
|
|
76
|
+
].join("\n");
|
|
77
|
+
},
|
|
47
78
|
};
|
|
79
|
+
|
|
80
|
+
/** Line prefix `saveLearnedLesson` writes for verified/unverified lessons in `learned.md`. */
|
|
81
|
+
const VERIFIED_PREFIX = "- [VERIFIED]";
|
|
@@ -60,6 +60,17 @@ export interface MemoryBackendSaveInput {
|
|
|
60
60
|
context?: string;
|
|
61
61
|
source?: string;
|
|
62
62
|
importance?: number;
|
|
63
|
+
/**
|
|
64
|
+
* Marks this lesson as independently re-checked against the repo (a
|
|
65
|
+
* command run, a file read, a test executed) rather than merely asserted
|
|
66
|
+
* from inference. Mirrors the "verify" stage of the fail → investigate →
|
|
67
|
+
* verify → distill → consult self-improvement progression: a lesson that
|
|
68
|
+
* skips verification is still worth capturing, but downstream consumers
|
|
69
|
+
* (e.g. `stats()`) should be able to tell the two apart.
|
|
70
|
+
*/
|
|
71
|
+
verified?: boolean;
|
|
72
|
+
/** The short check that was run to establish `verified` (a command, a file path, a test name). Ignored when `verified` is not `true`. */
|
|
73
|
+
evidence?: string;
|
|
63
74
|
}
|
|
64
75
|
|
|
65
76
|
export interface MemoryBackendSaveResult {
|
|
@@ -20,6 +20,7 @@ Treat the design system as the foundation — UI built without one collapses int
|
|
|
20
20
|
2. **No coherent system? Build the minimal one first.** Extract what exists, then define a palette, type scale, spacing scale (4px/8px base), radii/shadows/transitions, and primitive components — THEN implement the request against it.
|
|
21
21
|
3. **Compose with the system, never around it.** Colors → tokens/CSS variables, never hardcoded hex; spacing → scale values, never arbitrary px; type → scale steps; components → extend/compose existing primitives, not one-off div soup. Need something outside the system? Add the new token to the system first, then use it — never a one-off override.
|
|
22
22
|
4. **Verify before done.** Every color a token, every spacing on the scale, every component on the existing composition pattern, zero magic numbers — a designer would see consistency across old and new. Any "no" → not done.
|
|
23
|
+
5. **Visually verify the rendered result.** Code-level checks (tokens/spacing) don't catch layout/rendering bugs. After producing a UI change, capture it (`tab.screenshot()` for browser-rendered UI, or the existing screenshot path) and call `vision_verify` with the task's stated goal — pass `baseline` when checking for regression against a known-good prior state.
|
|
23
24
|
</design-system>
|
|
24
25
|
|
|
25
26
|
<procedure>
|
|
@@ -5,3 +5,5 @@ Use after solving something whose insight will pay off again: a non-obvious fix,
|
|
|
5
5
|
Provide the optional `skill` object when the lesson is a repeatable *procedure* worth codifying as a `SKILL.md` (not just a fact). Managed skills are written to an isolated directory (`~/.jeopi/agent/managed-skills`) and are surfaced like normal skills next session. They NEVER touch user-authored skills. Frontmatter is generated from `name` and `description`.
|
|
6
6
|
|
|
7
7
|
Capture sparingly and specifically. One strong, reusable lesson beats several vague ones.
|
|
8
|
+
|
|
9
|
+
Set `verified: true` when you independently re-checked the claim against the repo — you ran a command, read a file, or ran a test that directly confirmed it — and pass a short `evidence` string naming that check (e.g. "ran `bun test src/foo.test.ts`, all 12 pass" or "read `package.json:12`, confirms dependency pin"). Leave `verified` unset (the default) for a hypothesis or pattern you noticed but did not independently confirm.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
You are a UI/artifact verification assistant.
|
|
2
|
+
|
|
3
|
+
Core behavior:
|
|
4
|
+
- Be evidence-first: judge only what is visibly present in the image(s).
|
|
5
|
+
- If something is unclear or partially occluded, treat it as unmet rather than guessing it is fine.
|
|
6
|
+
- NEVER fabricate details that are not visible.
|
|
7
|
+
- When a baseline (prior) screenshot is provided, compare the current screenshot against it and treat any visible divergence as a gap unless the goal explicitly calls for that change.
|
|
8
|
+
|
|
9
|
+
Output contract — respond with ONLY a single JSON object, no prose before or after it, no markdown code fence, matching exactly:
|
|
10
|
+
{"matches": boolean, "summary": string, "gaps": string[]}
|
|
11
|
+
|
|
12
|
+
- `matches`: true only if the current screenshot fully satisfies the stated goal (and, when a baseline is given, shows no unintended regression from it).
|
|
13
|
+
- `summary`: one to three sentences describing what you observed.
|
|
14
|
+
- `gaps`: specific, concrete differences between what is observed and what the goal (or baseline) requires. Empty array when `matches` is true.
|
|
15
|
+
|
|
16
|
+
Return raw JSON only — any other format is a failure to follow instructions.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Verifies a screenshot or UI artifact against a stated goal with a vision-capable model and returns a structured verdict.
|
|
2
|
+
|
|
3
|
+
<instruction>
|
|
4
|
+
- Use this after producing or modifying a UI to check the result against the task's stated goal
|
|
5
|
+
- Provide `screenshot` as a local image file path, `Image #N` attachment label, or `attachment://N` URI
|
|
6
|
+
- Write a specific `goal`: what the UI/artifact should look like or achieve
|
|
7
|
+
- Pass `baseline` (a path to a prior screenshot of the same view) to check for visual regression against a known-good state
|
|
8
|
+
- Use this tool over `inspect_image` when you need a pass/fail verdict with structured gaps, not free-text description
|
|
9
|
+
</instruction>
|
|
10
|
+
|
|
11
|
+
<output>
|
|
12
|
+
- Returns a structured verdict: `matches` (boolean), `summary`, and `gaps` (empty when `matches` is true)
|
|
13
|
+
- No image content blocks are returned in tool output
|
|
14
|
+
</output>
|
|
15
|
+
|
|
16
|
+
<critical>
|
|
17
|
+
- If image submission is blocked by settings, the tool will fail with an actionable error
|
|
18
|
+
- If configured model does not support image input, configure a vision-capable model role before retrying
|
|
19
|
+
- If the model's response cannot be parsed as the expected JSON shape, the tool fails with the raw response included — retry rather than trust an unstructured fallback
|
|
20
|
+
</critical>
|
|
@@ -43,6 +43,7 @@ import { createMarketplaceManager } from "./helpers/marketplace-manager";
|
|
|
43
43
|
import { handleMcpAcp } from "./helpers/mcp";
|
|
44
44
|
import { commandConsumed, errorMessage, parseSlashCommand, parseSubcommand, usage } from "./helpers/parse";
|
|
45
45
|
import { describeRedeemOutcome, type ResetUsageAccount, toResetUsageAccounts } from "./helpers/reset-usage";
|
|
46
|
+
import { handleSkillCommand } from "./helpers/skill-verify";
|
|
46
47
|
import { handleSshAcp } from "./helpers/ssh";
|
|
47
48
|
import { launchStatsDashboard, parseStatsDashboardArgs } from "./helpers/stats-dashboard";
|
|
48
49
|
import { handleTodoAcp } from "./helpers/todo";
|
|
@@ -1348,6 +1349,14 @@ const BUILTIN_SLASH_COMMAND_REGISTRY: ReadonlyArray<SlashCommandSpec> = [
|
|
|
1348
1349
|
await runtime.ctx.handleSSHCommand(command.text);
|
|
1349
1350
|
},
|
|
1350
1351
|
},
|
|
1352
|
+
{
|
|
1353
|
+
name: "verify-skill",
|
|
1354
|
+
description: "Run a skill's `## Eval Cases` and report pass/fail",
|
|
1355
|
+
acpDescription: "Verify a skill's eval cases",
|
|
1356
|
+
inlineHint: "<name>",
|
|
1357
|
+
allowArgs: true,
|
|
1358
|
+
handle: handleSkillCommand,
|
|
1359
|
+
},
|
|
1351
1360
|
{
|
|
1352
1361
|
name: "new",
|
|
1353
1362
|
priority: 96,
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `/verify-skill <name>` — self-verification for a skill's `## Eval Cases`.
|
|
3
|
+
*
|
|
4
|
+
* Resolves a skill from the active session, extracts a bulleted `## Eval
|
|
5
|
+
* Cases` markdown section from its `SKILL.md` body (if present), and spawns
|
|
6
|
+
* one read-only `critic` subagent per case to verdict whether the claim still
|
|
7
|
+
* holds against the current repo state. Renders a pass/fail markdown report;
|
|
8
|
+
* never mutates the skill file.
|
|
9
|
+
*
|
|
10
|
+
* Named `verify-skill` (not `skill`) to avoid colliding with the pre-existing
|
|
11
|
+
* `/skill:<name>` direct-skill-invocation convention (both parse to the
|
|
12
|
+
* command-name token `skill` under `parseSlashCommand`'s earliest-whitespace-
|
|
13
|
+
* or-colon splitting rule — see slash-commands/helpers/parse.ts).
|
|
14
|
+
*/
|
|
15
|
+
import { MANAGED_SKILLS_PROVIDER_ID } from "../../autolearn/managed-skills";
|
|
16
|
+
import { runEvalAgent } from "../../eval/agent-bridge";
|
|
17
|
+
import { getBundledAgent } from "../../task/agents";
|
|
18
|
+
import * as taskDiscovery from "../../task/discovery";
|
|
19
|
+
import type { ToolSession } from "../../tools";
|
|
20
|
+
import type { ParsedSlashCommand, SlashCommandResult, SlashCommandRuntime } from "../types";
|
|
21
|
+
import { commandConsumed, usage } from "./parse";
|
|
22
|
+
|
|
23
|
+
const VERIFIER_AGENT_NAME = "critic";
|
|
24
|
+
|
|
25
|
+
interface EvalCaseVerdict {
|
|
26
|
+
claim: string;
|
|
27
|
+
passed: boolean;
|
|
28
|
+
reason: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Extract the bulleted list under a `## Eval Cases` heading, if present. */
|
|
32
|
+
export function extractEvalCases(skillBody: string): string[] {
|
|
33
|
+
const headingMatch = /^##\s+Eval Cases\s*$/m.exec(skillBody);
|
|
34
|
+
if (!headingMatch) return [];
|
|
35
|
+
const sectionStart = headingMatch.index + headingMatch[0].length;
|
|
36
|
+
const rest = skillBody.slice(sectionStart);
|
|
37
|
+
const nextHeadingMatch = /^#{1,6}\s+\S/m.exec(rest);
|
|
38
|
+
const sectionBody = nextHeadingMatch ? rest.slice(0, nextHeadingMatch.index) : rest;
|
|
39
|
+
const cases: string[] = [];
|
|
40
|
+
for (const line of sectionBody.split("\n")) {
|
|
41
|
+
const bulletMatch = /^\s*[-*+]\s+(.+)$/.exec(line);
|
|
42
|
+
if (bulletMatch?.[1]) cases.push(bulletMatch[1].trim());
|
|
43
|
+
}
|
|
44
|
+
return cases;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Verdict a single eval-case claim against the current repo state via the bundled `critic` agent. */
|
|
48
|
+
async function verifyEvalCase(session: ToolSession, claim: string): Promise<EvalCaseVerdict> {
|
|
49
|
+
const verifyPrompt =
|
|
50
|
+
`Verify this claim against the current repository state: ${claim}\n\n` +
|
|
51
|
+
"Report whether it still holds (yes/no) and why, citing files/commands.";
|
|
52
|
+
// Explicit schema, not the agent's own frontmatter default: runEvalAgent (the
|
|
53
|
+
// eval `agent()` bridge) only applies `outputSchema` when the caller supplies
|
|
54
|
+
// one — it never falls back to the agent's `output` frontmatter on its own
|
|
55
|
+
// (see goals/verifier.ts for the same pattern). Read the bundled definition's
|
|
56
|
+
// declared shape so this can't drift from critic.md.
|
|
57
|
+
const schema = getBundledAgent(VERIFIER_AGENT_NAME)?.output;
|
|
58
|
+
const args =
|
|
59
|
+
schema === undefined
|
|
60
|
+
? { prompt: verifyPrompt, agent: VERIFIER_AGENT_NAME }
|
|
61
|
+
: { prompt: verifyPrompt, agent: VERIFIER_AGENT_NAME, schema };
|
|
62
|
+
try {
|
|
63
|
+
const result = await runEvalAgent(args, { session });
|
|
64
|
+
let verdict: unknown;
|
|
65
|
+
try {
|
|
66
|
+
verdict = JSON.parse(result.text);
|
|
67
|
+
} catch {
|
|
68
|
+
// Freeform fallback: no parseable JSON, treat non-empty output as inconclusive-pass with raw text as reason.
|
|
69
|
+
return { claim, passed: true, reason: result.text.trim().slice(0, 300) || "No structured verdict returned." };
|
|
70
|
+
}
|
|
71
|
+
if (verdict && typeof verdict === "object") {
|
|
72
|
+
const record = verdict as Record<string, unknown>;
|
|
73
|
+
const rawVerdict = record.verdict;
|
|
74
|
+
const passed = rawVerdict === "okay";
|
|
75
|
+
const reason =
|
|
76
|
+
(typeof record.summary === "string" && record.summary) ||
|
|
77
|
+
(typeof record.justification === "string" && record.justification) ||
|
|
78
|
+
"No justification returned.";
|
|
79
|
+
return { claim, passed, reason: reason.slice(0, 400) };
|
|
80
|
+
}
|
|
81
|
+
return { claim, passed: false, reason: "Verifier returned an unparseable verdict." };
|
|
82
|
+
} catch (err) {
|
|
83
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
84
|
+
return { claim, passed: false, reason: `Verification subagent failed: ${message}` };
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function renderReport(skillName: string, isManaged: boolean, verdicts: EvalCaseVerdict[]): string {
|
|
89
|
+
const lines: string[] = [`## Skill verification: \`${skillName}\``, ""];
|
|
90
|
+
const failures: EvalCaseVerdict[] = [];
|
|
91
|
+
for (const verdict of verdicts) {
|
|
92
|
+
lines.push(`- ${verdict.passed ? "✅ PASS" : "❌ FAIL"} — ${verdict.claim}`);
|
|
93
|
+
lines.push(` ${verdict.reason}`);
|
|
94
|
+
if (!verdict.passed) failures.push(verdict);
|
|
95
|
+
}
|
|
96
|
+
const passCount = verdicts.length - failures.length;
|
|
97
|
+
lines.push("", `${passCount}/${verdicts.length} eval cases passed.`);
|
|
98
|
+
if (failures.length > 0 && isManaged) {
|
|
99
|
+
lines.push(
|
|
100
|
+
"",
|
|
101
|
+
"This is a managed (auto-learn) skill. To record the failure mode, append a line like this under a `## Known Failure Modes` section (not applied automatically):",
|
|
102
|
+
"",
|
|
103
|
+
);
|
|
104
|
+
for (const failure of failures) {
|
|
105
|
+
lines.push(`- "${failure.claim}" — ${failure.reason}`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return lines.join("\n");
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** `/verify-skill <name>` handler — resolves a skill, runs its `## Eval Cases`, reports pass/fail. */
|
|
112
|
+
export async function handleSkillCommand(
|
|
113
|
+
command: ParsedSlashCommand,
|
|
114
|
+
runtime: SlashCommandRuntime,
|
|
115
|
+
): Promise<SlashCommandResult> {
|
|
116
|
+
const skillName = command.args.trim();
|
|
117
|
+
if (!skillName) {
|
|
118
|
+
return usage("Usage: /verify-skill <name>", runtime);
|
|
119
|
+
}
|
|
120
|
+
const skill = runtime.session.skills.find(candidate => candidate.name === skillName);
|
|
121
|
+
if (!skill) {
|
|
122
|
+
const available = runtime.session.skills.map(candidate => candidate.name).join(", ") || "none";
|
|
123
|
+
return usage(`Unknown skill "${skillName}". Available: ${available}`, runtime);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const body = await Bun.file(skill.filePath).text();
|
|
127
|
+
const evalCases = extractEvalCases(body);
|
|
128
|
+
if (evalCases.length === 0) {
|
|
129
|
+
await runtime.output(
|
|
130
|
+
`No eval cases defined for this skill — add an \`## Eval Cases\` section with bulleted verifiable claims to enable self-verification`,
|
|
131
|
+
);
|
|
132
|
+
return commandConsumed();
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
await runtime.output(`Verifying ${evalCases.length} eval case(s) for \`${skillName}\`…`);
|
|
136
|
+
|
|
137
|
+
// Discover once, upfront, to fail fast with a clear message if the critic
|
|
138
|
+
// agent is unavailable (disabled, or bundled agents failed to load) rather
|
|
139
|
+
// than surfacing N identical spawn errors.
|
|
140
|
+
const { agents } = await taskDiscovery.discoverAgents(runtime.cwd);
|
|
141
|
+
if (!taskDiscovery.getAgent(agents, VERIFIER_AGENT_NAME)) {
|
|
142
|
+
return usage(
|
|
143
|
+
`The "${VERIFIER_AGENT_NAME}" agent is unavailable in this session; cannot run eval cases.`,
|
|
144
|
+
runtime,
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const session: ToolSession = {
|
|
149
|
+
cwd: runtime.cwd,
|
|
150
|
+
hasUI: false,
|
|
151
|
+
settings: runtime.settings,
|
|
152
|
+
skills: [...runtime.session.skills],
|
|
153
|
+
taskDepth: 0,
|
|
154
|
+
getSessionFile: () => runtime.session.sessionManager.getSessionFile() ?? null,
|
|
155
|
+
getSessionSpawns: () => "*",
|
|
156
|
+
getPlanModeState: () => runtime.session.getPlanModeState(),
|
|
157
|
+
getActiveModelString: () =>
|
|
158
|
+
runtime.session.model ? `${runtime.session.model.provider}/${runtime.session.model.id}` : undefined,
|
|
159
|
+
modelRegistry: runtime.session.modelRegistry,
|
|
160
|
+
authStorage: runtime.session.modelRegistry.authStorage,
|
|
161
|
+
getAgentId: () => runtime.session.getAgentId() ?? null,
|
|
162
|
+
};
|
|
163
|
+
const verdicts: EvalCaseVerdict[] = [];
|
|
164
|
+
for (const claim of evalCases) {
|
|
165
|
+
verdicts.push(await verifyEvalCase(session, claim));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const isManaged = skill._source?.provider === MANAGED_SKILLS_PROVIDER_ID;
|
|
169
|
+
await runtime.output(renderReport(skillName, isManaged, verdicts));
|
|
170
|
+
return commandConsumed();
|
|
171
|
+
}
|
package/src/task/index.ts
CHANGED
|
@@ -67,6 +67,7 @@ import {
|
|
|
67
67
|
prepareIsolationContext,
|
|
68
68
|
runIsolatedSubprocess,
|
|
69
69
|
} from "./isolation-runner";
|
|
70
|
+
import { classifyDefaultTaskModelTier } from "./model-tier-classifier";
|
|
70
71
|
import { generateTaskName } from "./name-generator";
|
|
71
72
|
import { AgentOutputManager } from "./output-manager";
|
|
72
73
|
import { mapWithConcurrencyLimit, normalizeConcurrencyLimit, Semaphore } from "./parallel";
|
|
@@ -1202,7 +1203,7 @@ export class TaskTool implements AgentTool<TaskToolSchemaInstance, TaskToolDetai
|
|
|
1202
1203
|
const agentModelOverrides = this.session.settings.get("task.agentModelOverrides");
|
|
1203
1204
|
const settingsModelOverride = agentModelOverrides[agentName];
|
|
1204
1205
|
const parentActiveModelPattern = this.session.getActiveModelString?.();
|
|
1205
|
-
|
|
1206
|
+
let modelOverride = resolveAgentModelPatterns({
|
|
1206
1207
|
settingsOverride: settingsModelOverride,
|
|
1207
1208
|
agentModel: effectiveAgent.model,
|
|
1208
1209
|
settings: this.session.settings,
|
|
@@ -1211,6 +1212,30 @@ export class TaskTool implements AgentTool<TaskToolSchemaInstance, TaskToolDetai
|
|
|
1211
1212
|
});
|
|
1212
1213
|
const thinkingLevelOverride = effectiveAgent.thinkingLevel;
|
|
1213
1214
|
|
|
1215
|
+
// Auto-tier the generic default `task` agent to the cheap `smol` model
|
|
1216
|
+
// when the assignment classifies as trivially simple and nothing else
|
|
1217
|
+
// (settings override) already pinned the model. Named specialists carry
|
|
1218
|
+
// their own `model:` frontmatter, so gating on the literal "task" agent
|
|
1219
|
+
// name naturally excludes them without inspecting frontmatter here.
|
|
1220
|
+
const autoModel = this.session.getActiveModel?.();
|
|
1221
|
+
const autoModelRegistry = this.session.modelRegistry;
|
|
1222
|
+
if (
|
|
1223
|
+
agentName === "task" &&
|
|
1224
|
+
settingsModelOverride === undefined &&
|
|
1225
|
+
this.session.settings.get("task.autoModelTier") &&
|
|
1226
|
+
autoModel &&
|
|
1227
|
+
autoModelRegistry
|
|
1228
|
+
) {
|
|
1229
|
+
const tier = await classifyDefaultTaskModelTier(assignment, {
|
|
1230
|
+
settings: this.session.settings,
|
|
1231
|
+
registry: autoModelRegistry,
|
|
1232
|
+
model: autoModel,
|
|
1233
|
+
sessionId: this.session.getSessionId?.() ?? undefined,
|
|
1234
|
+
signal,
|
|
1235
|
+
});
|
|
1236
|
+
if (tier === "smol") modelOverride = ["pi/smol"];
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1214
1239
|
// Output schema priority: agent frontmatter > inherited parent session.
|
|
1215
1240
|
// The task call itself never carries a schema; workflows needing ad-hoc
|
|
1216
1241
|
// structured output go through eval agent(prompt, schema).
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default `task`-agent spawn model-tier classifier.
|
|
3
|
+
*
|
|
4
|
+
* Reuses the auto-thinking difficulty classifier ({@link classifyDifficulty})
|
|
5
|
+
* to decide whether a `task()` spawn's assignment is trivially simple enough
|
|
6
|
+
* to route to the cheap `smol` model tier instead of the default `task`
|
|
7
|
+
* tier. Only ever suggests a downgrade — never blocks or upgrades — and
|
|
8
|
+
* fails open (returns `undefined`, meaning "don't override") on any
|
|
9
|
+
* classification error or timeout, mirroring `AgentSession`'s
|
|
10
|
+
* `#applyAutoThinkingLevel` fallback behavior.
|
|
11
|
+
*/
|
|
12
|
+
import { Effort } from "jeopi-ai";
|
|
13
|
+
import { logger } from "jeopi-utils";
|
|
14
|
+
import { type ClassifyDifficultyDeps, classifyDifficulty } from "../auto-thinking/classifier";
|
|
15
|
+
|
|
16
|
+
/** Timeout (ms) for the spawn-time model-tier classification before failing open. */
|
|
17
|
+
const MODEL_TIER_CLASSIFICATION_TIMEOUT_MS = 2000;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Classify `assignment` and return `"smol"` when the work is trivially
|
|
21
|
+
* simple ({@link Effort.Minimal} or {@link Effort.Low}), or `undefined` when
|
|
22
|
+
* the caller should leave normal model resolution in place (not-simple
|
|
23
|
+
* enough, no classifier available, or the classification failed/timed out).
|
|
24
|
+
*/
|
|
25
|
+
export async function classifyDefaultTaskModelTier(
|
|
26
|
+
assignment: string,
|
|
27
|
+
deps: ClassifyDifficultyDeps,
|
|
28
|
+
): Promise<"smol" | undefined> {
|
|
29
|
+
const controller = new AbortController();
|
|
30
|
+
const timer = setTimeout(() => controller.abort(), MODEL_TIER_CLASSIFICATION_TIMEOUT_MS);
|
|
31
|
+
const signal = deps.signal ? AbortSignal.any([deps.signal, controller.signal]) : controller.signal;
|
|
32
|
+
try {
|
|
33
|
+
const effort = await classifyDifficulty(assignment, { ...deps, signal });
|
|
34
|
+
return effort === Effort.Minimal || effort === Effort.Low ? "smol" : undefined;
|
|
35
|
+
} catch (error) {
|
|
36
|
+
logger.debug("task: model-tier classification failed; leaving model resolution unchanged", {
|
|
37
|
+
error: error instanceof Error ? error.message : String(error),
|
|
38
|
+
});
|
|
39
|
+
return undefined;
|
|
40
|
+
} finally {
|
|
41
|
+
clearTimeout(timer);
|
|
42
|
+
}
|
|
43
|
+
}
|
package/src/tools/index.ts
CHANGED
|
@@ -66,6 +66,7 @@ import { SearchToolBm25Tool } from "./search-tool-bm25";
|
|
|
66
66
|
import { loadSshTool } from "./ssh";
|
|
67
67
|
import { isSubagentToolEnabled, SubagentTool } from "./subagent";
|
|
68
68
|
import { type TodoPhase, TodoTool } from "./todo";
|
|
69
|
+
import { VisionVerifyTool } from "./vision-verify";
|
|
69
70
|
import { WriteTool } from "./write";
|
|
70
71
|
import { YieldTool } from "./yield";
|
|
71
72
|
|
|
@@ -106,6 +107,7 @@ export * from "./ssh";
|
|
|
106
107
|
export * from "./subagent";
|
|
107
108
|
export * from "./todo";
|
|
108
109
|
export * from "./tts";
|
|
110
|
+
export * from "./vision-verify";
|
|
109
111
|
export * from "./write";
|
|
110
112
|
export * from "./yield";
|
|
111
113
|
|
|
@@ -466,6 +468,7 @@ export const BUILTIN_TOOLS: Record<BuiltinToolName, ToolFactory> = {
|
|
|
466
468
|
grep: s => new GrepTool(s),
|
|
467
469
|
lsp: LspTool.createIf,
|
|
468
470
|
inspect_image: s => new InspectImageTool(s),
|
|
471
|
+
vision_verify: s => new VisionVerifyTool(s),
|
|
469
472
|
browser: s => new BrowserTool(s),
|
|
470
473
|
checkpoint: CheckpointTool.createIf,
|
|
471
474
|
rewind: RewindTool.createIf,
|
|
@@ -621,6 +624,7 @@ export async function createTools(session: ToolSession, toolNames?: string[]): P
|
|
|
621
624
|
if (name === "ast_grep") return session.settings.get("astGrep.enabled");
|
|
622
625
|
if (name === "ast_edit") return session.settings.get("astEdit.enabled");
|
|
623
626
|
if (name === "inspect_image") return session.settings.get("inspect_image.enabled");
|
|
627
|
+
if (name === "vision_verify") return session.settings.get("vision_verify.enabled");
|
|
624
628
|
if (name === "web_search") return session.settings.get("web_search.enabled");
|
|
625
629
|
// search_tool_bm25 is allowed when either legacy mcp.discoveryMode or new tools.discoveryMode is active.
|
|
626
630
|
if (name === "search_tool_bm25") return discoveryActive;
|
package/src/tools/learn.ts
CHANGED
|
@@ -9,6 +9,12 @@ import type { ToolSession } from ".";
|
|
|
9
9
|
const learnSchema = type({
|
|
10
10
|
memory: type("string").describe("the durable, self-contained lesson to remember (what, when, why)"),
|
|
11
11
|
"context?": type("string").describe("optional source context for the lesson"),
|
|
12
|
+
"verified?": type("boolean").describe(
|
|
13
|
+
"true only if you independently re-checked this claim against the repo (ran a command, read a file, ran a test) rather than merely inferring it",
|
|
14
|
+
),
|
|
15
|
+
"evidence?": type("string").describe(
|
|
16
|
+
"the short check that confirmed the claim (a command, a file path, a test name); only meaningful when verified is true",
|
|
17
|
+
),
|
|
12
18
|
"skill?": type({
|
|
13
19
|
action: "'create' | 'update'",
|
|
14
20
|
name: type("string").describe("kebab-case skill name"),
|
|
@@ -65,6 +71,8 @@ export class LearnTool implements AgentTool<typeof learnSchema> {
|
|
|
65
71
|
cwd: state.session.sessionManager.getCwd(),
|
|
66
72
|
context: params.context ?? null,
|
|
67
73
|
tool: "learn",
|
|
74
|
+
verified: params.verified ?? false,
|
|
75
|
+
evidence: params.evidence ?? null,
|
|
68
76
|
},
|
|
69
77
|
scope: "bank",
|
|
70
78
|
extract: true,
|
|
@@ -81,7 +89,14 @@ export class LearnTool implements AgentTool<typeof learnSchema> {
|
|
|
81
89
|
} else if (backend === "local") {
|
|
82
90
|
const result = await localBackend.save?.(
|
|
83
91
|
{ agentDir: this.session.settings.getAgentDir(), cwd: this.session.settings.getCwd() },
|
|
84
|
-
{
|
|
92
|
+
{
|
|
93
|
+
content: params.memory,
|
|
94
|
+
context: params.context,
|
|
95
|
+
source: "coding-agent-learn",
|
|
96
|
+
importance: 0.8,
|
|
97
|
+
verified: params.verified,
|
|
98
|
+
evidence: params.evidence,
|
|
99
|
+
},
|
|
85
100
|
);
|
|
86
101
|
if (!result || result.stored === 0) {
|
|
87
102
|
throw new Error("Lesson was empty after sanitization; nothing stored.");
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { WritethroughCallback } from "../lsp";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Non-isolated concurrent subagents (`task.isolation.mode="none"`, the
|
|
5
|
+
* default) share the parent session's cwd with no other collision-avoidance
|
|
6
|
+
* mechanism: two sibling subagents editing the same file can each perform an
|
|
7
|
+
* unguarded read-modify-write, and the interleaving of their disk writes can
|
|
8
|
+
* silently drop one of them. This wrapper does not prevent LOGICAL conflicts
|
|
9
|
+
* (two agents making incompatible edits to the same file) — it only
|
|
10
|
+
* serializes the physical writethrough call per destination path, so one
|
|
11
|
+
* write can no longer be lost mid-flight to another write racing the same
|
|
12
|
+
* path.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/** Per-path write chains serializing the writethrough callback for a given destination. */
|
|
16
|
+
const pathWriteChains = new Map<string, Promise<unknown>>();
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Wrap a {@link WritethroughCallback} so concurrent writes to the same `dst`
|
|
20
|
+
* are serialized instead of racing. `dst` is used as-is for the chain key:
|
|
21
|
+
* every caller (edit's hashline/patch/replace paths, and the write tool)
|
|
22
|
+
* resolves `dst` to an absolute path via `resolvePlanPath`/`resolveToCwd`
|
|
23
|
+
* before invoking the writethrough, so the same logical file always arrives
|
|
24
|
+
* under the same string.
|
|
25
|
+
*/
|
|
26
|
+
export function withPathWriteLock(callback: WritethroughCallback): WritethroughCallback {
|
|
27
|
+
return async (dst, content, signal, file, batch, getDeferred) => {
|
|
28
|
+
// Serialize the writethrough per path: parallel edits to the same file
|
|
29
|
+
// (sibling subagents sharing a cwd, or two shared tool calls in one
|
|
30
|
+
// turn) would otherwise let concurrent disk writes race and drop one.
|
|
31
|
+
const run = (pathWriteChains.get(dst) ?? Promise.resolve()).then(() =>
|
|
32
|
+
callback(dst, content, signal, file, batch, getDeferred),
|
|
33
|
+
);
|
|
34
|
+
const guarded = run.catch(() => {});
|
|
35
|
+
pathWriteChains.set(dst, guarded);
|
|
36
|
+
try {
|
|
37
|
+
return await run;
|
|
38
|
+
} finally {
|
|
39
|
+
// Drop the entry once this write is the chain tail, so the map does
|
|
40
|
+
// not retain one promise per distinct path for the process lifetime.
|
|
41
|
+
if (pathWriteChains.get(dst) === guarded) pathWriteChains.delete(dst);
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
}
|