jeo-code 0.8.22 → 0.8.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.ja.md +19 -3
- package/README.ko.md +19 -3
- package/README.md +19 -3
- package/README.zh.md +19 -3
- package/package.json +1 -1
- package/src/agent/AGENTS.md +1 -1
- package/src/agent/browser-tool.ts +32 -3
- package/src/agent/engine.ts +80 -4
- package/src/agent/eval-tool.ts +433 -0
- package/src/agent/loop.ts +3 -2
- package/src/agent/memory.ts +24 -3
- package/src/agent/plan.ts +55 -0
- package/src/agent/prompt-router.ts +83 -4
- package/src/agent/skill-lessons.ts +316 -0
- package/src/agent/state.ts +26 -0
- package/src/agent/subagents.ts +7 -6
- package/src/agent/task-tool.ts +261 -72
- package/src/agent/vision-verify.ts +75 -0
- package/src/ai/model-catalog.ts +16 -0
- package/src/ai/model-manager.ts +11 -2
- package/src/ai/types.ts +4 -2
- package/src/cli/runner.ts +13 -1
- package/src/commands/launch.ts +197 -26
- package/src/commands/ralplan.ts +85 -15
- package/src/commands/routine.ts +249 -0
- package/src/commands/skills.ts +57 -0
- package/src/commands/team.ts +19 -0
- package/src/util/provider-error.ts +15 -0
- package/src/util/retry.ts +15 -4
- package/src/util/routine-template.ts +206 -0
|
@@ -48,7 +48,7 @@
|
|
|
48
48
|
* contract for users who never opted in.
|
|
49
49
|
*/
|
|
50
50
|
import { callLlm } from "./loop";
|
|
51
|
-
import { resolveRoleModel, modelServableWithConfig } from "../ai/model-manager";
|
|
51
|
+
import { resolveRoleModel, modelServableWithConfig, resolveProvider, oauthServesModel } from "../ai/model-manager";
|
|
52
52
|
import { catalogMetadata, MODEL_CATALOG, compareReleaseDate, liveProviderCatalogModels, isLiveProviderModel, type CatalogModel, type ThinkLevel } from "../ai/model-catalog";
|
|
53
53
|
import { priceForModel } from "../ai/pricing";
|
|
54
54
|
import { tryExtractJsonObject } from "./json";
|
|
@@ -222,6 +222,60 @@ function hasAntigravityOauth(config: RoutingConfig): boolean {
|
|
|
222
222
|
return !!(config.oauth?.antigravity || config.oauth?.gemini);
|
|
223
223
|
}
|
|
224
224
|
|
|
225
|
+
/** A model's classified credential-sharing SCOPE — models on the SAME scope share
|
|
226
|
+
* ONE account-wide rate-limit/billing window, so an account-level failure (429,
|
|
227
|
+
* usage/quota limit, credential rejection, billing block) on one model means every
|
|
228
|
+
* OTHER model in the same scope is doomed the same way THIS instant. `"oauth-subscription"`:
|
|
229
|
+
* an OAuth SUBSCRIPTION (Claude Pro/Max, ChatGPT/Codex, Kimi Code, Antigravity Cloud
|
|
230
|
+
* Code Assist) serves this model — `key` is `${provider}:oauth` (bare `"antigravity"`
|
|
231
|
+
* for that provider: all three re-exported companies route through the SAME Cloud
|
|
232
|
+
* Code Assist account/credential, confirmed no per-company quota separation exists).
|
|
233
|
+
* `"no-credential"`: the provider has ZERO usable credential of any kind — every
|
|
234
|
+
* model on it is equally dead. */
|
|
235
|
+
export type CredentialScope = { kind: "no-credential" | "oauth-subscription"; key: string };
|
|
236
|
+
|
|
237
|
+
/** Classify `model` into the credential it actually resolves through, mirroring
|
|
238
|
+
* `resolveCall`/`effectiveCredentialForProvider`'s real precedence (OAuth wins over
|
|
239
|
+
* a configured API key whenever it serves the model) WITHOUT touching the network —
|
|
240
|
+
* config-only, so this stays safely callable from a synchronous fast-path predicate
|
|
241
|
+
* (e.g. the engine's per-retry `rateLimitFallbackAvailable` check).
|
|
242
|
+
*
|
|
243
|
+
* `null` = API-key-served (or keyless/local) — an API key is typically its OWN
|
|
244
|
+
* independent per-key/per-model budget (Groq/OpenAI/etc rate-limit each model
|
|
245
|
+
* independently, unlike an OAuth subscription's single shared window), so a `null`
|
|
246
|
+
* candidate is NEVER excluded as a group — only its exact model id is excluded on
|
|
247
|
+
* failure, preserving the intentionally-tested "same-provider API-key fallback"
|
|
248
|
+
* behavior (see test/launch-fallback-live-discovery.test.ts). */
|
|
249
|
+
export function credentialScopeFor(model: string, config: RoutingConfig): CredentialScope | null {
|
|
250
|
+
const provider = resolveProvider(model);
|
|
251
|
+
if (provider === "ollama" || provider === "lmstudio") return null; // keyless, no shared account limit
|
|
252
|
+
if (provider === "antigravity") {
|
|
253
|
+
const hasOauth = !!(config.oauth?.antigravity || config.oauth?.gemini);
|
|
254
|
+
return hasOauth ? { kind: "oauth-subscription", key: "antigravity" } : { kind: "no-credential", key: "antigravity" };
|
|
255
|
+
}
|
|
256
|
+
const auth = provider as AuthProvider;
|
|
257
|
+
const hasApiKey = !!config.providers?.[auth];
|
|
258
|
+
const hasOauth = !!config.oauth?.[auth];
|
|
259
|
+
if (!hasApiKey && !hasOauth) return { kind: "no-credential", key: provider };
|
|
260
|
+
// OAuth wins over a configured API key whenever it actually serves this model
|
|
261
|
+
// (same precedence resolveCall/effectiveCredentialForProvider apply live) — that
|
|
262
|
+
// is the credential this call will really use, and its subscription window is
|
|
263
|
+
// what a 429 here reports on. When OAuth is present but does NOT serve this
|
|
264
|
+
// model (e.g. a non-Codex id under OpenAI OAuth), the call falls through to the
|
|
265
|
+
// API key (if any) — API-key-served, so `null` (independent budget).
|
|
266
|
+
if (hasOauth && oauthServesModel(auth, model)) return { kind: "oauth-subscription", key: `${provider}:oauth` };
|
|
267
|
+
// Defense-in-depth (currently unreachable in practice — every OAuth provider
|
|
268
|
+
// whose token doesn't serve a given model AND has no verified end-to-end backend
|
|
269
|
+
// is already excluded upstream from every auto-select pool, per
|
|
270
|
+
// isCoreEligible/hasAntigravityOauth): OAuth is configured, has NO API key
|
|
271
|
+
// fallback, and does not actually serve this model (e.g. a bare `gemini`-provider
|
|
272
|
+
// model under gemini-OAuth-only, since gemini's OAuth backend is not verified
|
|
273
|
+
// end-to-end) — genuinely zero usable credential for this specific model, not an
|
|
274
|
+
// independent API-key budget.
|
|
275
|
+
if (hasOauth && !hasApiKey) return { kind: "no-credential", key: provider };
|
|
276
|
+
return null;
|
|
277
|
+
}
|
|
278
|
+
|
|
225
279
|
/** Builds the routing catalog once per call: static `MODEL_CATALOG` rows need the
|
|
226
280
|
* full `isAutoSelectCandidate` gate (including the `isLiveProviderModel` check,
|
|
227
281
|
* when `openaiBaseUrl` is set), but rows already returned by
|
|
@@ -373,11 +427,15 @@ export function selectFromPool(pool: readonly string[], sessionId: string | unde
|
|
|
373
427
|
* model has a known price (caller falls back to `defaultModel`). Tiebreak on
|
|
374
428
|
* an exact cost tie: NEWER `releaseDate` wins (a same-priced newer model is
|
|
375
429
|
* likely a refined successor), then canonical id (deterministic fallback
|
|
376
|
-
* when recency also ties or is unconfirmed on both sides).
|
|
377
|
-
|
|
430
|
+
* when recency also ties or is unconfirmed on both sides). Optional `filter`
|
|
431
|
+
* narrows the candidate pool BEFORE price comparison (e.g. vision-capable
|
|
432
|
+
* only, matching `strongestCredentialed`'s filter param) — a cheap model that
|
|
433
|
+
* fails the filter is never picked over a pricier one that passes it. */
|
|
434
|
+
export function cheapestCredentialed(config: RoutingConfig, filter?: (m: CatalogModel) => boolean): string | null {
|
|
378
435
|
let best: CatalogModel | null = null;
|
|
379
436
|
let bestCost = Infinity;
|
|
380
437
|
for (const m of routingCatalog(config)) {
|
|
438
|
+
if (filter && !filter(m)) continue;
|
|
381
439
|
// `routingCatalog(config)` already filtered to eligible rows (static rows via
|
|
382
440
|
// `isAutoSelectCandidate`, live rows via `isAutoSelectCandidateLive`) — no need
|
|
383
441
|
// to re-check here. See cheapestCredentialed's own doc comment for the
|
|
@@ -406,6 +464,27 @@ export function cheapestCredentialed(config: RoutingConfig): string | null {
|
|
|
406
464
|
return best?.canonical ?? null;
|
|
407
465
|
}
|
|
408
466
|
|
|
467
|
+
/** Cheap-tier model for a GRADER/VERIFIER call (goal-verifier, vision-verify,
|
|
468
|
+
* `critic` role) — the blog-post "Haiku-for-graders" principle: a verdict call
|
|
469
|
+
* reads an artifact + a rubric, it does not need frontier reasoning strength,
|
|
470
|
+
* and defaulting graders to the SAME expensive model driving the turn they are
|
|
471
|
+
* grading both wastes budget and (for self-graded turns) reintroduces the very
|
|
472
|
+
* self-preference bias an independent grader exists to avoid. Resolution order:
|
|
473
|
+
* `roles.smol` (explicit user pin, but only when it also satisfies `requireImages`)
|
|
474
|
+
* → live-cheapest credentialed (filtered the same way) → `defaultModel` (never
|
|
475
|
+
* null — every caller needs a concrete model id to pass to `callLlm`).
|
|
476
|
+
* `opts.requireImages` (the browser vision-verify caller) excludes catalog rows
|
|
477
|
+
* with `images: false` — a text-only cheap model silently ignores an attached
|
|
478
|
+
* screenshot rather than erroring, which would surface as a hallucinated verdict
|
|
479
|
+
* instead of a loud failure, so the filter runs BEFORE cost/pin selection, not after. */
|
|
480
|
+
export function resolveVerifierModel(config: RoutingConfig, opts: { requireImages?: boolean } = {}): string {
|
|
481
|
+
const pinned = config.roles?.smol;
|
|
482
|
+
const pinnedOk = pinned && (!opts.requireImages || catalogMetadata(pinned)?.images !== false);
|
|
483
|
+
if (pinnedOk) return pinned!;
|
|
484
|
+
const filter = opts.requireImages ? (m: CatalogModel) => m.images !== false : undefined;
|
|
485
|
+
return cheapestCredentialed(config, filter) || config.defaultModel;
|
|
486
|
+
}
|
|
487
|
+
|
|
409
488
|
/** Most capable cloud model jeo has a credential for, computed LIVE off
|
|
410
489
|
* `MODEL_CATALOG` — ranks by full (xhigh) thinking support first, then max
|
|
411
490
|
* output tokens, then context window, then NEWER `releaseDate` (a same-specs
|
|
@@ -450,7 +529,7 @@ export function strongestCredentialed(
|
|
|
450
529
|
* a genuinely mid-class model like `claude-sonnet-4-6`. `null` when no
|
|
451
530
|
* mid-class-suffixed model is credentialed (caller falls through to
|
|
452
531
|
* `roles.medium`/`defaultModel` — never to `complex`'s flagship pick). */
|
|
453
|
-
function strongestMidTierCredentialed(config: RoutingConfig): string | null {
|
|
532
|
+
export function strongestMidTierCredentialed(config: RoutingConfig): string | null {
|
|
454
533
|
const midClass = routingCatalog(config).filter(
|
|
455
534
|
m => sizeClassFor(m) === "mid",
|
|
456
535
|
);
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Skill self-compounding: the durable link the bundled/user skill docs were
|
|
3
|
+
* missing. `recordFailedAttempt` (memory.ts) already writes stalled turns into
|
|
4
|
+
* OKF concept memory, but nothing ever fed that signal BACK into the skill
|
|
5
|
+
* files themselves — a skill that repeatedly leads an agent into the same dead
|
|
6
|
+
* end never learns to warn about it. This module closes that loop:
|
|
7
|
+
*
|
|
8
|
+
* - `matchLessonToSkill` deterministically (no LLM) maps free text to the
|
|
9
|
+
* bundled skill it most plausibly concerns, conservatively preferring
|
|
10
|
+
* `undefined` over a wrong guess.
|
|
11
|
+
* - `appendSkillLesson`/`recordSkillLesson` append a lesson bullet into a
|
|
12
|
+
* `## Known Failure Modes` or `## Anti-Patterns (do NOT do)` section of the
|
|
13
|
+
* project-level `.jeo/skills/<name>.md` file (seeded from the bundled
|
|
14
|
+
* skill on first write), idempotent on exact-title re-append.
|
|
15
|
+
* - `evalSkillLessons` is the self-eval half: ONE batched LLM call judges
|
|
16
|
+
* whether each recorded lesson is still "covered" by the skill's current
|
|
17
|
+
* guidance or has gone "stale" (guidance drifted, entry looks obsolete),
|
|
18
|
+
* so a human/agent can prune or fold lessons back into the skill body.
|
|
19
|
+
*/
|
|
20
|
+
import * as fs from "node:fs/promises";
|
|
21
|
+
import * as path from "node:path";
|
|
22
|
+
import { SKILLS, getSkill, bundledSkillFileContent } from "../skills/catalog";
|
|
23
|
+
import { callLlm } from "./loop";
|
|
24
|
+
import { resolveVerifierModel } from "./prompt-router";
|
|
25
|
+
import { readGlobalConfig, ensureJeoGitignore } from "./state";
|
|
26
|
+
import { tryExtractJsonObject } from "./json";
|
|
27
|
+
|
|
28
|
+
export interface SkillLesson {
|
|
29
|
+
skill: string;
|
|
30
|
+
kind: "failure-mode" | "anti-pattern";
|
|
31
|
+
title: string;
|
|
32
|
+
detail: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const SECTION_HEADER: Record<SkillLesson["kind"], string> = {
|
|
36
|
+
"failure-mode": "## Known Failure Modes",
|
|
37
|
+
"anti-pattern": "## Anti-Patterns (do NOT do)",
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
/** Hand-picked keyword sets per bundled skill, used ONLY for the deterministic
|
|
41
|
+
* free-text -> skill matcher below. Kept as one small record (not inline in
|
|
42
|
+
* the function) so the mapping is easy to audit/extend as skills are added. */
|
|
43
|
+
const SKILL_KEYWORDS: Record<string, string[]> = {
|
|
44
|
+
"deep-interview": ["ambiguity", "interview", "clarify", "clarification", "requirements", "socratic", "vague"],
|
|
45
|
+
"deep-dive": ["root cause", "root-cause", "diagnose", "diagnosis", "trace", "hypothesis", "causal"],
|
|
46
|
+
"ralplan": ["critic", "consensus", "[okay]", "[iterate]", "plan draft", "plan critique", "iterate verdict"],
|
|
47
|
+
"team": ["parallel_group", "worktree", "team executor", "plan step", "coordinated agents", "shared task list"],
|
|
48
|
+
"ultragoal": ["verification", "acceptance criteria", "goal verifier", "goal state", "verdict: met", "not_met"],
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Deterministic (no LLM) free-text -> bundled-skill match. Scores each SKILLS
|
|
53
|
+
* entry by counting keyword/substring hits (the skill's own `name` plus its
|
|
54
|
+
* hand-picked related terms from {@link SKILL_KEYWORDS}) against the lowercased
|
|
55
|
+
* input text, and returns the highest-scoring skill name — or `undefined` when
|
|
56
|
+
* nothing scores at all, since a wrong guess is worse than no guess here.
|
|
57
|
+
*/
|
|
58
|
+
export function matchLessonToSkill(text: string): string | undefined {
|
|
59
|
+
const haystack = text.toLowerCase();
|
|
60
|
+
if (!haystack.trim()) return undefined;
|
|
61
|
+
|
|
62
|
+
let bestName: string | undefined;
|
|
63
|
+
let bestScore = 0;
|
|
64
|
+
for (const skill of SKILLS) {
|
|
65
|
+
const terms = [skill.name.toLowerCase(), ...(SKILL_KEYWORDS[skill.name] ?? [])];
|
|
66
|
+
let score = 0;
|
|
67
|
+
for (const term of terms) {
|
|
68
|
+
if (term && haystack.includes(term)) score++;
|
|
69
|
+
}
|
|
70
|
+
if (score > bestScore) {
|
|
71
|
+
bestScore = score;
|
|
72
|
+
bestName = skill.name;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return bestScore > 0 ? bestName : undefined;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Project-level skill file path — `<cwd>/.jeo/skills/<name>.md`, the same
|
|
79
|
+
* highest-precedence directory `skillDirs()` (catalog.ts) resolves last, so a
|
|
80
|
+
* lesson written here transparently overrides/extends the bundled version by
|
|
81
|
+
* name the next time skills are loaded. */
|
|
82
|
+
export function skillFilePath(cwd: string, skillName: string): string {
|
|
83
|
+
return path.join(cwd, ".jeo", "skills", `${skillName}.md`);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Locate the target H2 section within `content`'s lines. Returns the line
|
|
87
|
+
* index range `[headerLine, bulletsEnd)` where `bulletsEnd` is the index to
|
|
88
|
+
* insert a new bullet at (immediately before the next `## ` header, or
|
|
89
|
+
* end-of-file, trimming any trailing blank lines inside the section so the
|
|
90
|
+
* new bullet lands directly after the last existing one). `null` when the
|
|
91
|
+
* section header isn't present yet. */
|
|
92
|
+
function findSection(lines: string[], header: string): { headerLine: number; insertAt: number } | null {
|
|
93
|
+
const headerLine = lines.findIndex(l => l.trim() === header);
|
|
94
|
+
if (headerLine < 0) return null;
|
|
95
|
+
let end = lines.length;
|
|
96
|
+
for (let i = headerLine + 1; i < lines.length; i++) {
|
|
97
|
+
if (lines[i]!.startsWith("## ")) { end = i; break; }
|
|
98
|
+
}
|
|
99
|
+
let insertAt = end;
|
|
100
|
+
while (insertAt > headerLine + 1 && lines[insertAt - 1]!.trim() === "") insertAt--;
|
|
101
|
+
return { headerLine, insertAt };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Does a `- **title**: ...` (or bare `- title...`) bullet with this exact
|
|
105
|
+
* title (case-insensitive) already exist between `[start, end)`? */
|
|
106
|
+
function sectionHasTitle(lines: string[], start: number, end: number, title: string): boolean {
|
|
107
|
+
const needle = title.trim().toLowerCase();
|
|
108
|
+
for (let i = start; i < end; i++) {
|
|
109
|
+
const m = lines[i]!.match(/^-\s+\*\*(.+?)\*\*\s*:/);
|
|
110
|
+
if (m && m[1]!.trim().toLowerCase() === needle) return true;
|
|
111
|
+
}
|
|
112
|
+
return false;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Core compounding write: append `lesson` as a markdown bullet into the
|
|
117
|
+
* project-level skill file's failure-mode/anti-pattern section, seeding the
|
|
118
|
+
* file from the bundled skill's own on-disk text on first write so existing
|
|
119
|
+
* guidance is never lost. Idempotent on exact-title re-append (case-
|
|
120
|
+
* insensitive) — returns `{ appended: false, reason: "duplicate" }` rather
|
|
121
|
+
* than writing a second copy. Best-effort: any I/O error is caught and
|
|
122
|
+
* reported, never thrown (mirrors `recordFailedAttempt`'s fail-open contract).
|
|
123
|
+
*/
|
|
124
|
+
export async function appendSkillLesson(cwd: string, lesson: SkillLesson): Promise<{ appended: boolean; reason?: string }> {
|
|
125
|
+
const title = lesson.title.trim();
|
|
126
|
+
if (!title) return { appended: false, reason: "empty title" };
|
|
127
|
+
const detail = lesson.detail.trim();
|
|
128
|
+
const header = SECTION_HEADER[lesson.kind];
|
|
129
|
+
|
|
130
|
+
try {
|
|
131
|
+
const filePath = skillFilePath(cwd, lesson.skill);
|
|
132
|
+
let content: string;
|
|
133
|
+
try {
|
|
134
|
+
content = await fs.readFile(filePath, "utf-8");
|
|
135
|
+
} catch {
|
|
136
|
+
const bundled = getSkill(lesson.skill);
|
|
137
|
+
content = bundled ? bundledSkillFileContent(bundled) : `# ${lesson.skill}\n\n`;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
let lines = content.replace(/\r\n/g, "\n").split("\n");
|
|
141
|
+
let section = findSection(lines, header);
|
|
142
|
+
if (!section) {
|
|
143
|
+
// Append the section header at the end of the file (blank-line separated).
|
|
144
|
+
if (lines.length > 0 && lines[lines.length - 1]!.trim() !== "") lines.push("");
|
|
145
|
+
if (!(lines.length > 0 && lines[lines.length - 1] === "")) lines.push("");
|
|
146
|
+
lines.push(header, "");
|
|
147
|
+
section = { headerLine: lines.length - 2, insertAt: lines.length };
|
|
148
|
+
} else if (sectionHasTitle(lines, section.headerLine + 1, section.insertAt, title)) {
|
|
149
|
+
return { appended: false, reason: "duplicate" };
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const bullet = `- **${title}**: ${detail}`;
|
|
153
|
+
lines.splice(section.insertAt, 0, bullet);
|
|
154
|
+
|
|
155
|
+
const finalContent = lines.join("\n").replace(/\n{3,}$/, "\n\n");
|
|
156
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
157
|
+
await ensureJeoGitignore(cwd);
|
|
158
|
+
const tmpPath = `${filePath}.tmp-${process.pid}`;
|
|
159
|
+
await fs.writeFile(tmpPath, finalContent, "utf-8");
|
|
160
|
+
await fs.rename(tmpPath, filePath);
|
|
161
|
+
return { appended: true };
|
|
162
|
+
} catch (err) {
|
|
163
|
+
return { appended: false, reason: err instanceof Error ? err.message : String(err) };
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Convenience wrapper for the automatic (stall-guard) path: matches `freeText`
|
|
169
|
+
* to a bundled skill via {@link matchLessonToSkill} and, only on a match,
|
|
170
|
+
* writes the lesson via {@link appendSkillLesson}. Unrelated text (no skill
|
|
171
|
+
* match) correctly writes nothing.
|
|
172
|
+
*/
|
|
173
|
+
export async function recordSkillLesson(
|
|
174
|
+
cwd: string,
|
|
175
|
+
freeText: string,
|
|
176
|
+
lesson: Omit<SkillLesson, "skill">,
|
|
177
|
+
): Promise<{ appended: boolean; skill?: string; reason?: string }> {
|
|
178
|
+
const skill = matchLessonToSkill(freeText);
|
|
179
|
+
if (!skill) return { appended: false, reason: "no matching skill" };
|
|
180
|
+
const result = await appendSkillLesson(cwd, { ...lesson, skill });
|
|
181
|
+
return { ...result, skill };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export interface LessonEvalResult {
|
|
185
|
+
skill: string;
|
|
186
|
+
total: number;
|
|
187
|
+
covered: number;
|
|
188
|
+
stale: Array<{ title: string; reason: string }>;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
interface ParsedBullet {
|
|
192
|
+
title: string;
|
|
193
|
+
detail: string;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** Extract every bullet from `[start, end)`, splitting the `- **title**:
|
|
197
|
+
* detail` format {@link appendSkillLesson} produces — a plain `- detail`
|
|
198
|
+
* bullet (no bold title) tolerantly uses the whole bullet text as both. */
|
|
199
|
+
function extractBullets(lines: string[], start: number, end: number): ParsedBullet[] {
|
|
200
|
+
const out: ParsedBullet[] = [];
|
|
201
|
+
for (let i = start; i < end; i++) {
|
|
202
|
+
const line = lines[i]!;
|
|
203
|
+
if (!line.startsWith("- ")) continue;
|
|
204
|
+
const m = line.match(/^-\s+\*\*(.+?)\*\*\s*:\s*(.*)$/);
|
|
205
|
+
if (m) {
|
|
206
|
+
out.push({ title: m[1]!.trim(), detail: m[2]!.trim() });
|
|
207
|
+
} else {
|
|
208
|
+
const text = line.slice(2).trim();
|
|
209
|
+
if (text) out.push({ title: text, detail: text });
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return out;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** The skill body with BOTH lesson sections (header + their bullets) removed —
|
|
216
|
+
* i.e. just the skill's core guidance, which is what the eval judge should
|
|
217
|
+
* compare each lesson against. */
|
|
218
|
+
function coreBodyWithoutLessons(lines: string[]): string {
|
|
219
|
+
const headers = new Set(Object.values(SECTION_HEADER));
|
|
220
|
+
const kept: string[] = [];
|
|
221
|
+
let skipping = false;
|
|
222
|
+
for (const line of lines) {
|
|
223
|
+
if (headers.has(line.trim())) { skipping = true; continue; }
|
|
224
|
+
if (skipping && line.startsWith("## ")) skipping = false;
|
|
225
|
+
if (!skipping) kept.push(line);
|
|
226
|
+
}
|
|
227
|
+
return kept.join("\n").trim();
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Self-eval half of the compounding loop: reads the project-level skill file
|
|
232
|
+
* and, for every recorded failure-mode/anti-pattern bullet, asks an
|
|
233
|
+
* independent LLM call whether the skill's CURRENT core guidance still
|
|
234
|
+
* addresses it ("covered") or has drifted ("stale"). Zero bullets short-
|
|
235
|
+
* circuits to a zero-cost result (no LLM call). `null` when the project-level
|
|
236
|
+
* file doesn't exist at all (no lessons ever recorded for this skill).
|
|
237
|
+
* Parse failure fails conservatively: every entry is treated as "covered"
|
|
238
|
+
* rather than risk falsely flagging a lesson as obsolete.
|
|
239
|
+
*/
|
|
240
|
+
export async function evalSkillLessons(
|
|
241
|
+
cwd: string,
|
|
242
|
+
skillName: string,
|
|
243
|
+
opts: { model?: string } = {},
|
|
244
|
+
): Promise<LessonEvalResult | null> {
|
|
245
|
+
const filePath = skillFilePath(cwd, skillName);
|
|
246
|
+
let content: string;
|
|
247
|
+
try {
|
|
248
|
+
content = await fs.readFile(filePath, "utf-8");
|
|
249
|
+
} catch {
|
|
250
|
+
return null;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const lines = content.replace(/\r\n/g, "\n").split("\n");
|
|
254
|
+
const failureSection = findSection(lines, SECTION_HEADER["failure-mode"]);
|
|
255
|
+
const antiSection = findSection(lines, SECTION_HEADER["anti-pattern"]);
|
|
256
|
+
const bullets: ParsedBullet[] = [
|
|
257
|
+
...(failureSection ? extractBullets(lines, failureSection.headerLine + 1, failureSection.insertAt) : []),
|
|
258
|
+
...(antiSection ? extractBullets(lines, antiSection.headerLine + 1, antiSection.insertAt) : []),
|
|
259
|
+
];
|
|
260
|
+
|
|
261
|
+
if (bullets.length === 0) {
|
|
262
|
+
return { skill: skillName, total: 0, covered: 0, stale: [] };
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const model = opts.model ?? resolveVerifierModel(await readGlobalConfig());
|
|
266
|
+
const coreBody = coreBodyWithoutLessons(lines);
|
|
267
|
+
const lessonList = bullets.map((b, i) => `${i + 1}. "${b.title}": ${b.detail}`).join("\n");
|
|
268
|
+
|
|
269
|
+
const systemPrompt = `You are an independent Skill Auditor. You are shown a skill's CURRENT core guidance text and a list of lessons (failure modes / anti-patterns) previously recorded against earlier versions of this skill.
|
|
270
|
+
|
|
271
|
+
For EACH lesson, judge:
|
|
272
|
+
- "covered": the skill's current guidance still clearly addresses this lesson (a reader following the current guidance would avoid/handle it).
|
|
273
|
+
- "stale": the guidance has drifted and no longer clearly addresses this lesson, OR the lesson itself looks obsolete/no-longer-applicable.
|
|
274
|
+
|
|
275
|
+
Respond with ONLY a JSON object: {"results": [{"title": string, "verdict": "covered" | "stale", "reason": string}]}. Include EVERY lesson by its exact title. No markdown, no code fences, no other text.`;
|
|
276
|
+
|
|
277
|
+
const userMessage = `Skill's current core guidance:\n\n${coreBody}\n\nRecorded lessons:\n\n${lessonList}\n\nJudge each lesson.`;
|
|
278
|
+
|
|
279
|
+
const verdictByTitle = new Map<string, { verdict: "covered" | "stale"; reason: string }>();
|
|
280
|
+
try {
|
|
281
|
+
const response = await callLlm(
|
|
282
|
+
[
|
|
283
|
+
{ role: "system", content: systemPrompt },
|
|
284
|
+
{ role: "user", content: userMessage },
|
|
285
|
+
],
|
|
286
|
+
{ model, jsonMode: true, maxTokens: 1500, reasoningEffort: "none" },
|
|
287
|
+
);
|
|
288
|
+
const parsed = tryExtractJsonObject<{ results?: unknown }>(response);
|
|
289
|
+
if (parsed && Array.isArray(parsed.results)) {
|
|
290
|
+
for (const entry of parsed.results) {
|
|
291
|
+
if (
|
|
292
|
+
entry && typeof entry === "object" &&
|
|
293
|
+
"title" in entry && typeof entry.title === "string" &&
|
|
294
|
+
"verdict" in entry && (entry.verdict === "covered" || entry.verdict === "stale")
|
|
295
|
+
) {
|
|
296
|
+
const reason = "reason" in entry && typeof entry.reason === "string" ? entry.reason : "";
|
|
297
|
+
verdictByTitle.set(entry.title.trim().toLowerCase(), { verdict: entry.verdict, reason });
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
} catch {
|
|
302
|
+
// fail conservatively below: an empty verdictByTitle treats every bullet as "covered"
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
let covered = 0;
|
|
306
|
+
const stale: Array<{ title: string; reason: string }> = [];
|
|
307
|
+
for (const b of bullets) {
|
|
308
|
+
const v = verdictByTitle.get(b.title.trim().toLowerCase());
|
|
309
|
+
if (v && v.verdict === "stale") {
|
|
310
|
+
stale.push({ title: b.title, reason: v.reason || "flagged stale by skill auditor" });
|
|
311
|
+
} else {
|
|
312
|
+
covered++;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
return { skill: skillName, total: bullets.length, covered, stale };
|
|
316
|
+
}
|
package/src/agent/state.ts
CHANGED
|
@@ -541,6 +541,32 @@ export function getLocalJeoDir(cwd: string = process.cwd()): string {
|
|
|
541
541
|
return path.join(cwd, ".jeo");
|
|
542
542
|
}
|
|
543
543
|
|
|
544
|
+
/** Ensure `.jeo/` self-ignores in git — memory/skill-lesson writes under `.jeo/`
|
|
545
|
+
* are agent-local working state, not source; without this a CI routine (`jeo
|
|
546
|
+
* routine init`, running `jeo -p` then committing via create-pull-request)
|
|
547
|
+
* ships `.jeo/memory/*`/`.jeo/skills/*` churn into every generated PR. Writes
|
|
548
|
+
* a bare `*` pattern ONCE on first `.jeo/` creation — idempotent (checked via
|
|
549
|
+
* a stat, never overwrites a file that already exists, so a user's own
|
|
550
|
+
* customized `.jeo/.gitignore` is never clobbered). Best-effort: any I/O
|
|
551
|
+
* failure is swallowed, never thrown (this is a hygiene nicety, not a
|
|
552
|
+
* correctness requirement — a missing .gitignore degrades gracefully to "no
|
|
553
|
+
* ignore", not a broken feature). */
|
|
554
|
+
export async function ensureJeoGitignore(cwd: string = process.cwd()): Promise<void> {
|
|
555
|
+
const gitignorePath = path.join(getLocalJeoDir(cwd), ".gitignore");
|
|
556
|
+
try {
|
|
557
|
+
await fs.access(gitignorePath);
|
|
558
|
+
return; // already exists — never overwrite (user customization respected)
|
|
559
|
+
} catch {
|
|
560
|
+
// does not exist — fall through to create
|
|
561
|
+
}
|
|
562
|
+
try {
|
|
563
|
+
await fs.mkdir(getLocalJeoDir(cwd), { recursive: true });
|
|
564
|
+
await fs.writeFile(gitignorePath, "*\n", "utf-8");
|
|
565
|
+
} catch {
|
|
566
|
+
// best-effort — a failed write here must never break the caller's real work
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
|
|
544
570
|
// mtime+size-validated cache for the small per-skill workflow-state JSON. readWorkflowState
|
|
545
571
|
// runs repeatedly (the mutation guard reads before every mutating tool), so re-reading +
|
|
546
572
|
// re-parsing each call is wasteful. CROSS-PROCESS-SAFE: a write by another process bumps
|
package/src/agent/subagents.ts
CHANGED
|
@@ -16,7 +16,7 @@ import architectPrompt from "../prompts/agents/architect.md" with { type: "text"
|
|
|
16
16
|
import criticPrompt from "../prompts/agents/critic.md" with { type: "text" };
|
|
17
17
|
import executorPrompt from "../prompts/agents/executor.md" with { type: "text" };
|
|
18
18
|
import plannerPrompt from "../prompts/agents/planner.md" with { type: "text" };
|
|
19
|
-
import type
|
|
19
|
+
import { strongestCredentialed, cheapestCredentialed, type PromptTier } from "./prompt-router";
|
|
20
20
|
|
|
21
21
|
const PROMPT_ROUTE_TIERS = ["trivial", "standard", "high", "complex"] as const satisfies readonly PromptTier[];
|
|
22
22
|
|
|
@@ -176,9 +176,6 @@ export function resolveSubagentModel(
|
|
|
176
176
|
const entry = config.subagents?.[normalized];
|
|
177
177
|
if (entry?.model) return entry.model;
|
|
178
178
|
|
|
179
|
-
// Exception: load strongestCredentialed dynamically using require to prevent eager loading and circular dependency issues that break test mocking.
|
|
180
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
181
|
-
const { strongestCredentialed } = require("./prompt-router");
|
|
182
179
|
if (normalized === "executor") {
|
|
183
180
|
return config.roles?.xhigh || config.roles?.slow || strongestCredentialed(config) || config.defaultModel;
|
|
184
181
|
}
|
|
@@ -189,11 +186,15 @@ export function resolveSubagentModel(
|
|
|
189
186
|
return config.roles?.high || strongestCredentialed(config) || config.defaultModel;
|
|
190
187
|
}
|
|
191
188
|
if (normalized === "critic") {
|
|
189
|
+
// Grader role (plan reviewer): prefer an explicit mid/cheap pin, else the
|
|
190
|
+
// live-cheapest credentialed model — a GENERAL search (not 2 hardcoded ids),
|
|
191
|
+
// so critic tracks catalog drift the same way every other role's fallback
|
|
192
|
+
// does instead of silently collapsing to defaultModel when neither o3-mini
|
|
193
|
+
// nor gemini-2.5-flash happens to be credentialed.
|
|
192
194
|
return (
|
|
193
195
|
config.roles?.medium ||
|
|
194
196
|
config.roles?.smol ||
|
|
195
|
-
|
|
196
|
-
strongestCredentialed(config, (m: CatalogModel) => m.canonical === "gemini-2.5-flash") ||
|
|
197
|
+
cheapestCredentialed(config) ||
|
|
197
198
|
config.defaultModel
|
|
198
199
|
);
|
|
199
200
|
}
|