@nemus-cli/nemus 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -11
- package/dist/cli/ai-prompt.js +4 -0
- package/dist/commands/reflect.js +13 -4
- package/dist/utils/agent-judge.js +89 -12
- package/dist/utils/reflect-analyze.js +136 -0
- package/dist/utils/reflect.js +75 -39
- package/package.json +1 -1
- package/src/cli/ai-prompt.ts +4 -0
- package/src/commands/reflect.ts +14 -6
- package/src/utils/agent-judge.test.ts +37 -2
- package/src/utils/agent-judge.ts +98 -15
- package/src/utils/reflect-analyze.test.ts +126 -0
- package/src/utils/reflect-analyze.ts +246 -0
- package/src/utils/reflect.test.ts +30 -27
- package/src/utils/reflect.ts +83 -58
package/CHANGELOG.md
CHANGED
|
@@ -7,19 +7,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.3.2] - 2026-08-30
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
|
|
14
|
+
- **`reflect` no longer hangs/times out — the real root cause was open stdin.**
|
|
15
|
+
The judge was spawned with its **stdin left as an open pipe** (`execFile`'s
|
|
16
|
+
default), so a stdin-reading agent like `pi` blocked forever waiting on input
|
|
17
|
+
and the run died at the timeout — regardless of prompt size or model. The
|
|
18
|
+
child now gets stdin `ignore` (async) / `input: ''` (sync) → immediate EOF,
|
|
19
|
+
and a real run drops from *timeout* to **~40s**. Regression-tested with a
|
|
20
|
+
stdin-reading child that would otherwise hang.
|
|
21
|
+
|
|
22
|
+
### Changed
|
|
23
|
+
|
|
24
|
+
- **`reflect` now does the analysis in code and hands the LLM only compact
|
|
25
|
+
facts.** A new deterministic layer (`reflect-analyze.ts`) clusters recurring
|
|
26
|
+
failures into normalized **signatures** (paths/numbers/hashes stripped) with
|
|
27
|
+
counts + which workspaces they span, tallies tool usage, flags
|
|
28
|
+
correction/retry loops, and lists workspaces missing a context file. The judge
|
|
29
|
+
prompt is built from those aggregates, so it stays **~5KB regardless of how
|
|
30
|
+
many workspaces** are analyzed (was ~25KB and growing), the call is faster and
|
|
31
|
+
cheaper, less raw prompt text leaves your machine, and every recommendation is
|
|
32
|
+
grounded in a real count. Output shape (`--json`, the report) is unchanged;
|
|
33
|
+
`--dry-run` now shows the computed facts.
|
|
34
|
+
- **Faster judge + richer signal.** The judge now runs pi at **`--thinking low`**
|
|
35
|
+
by default (it's a mechanical facts→recommendations transform, not deep
|
|
36
|
+
reasoning) — overridable with `--thinking`/`NEMUS_JUDGE_THINKING` and
|
|
37
|
+
`--model`/`NEMUS_JUDGE_MODEL` (threaded through for claude/opencode where
|
|
38
|
+
supported; thinking is pi-only). The digest now includes **verbatim “re-steer”
|
|
39
|
+
quotes** (the user corrections/redirects that are the sharpest coaching signal)
|
|
40
|
+
and classifies each workspace's context file as **missing / boilerplate /
|
|
41
|
+
substantive** (distinguishing “has an AGENTS.md” from “has a *useful* one”), on
|
|
42
|
+
top of the error-signature clusters.
|
|
43
|
+
|
|
44
|
+
## [0.3.1] - 2026-08-30
|
|
45
|
+
|
|
10
46
|
### Fixed
|
|
11
47
|
|
|
12
|
-
- **`reflect`
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
13 failures`); (2) the judge runs **async** behind a live spinner with elapsed
|
|
18
|
-
seconds; (3) the timeout is raised to 300s, overridable via
|
|
19
|
-
`NEMUS_JUDGE_TIMEOUT_MS`, and a timeout now yields an **actionable** message
|
|
20
|
-
("try a smaller --limit, a faster agent, or raise the cap"); (4) the judge
|
|
21
|
-
prompt is leaner (fewer prompts/errors per session) so a local model actually
|
|
22
|
-
finishes.
|
|
48
|
+
- **`reflect` shows live progress instead of looking hung.** The gather phase
|
|
49
|
+
prints a **per-workspace line** as each session is read (`✓ 1/3 my-workspace
|
|
50
|
+
635 turns · 12 prompts · 13 failures`); the judge runs **async** behind a live
|
|
51
|
+
spinner with elapsed seconds; the timeout is raised to 300s (overridable via
|
|
52
|
+
`NEMUS_JUDGE_TIMEOUT_MS`) with an **actionable** message on timeout.
|
|
23
53
|
|
|
24
54
|
## [0.3.0] - 2026-08-27
|
|
25
55
|
|
package/dist/cli/ai-prompt.js
CHANGED
|
@@ -161,6 +161,10 @@ const EXTRACTION_TIMEOUT_MS = 120000;
|
|
|
161
161
|
function runExtraction(cmd, fullArgs, fallbackArgs) {
|
|
162
162
|
const exec = (args) => (0, child_process_1.execFileSync)(cmd, args, {
|
|
163
163
|
encoding: 'utf-8',
|
|
164
|
+
// `input: ''` closes the child's stdin (EOF) so a stdin-reading agent (pi)
|
|
165
|
+
// can't block this synchronous call forever in a headless/piped context
|
|
166
|
+
// — the same hang the async judge hit, guarded here for `nemus -- "…"`.
|
|
167
|
+
input: '',
|
|
164
168
|
timeout: EXTRACTION_TIMEOUT_MS,
|
|
165
169
|
maxBuffer: 10 * 1024 * 1024,
|
|
166
170
|
});
|
package/dist/commands/reflect.js
CHANGED
|
@@ -5,6 +5,7 @@ const logger_1 = require("../utils/logger");
|
|
|
5
5
|
const output_1 = require("../utils/output");
|
|
6
6
|
const colors_1 = require("../utils/colors");
|
|
7
7
|
const reflect_1 = require("../utils/reflect");
|
|
8
|
+
const reflect_analyze_1 = require("../utils/reflect-analyze");
|
|
8
9
|
const agent_judge_1 = require("../utils/agent-judge");
|
|
9
10
|
function registerReflectCommand(parent) {
|
|
10
11
|
parent
|
|
@@ -12,6 +13,8 @@ function registerReflectCommand(parent) {
|
|
|
12
13
|
.alias('retro')
|
|
13
14
|
.description('Analyze your recent workspace sessions and suggest skill/prompt/context improvements (LLM-as-a-judge)')
|
|
14
15
|
.option('-n, --limit <n>', 'How many recent workspaces to analyze', '10')
|
|
16
|
+
.option('--model <model>', 'Judge model override (agent-native pattern/id)')
|
|
17
|
+
.option('--thinking <level>', 'Judge thinking level for pi: off|minimal|low|medium|high|xhigh|max')
|
|
15
18
|
.option('--json', 'Output the report as JSON')
|
|
16
19
|
.option('--dry-run', 'Print the assembled corpus + judge prompt without calling the agent')
|
|
17
20
|
.action(async (opts) => {
|
|
@@ -27,11 +30,15 @@ async function handleReflect(opts) {
|
|
|
27
30
|
}
|
|
28
31
|
const corpus = await (0, reflect_1.gatherReflectionCorpus)(limit, showProgress ? printProgress : undefined);
|
|
29
32
|
const withSessions = corpus.workspaces.filter((w) => w.session).length;
|
|
30
|
-
|
|
33
|
+
// A script does the heavy analysis (clustering failures, counting tools,
|
|
34
|
+
// spotting correction loops); the LLM only ever sees these compact facts,
|
|
35
|
+
// so the judge call stays small + fast regardless of workspace count.
|
|
36
|
+
const analysis = (0, reflect_analyze_1.analyzeCorpus)(corpus);
|
|
37
|
+
const prompt = (0, reflect_analyze_1.buildAnalysisPrompt)(analysis);
|
|
31
38
|
if (opts.dryRun) {
|
|
32
|
-
// No LLM call — surface exactly what the judge
|
|
39
|
+
// No LLM call — surface the computed facts + exactly what the judge sees.
|
|
33
40
|
if (opts.json)
|
|
34
|
-
(0, output_1.outputJson)({
|
|
41
|
+
(0, output_1.outputJson)({ analysis, prompt });
|
|
35
42
|
else {
|
|
36
43
|
process.stdout.write(prompt + '\n');
|
|
37
44
|
}
|
|
@@ -49,12 +56,14 @@ async function handleReflect(opts) {
|
|
|
49
56
|
// (non-blocking) so a live spinner shows it's alive, not hung. Timeout is
|
|
50
57
|
// overridable for slow local models.
|
|
51
58
|
const timeoutMs = Number.parseInt(process.env.NEMUS_JUDGE_TIMEOUT_MS ?? '', 10) || undefined;
|
|
59
|
+
const model = opts.model ?? process.env.NEMUS_JUDGE_MODEL ?? undefined;
|
|
60
|
+
const thinking = opts.thinking ?? process.env.NEMUS_JUDGE_THINKING ?? undefined;
|
|
52
61
|
const stopSpinner = opts.json
|
|
53
62
|
? () => { }
|
|
54
63
|
: startSpinner(`Judging ${withSessions} session(s) with your configured agent (this can take a minute)…`);
|
|
55
64
|
let parsed;
|
|
56
65
|
try {
|
|
57
|
-
parsed = await (0, agent_judge_1.runAgentJsonAsync)(prompt, { schema: reflect_1.REFLECT_SCHEMA, timeoutMs });
|
|
66
|
+
parsed = await (0, agent_judge_1.runAgentJsonAsync)(prompt, { schema: reflect_1.REFLECT_SCHEMA, timeoutMs, model, thinking });
|
|
58
67
|
}
|
|
59
68
|
finally {
|
|
60
69
|
stopSpinner();
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_JUDGE_THINKING = void 0;
|
|
4
|
+
exports.spawnCollect = spawnCollect;
|
|
3
5
|
exports.agentAttempts = agentAttempts;
|
|
4
6
|
exports.runAgentRaw = runAgentRaw;
|
|
5
7
|
exports.runAgentRawAsync = runAgentRawAsync;
|
|
@@ -7,28 +9,102 @@ exports.runAgentJson = runAgentJson;
|
|
|
7
9
|
exports.runAgentJsonAsync = runAgentJsonAsync;
|
|
8
10
|
exports.parseAgentJson = parseAgentJson;
|
|
9
11
|
const child_process_1 = require("child_process");
|
|
10
|
-
const util_1 = require("util");
|
|
11
12
|
const agent_config_1 = require("./agent-config");
|
|
12
|
-
|
|
13
|
+
/**
|
|
14
|
+
* Run a child to completion, capturing stdout, with stdin set to /dev/null.
|
|
15
|
+
*
|
|
16
|
+
* The stdin part is load-bearing: agents like `pi` block waiting on stdin if
|
|
17
|
+
* it's an open pipe (the default for execFile), which made the judge hang until
|
|
18
|
+
* the timeout regardless of prompt size or model speed. `stdio: ['ignore', …]`
|
|
19
|
+
* gives the child an immediate EOF, exactly like a non-interactive shell.
|
|
20
|
+
*/
|
|
21
|
+
function spawnCollect(cmd, args, opts) {
|
|
22
|
+
return new Promise((resolve, reject) => {
|
|
23
|
+
const child = (0, child_process_1.spawn)(cmd, args, {
|
|
24
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
25
|
+
timeout: opts.timeout,
|
|
26
|
+
killSignal: 'SIGKILL',
|
|
27
|
+
});
|
|
28
|
+
// Decode as UTF-8 at the stream boundary so a multi-byte char split across
|
|
29
|
+
// two chunks isn't corrupted (which would break JSON.parse downstream).
|
|
30
|
+
child.stdout.setEncoding('utf8');
|
|
31
|
+
child.stderr.setEncoding('utf8');
|
|
32
|
+
let stdout = '';
|
|
33
|
+
let stderr = '';
|
|
34
|
+
let overflow = false;
|
|
35
|
+
child.stdout.on('data', (d) => {
|
|
36
|
+
stdout += d;
|
|
37
|
+
if (stdout.length > opts.maxBuffer) {
|
|
38
|
+
overflow = true;
|
|
39
|
+
child.kill('SIGKILL');
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
child.stderr.on('data', (d) => {
|
|
43
|
+
stderr += d;
|
|
44
|
+
});
|
|
45
|
+
child.on('error', (e) => reject(e));
|
|
46
|
+
child.on('close', (code, signal) => {
|
|
47
|
+
if (overflow)
|
|
48
|
+
return reject(Object.assign(new Error('maxBuffer exceeded'), { stdout, stderr }));
|
|
49
|
+
if (signal)
|
|
50
|
+
return reject(Object.assign(new Error(`killed by ${signal}`), { killed: true, signal, stdout, stderr }));
|
|
51
|
+
if (code !== 0)
|
|
52
|
+
return reject(Object.assign(new Error(`exit ${code}`), { code, stdout, stderr }));
|
|
53
|
+
resolve(stdout);
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Default thinking level for the judge. The judge is a mechanical transform
|
|
59
|
+
* (facts → recommendations), not deep reasoning, so a heavy default like Opus
|
|
60
|
+
* @ medium thinking just makes it slow. `low` keeps pi fast; override per-run.
|
|
61
|
+
*/
|
|
62
|
+
exports.DEFAULT_JUDGE_THINKING = 'low';
|
|
13
63
|
/**
|
|
14
64
|
* The ordered invocation attempts for an agent (preferred → fallback), as pure
|
|
15
65
|
* data so both the sync and async runners share ONE flag ladder (and it's
|
|
16
|
-
* unit-testable without spawning anything).
|
|
66
|
+
* unit-testable without spawning anything). `--model` applies to all; pi also
|
|
67
|
+
* takes `--thinking` (the speed lever); both are ignored where unsupported.
|
|
17
68
|
*/
|
|
18
|
-
function agentAttempts(agentType, prompt,
|
|
69
|
+
function agentAttempts(agentType, prompt, opts = {}) {
|
|
70
|
+
const { schema, model, thinking } = opts;
|
|
19
71
|
if (agentType === 'claude') {
|
|
20
72
|
const preferred = ['-p', prompt, '--output-format', 'json', '--bare', '--strict-mcp-config', '--disable-slash-commands'];
|
|
73
|
+
if (model)
|
|
74
|
+
preferred.push('--model', model);
|
|
21
75
|
if (schema)
|
|
22
76
|
preferred.push('--json-schema', schema);
|
|
23
77
|
// Older claude may reject the newer flags — fall back to the plainest form.
|
|
24
|
-
|
|
78
|
+
const plain = ['-p', prompt];
|
|
79
|
+
if (model)
|
|
80
|
+
plain.push('--model', model);
|
|
81
|
+
return [{ cmd: 'claude', args: preferred }, { cmd: 'claude', args: plain }];
|
|
25
82
|
}
|
|
26
83
|
if (agentType === 'opencode') {
|
|
27
|
-
|
|
84
|
+
const args = ['run', prompt];
|
|
85
|
+
if (model)
|
|
86
|
+
args.push('--model', model);
|
|
87
|
+
return [{ cmd: 'opencode', args }];
|
|
28
88
|
}
|
|
29
89
|
// pi (and any other): run as lean as possible so a bloated env can't hang it.
|
|
30
90
|
const piLean = ['--no-extensions', '--no-skills', '--no-prompt-templates', '--no-context-files', '--no-tools', '--no-session'];
|
|
31
|
-
|
|
91
|
+
const modelArgs = model ? ['--model', model] : [];
|
|
92
|
+
const tune = thinking ? [...modelArgs, '--thinking', thinking] : modelArgs;
|
|
93
|
+
// Fallback keeps the stable --model but DROPS --thinking: --thinking is the
|
|
94
|
+
// newest flag and the most likely reason an older pi rejects the first
|
|
95
|
+
// attempt, so the safety net must not carry it (else both attempts fail).
|
|
96
|
+
return [
|
|
97
|
+
{ cmd: 'pi', args: [...piLean, ...tune, '-p', prompt] },
|
|
98
|
+
{ cmd: 'pi', args: [...modelArgs, '-p', prompt] },
|
|
99
|
+
];
|
|
100
|
+
}
|
|
101
|
+
/** Resolve the attempt options for a run, applying the pi thinking default. */
|
|
102
|
+
function attemptOptions(agentType, opts) {
|
|
103
|
+
return {
|
|
104
|
+
schema: opts.schema,
|
|
105
|
+
model: opts.model,
|
|
106
|
+
thinking: opts.thinking ?? (agentType === 'pi' ? exports.DEFAULT_JUDGE_THINKING : undefined),
|
|
107
|
+
};
|
|
32
108
|
}
|
|
33
109
|
function wrapJudgeError(err, agentType) {
|
|
34
110
|
// A timeout is the common failure (big prompt + slow local model), so make it
|
|
@@ -51,8 +127,10 @@ function runAgentRaw(prompt, opts = {}) {
|
|
|
51
127
|
const timeout = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
52
128
|
const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
|
|
53
129
|
const exec = opts.exec ??
|
|
54
|
-
|
|
55
|
-
|
|
130
|
+
// `input: ''` closes the child's stdin (EOF) so a stdin-reading agent (pi)
|
|
131
|
+
// can't hang the synchronous call — the sync twin of spawnCollect's fix.
|
|
132
|
+
((cmd, args, o) => (0, child_process_1.execFileSync)(cmd, args, { encoding: 'utf-8', input: '', timeout: o.timeout, maxBuffer: o.maxBuffer }));
|
|
133
|
+
const attempts = agentAttempts(agentType, prompt, attemptOptions(agentType, opts));
|
|
56
134
|
let lastErr;
|
|
57
135
|
for (const a of attempts) {
|
|
58
136
|
try {
|
|
@@ -73,9 +151,8 @@ async function runAgentRawAsync(prompt, opts = {}) {
|
|
|
73
151
|
const agentType = opts.agentType ?? (0, agent_config_1.getPrimaryAgent)().type;
|
|
74
152
|
const timeout = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
75
153
|
const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
|
|
76
|
-
const exec = opts.execAsync ??
|
|
77
|
-
|
|
78
|
-
const attempts = agentAttempts(agentType, prompt, opts.schema);
|
|
154
|
+
const exec = opts.execAsync ?? ((cmd, args, o) => spawnCollect(cmd, args, o));
|
|
155
|
+
const attempts = agentAttempts(agentType, prompt, attemptOptions(agentType, opts));
|
|
79
156
|
let lastErr;
|
|
80
157
|
for (const a of attempts) {
|
|
81
158
|
try {
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.normalizeErrorSignature = normalizeErrorSignature;
|
|
4
|
+
exports.analyzeCorpus = analyzeCorpus;
|
|
5
|
+
exports.buildAnalysisPrompt = buildAnalysisPrompt;
|
|
6
|
+
const reflect_1 = require("./reflect");
|
|
7
|
+
// --------------------------------------------------------------- normalizing
|
|
8
|
+
/**
|
|
9
|
+
* Reduce a raw error string to a stable signature so near-identical failures
|
|
10
|
+
* cluster together: lowercase, drop quotes, replace filesystem paths, hashes,
|
|
11
|
+
* hex ids and bare numbers with placeholders, collapse whitespace, and cap the
|
|
12
|
+
* length. `"fatal: not a git repository (or any of the parent up to /Users/x)"`
|
|
13
|
+
* and the same from another path collapse to one signature.
|
|
14
|
+
*/
|
|
15
|
+
function normalizeErrorSignature(raw) {
|
|
16
|
+
let s = (raw ?? '').toLowerCase();
|
|
17
|
+
s = s.replace(/[`'"]/g, ' ');
|
|
18
|
+
// Windows paths first (contain ':' and '\'), then unix paths.
|
|
19
|
+
s = s.replace(/[a-z]:\\[^\s]+/g, '<path>');
|
|
20
|
+
s = s.replace(/\/[^\s:)'"]+/g, '<path>');
|
|
21
|
+
// Long hex / uuids / sha-like tokens before bare numbers.
|
|
22
|
+
s = s.replace(/\b[0-9a-f]{7,}\b/g, '<hash>');
|
|
23
|
+
// No trailing \b, so a number glued to a unit ('4200ms', '9ms') still clusters.
|
|
24
|
+
s = s.replace(/\b\d[\d.,:]*/g, '<n>');
|
|
25
|
+
s = s.replace(/\s+/g, ' ').trim();
|
|
26
|
+
return s.slice(0, 100);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Turn a distilled corpus into aggregated facts. This is the "script processes
|
|
30
|
+
* the data" step — the LLM never sees the raw corpus, only the returned report.
|
|
31
|
+
*/
|
|
32
|
+
function analyzeCorpus(corpus, opts = {}) {
|
|
33
|
+
const topErrorsK = opts.topErrors ?? 8;
|
|
34
|
+
const topToolsK = opts.topTools ?? 10;
|
|
35
|
+
const exampleChars = opts.exampleChars ?? 160;
|
|
36
|
+
const maxReSteer = opts.maxReSteer ?? 8;
|
|
37
|
+
const errorMap = new Map();
|
|
38
|
+
const toolMap = new Map();
|
|
39
|
+
const workspacesMissingContext = [];
|
|
40
|
+
const workspacesBoilerplateContext = [];
|
|
41
|
+
const reSteerSamples = [];
|
|
42
|
+
const workspaces = [];
|
|
43
|
+
let sessionsAnalyzed = 0;
|
|
44
|
+
let totalTurns = 0;
|
|
45
|
+
let correctionSignals = 0;
|
|
46
|
+
for (const ws of corpus.workspaces) {
|
|
47
|
+
if (ws.contextQuality === 'missing')
|
|
48
|
+
workspacesMissingContext.push(ws.name);
|
|
49
|
+
else if (ws.contextQuality === 'boilerplate')
|
|
50
|
+
workspacesBoilerplateContext.push(ws.name);
|
|
51
|
+
const s = ws.session;
|
|
52
|
+
if (!s) {
|
|
53
|
+
workspaces.push({ name: ws.name, turns: 0, failures: 0, contextQuality: ws.contextQuality });
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
sessionsAnalyzed++;
|
|
57
|
+
totalTurns += s.turns;
|
|
58
|
+
for (const t of s.tools)
|
|
59
|
+
toolMap.set(t, (toolMap.get(t) ?? 0) + 1);
|
|
60
|
+
correctionSignals += s.userPrompts.filter(reflect_1.isCorrectionPrompt).length;
|
|
61
|
+
for (const q of s.reSteerSamples) {
|
|
62
|
+
if (reSteerSamples.length < maxReSteer)
|
|
63
|
+
reSteerSamples.push(q);
|
|
64
|
+
}
|
|
65
|
+
// Per-workspace signature tally (drives the global map + this ws's topFailure).
|
|
66
|
+
const localSig = new Map();
|
|
67
|
+
for (const e of s.errors) {
|
|
68
|
+
const sig = normalizeErrorSignature(e);
|
|
69
|
+
if (!sig)
|
|
70
|
+
continue;
|
|
71
|
+
localSig.set(sig, (localSig.get(sig) ?? 0) + 1);
|
|
72
|
+
let entry = errorMap.get(sig);
|
|
73
|
+
if (!entry) {
|
|
74
|
+
entry = { count: 0, ws: new Set(), example: e.slice(0, exampleChars) };
|
|
75
|
+
errorMap.set(sig, entry);
|
|
76
|
+
}
|
|
77
|
+
entry.count++;
|
|
78
|
+
entry.ws.add(ws.name);
|
|
79
|
+
}
|
|
80
|
+
const topFailure = [...localSig.entries()].sort((a, b) => b[1] - a[1])[0]?.[0];
|
|
81
|
+
workspaces.push({ name: ws.name, turns: s.turns, failures: s.errors.length, contextQuality: ws.contextQuality, topFailure });
|
|
82
|
+
}
|
|
83
|
+
const topErrors = [...errorMap.entries()]
|
|
84
|
+
.map(([signature, v]) => ({ signature, count: v.count, workspaces: [...v.ws], example: v.example }))
|
|
85
|
+
// Most frequent first; break ties by cross-workspace spread (recurrence).
|
|
86
|
+
.sort((a, b) => b.count - a.count || b.workspaces.length - a.workspaces.length)
|
|
87
|
+
.slice(0, topErrorsK);
|
|
88
|
+
const topTools = [...toolMap.entries()]
|
|
89
|
+
.map(([tool, sessions]) => ({ tool, sessions }))
|
|
90
|
+
.sort((a, b) => b.sessions - a.sessions || a.tool.localeCompare(b.tool))
|
|
91
|
+
.slice(0, topToolsK);
|
|
92
|
+
return {
|
|
93
|
+
totalWorkspaces: corpus.workspaces.length,
|
|
94
|
+
sessionsAnalyzed,
|
|
95
|
+
totalTurns,
|
|
96
|
+
topErrors,
|
|
97
|
+
topTools,
|
|
98
|
+
correctionSignals,
|
|
99
|
+
reSteerSamples,
|
|
100
|
+
workspacesMissingContext,
|
|
101
|
+
workspacesBoilerplateContext,
|
|
102
|
+
availableSkills: corpus.availableSkills,
|
|
103
|
+
workspaces,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
// --------------------------------------------------------------- judge prompt
|
|
107
|
+
/**
|
|
108
|
+
* Build the judge prompt from the pre-computed facts. Compact by construction
|
|
109
|
+
* (a handful of clusters + counts), so the LLM call stays small and fast no
|
|
110
|
+
* matter how many workspaces were analyzed. Output contract is unchanged
|
|
111
|
+
* (REFLECT_SCHEMA), so parsing/rendering are shared with the old path.
|
|
112
|
+
*/
|
|
113
|
+
function buildAnalysisPrompt(a) {
|
|
114
|
+
const L = [];
|
|
115
|
+
L.push('You are an expert reviewer ("LLM as a judge"). A script has already analyzed an engineer\'s', 'recent AI coding-agent sessions and distilled them into the FACTS below. Do not ask for the', 'raw transcripts — reason only from these facts.', '', 'Goal: recommend concrete improvements to their SETUP so the agent works better next time —', 'which skills to add and WHERE, which AGENTS.md/context rules are missing, missing connectivity/', 'smoke tests, and prompt/workflow habits. Ground every recommendation in a fact below', '(a recurring failure, a correction loop, a missing context file). Prefer a few high-signal', 'items over many generic ones. Include a concrete `example` snippet for skills/context rules.', '', `Sessions analyzed: ${a.sessionsAnalyzed} across ${a.totalWorkspaces} workspaces (${a.totalTurns} total turns).`, `Installed skills (don't re-suggest; find genuine gaps): ${a.availableSkills.join(', ') || '(none)'}`, `User correction/retry signals: ${a.correctionSignals} prompt(s) looked like corrections.`, `Workspaces with NO context file (AGENTS.md/CLAUDE.md): ${a.workspacesMissingContext.join(', ') || '(none)'}`, `Workspaces whose context file is just boilerplate/template: ${a.workspacesBoilerplateContext.join(', ') || '(none)'}`);
|
|
116
|
+
if (a.reSteerSamples.length) {
|
|
117
|
+
L.push('', 'Verbatim user corrections/re-steers (the sharpest signal — quote/act on these):');
|
|
118
|
+
for (const q of a.reSteerSamples)
|
|
119
|
+
L.push(` - “${q.replace(/\n/g, ' ')}”`);
|
|
120
|
+
}
|
|
121
|
+
L.push('', 'Top recurring failures (count × workspaces — example):');
|
|
122
|
+
if (a.topErrors.length === 0)
|
|
123
|
+
L.push(' (none captured)');
|
|
124
|
+
for (const e of a.topErrors) {
|
|
125
|
+
L.push(` - [${e.count}× in ${e.workspaces.length} ws] ${e.signature}`);
|
|
126
|
+
L.push(` e.g. ${e.example.replace(/\n/g, ' ')}`);
|
|
127
|
+
}
|
|
128
|
+
L.push('', `Most-used tools: ${a.topTools.map((t) => `${t.tool}(${t.sessions})`).join(', ') || '(none)'}`);
|
|
129
|
+
L.push('', 'Per-workspace:');
|
|
130
|
+
for (const w of a.workspaces) {
|
|
131
|
+
const top = w.topFailure ? ` · top failure: ${w.topFailure}` : '';
|
|
132
|
+
L.push(` - ${w.name}: ${w.turns} turns, ${w.failures} failures, context:${w.contextQuality}${top}`);
|
|
133
|
+
}
|
|
134
|
+
L.push('', 'Respond with ONLY a JSON object of this shape (no prose, no markdown fence):', '{"summary": string, "recommendations": [{"kind":"skill|context|test|prompt|connectivity|workflow|other",', '"title": string, "detail": string, "target": string(optional workspace/repo/path),', '"priority":"high|medium|low", "example": string(optional snippet)}]}');
|
|
135
|
+
return L.join('\n');
|
|
136
|
+
}
|
package/dist/utils/reflect.js
CHANGED
|
@@ -34,10 +34,11 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.REFLECT_SCHEMA = void 0;
|
|
37
|
+
exports.isCorrectionPrompt = isCorrectionPrompt;
|
|
37
38
|
exports.distillTranscript = distillTranscript;
|
|
39
|
+
exports.classifyAgentsMd = classifyAgentsMd;
|
|
38
40
|
exports.findLatestTranscriptFile = findLatestTranscriptFile;
|
|
39
41
|
exports.gatherReflectionCorpus = gatherReflectionCorpus;
|
|
40
|
-
exports.buildJudgePrompt = buildJudgePrompt;
|
|
41
42
|
exports.parseReflectionReport = parseReflectionReport;
|
|
42
43
|
const fs = __importStar(require("fs/promises"));
|
|
43
44
|
const path = __importStar(require("path"));
|
|
@@ -51,8 +52,18 @@ const claude_sessions_1 = require("./claude-sessions");
|
|
|
51
52
|
// fraction of the tokens (~halving the prompt), so the judge actually finishes.
|
|
52
53
|
const MAX_PROMPTS = 12;
|
|
53
54
|
const MAX_ERRORS = 15;
|
|
55
|
+
const MAX_RESTEER = 6;
|
|
54
56
|
const PROMPT_CHARS = 400;
|
|
55
57
|
const ERROR_CHARS = 200;
|
|
58
|
+
const RESTEER_CHARS = 240;
|
|
59
|
+
// A conservative “the user corrected / redirected the agent” cue. Soft signal:
|
|
60
|
+
// used to count re-steers and to capture the verbatim message for the judge.
|
|
61
|
+
const CORRECTION_RE = /\b(no|nope|wrong|incorrect|revert|undo|instead|actually|you (missed|forgot|broke|didn'?t)|that'?s? (wrong|not right|incorrect)|not what|don'?t)\b/i;
|
|
62
|
+
/** Whether a single user prompt reads like a correction/redirect. Exported +
|
|
63
|
+
* shared with the analyzer so the two never diverge. */
|
|
64
|
+
function isCorrectionPrompt(prompt) {
|
|
65
|
+
return CORRECTION_RE.test(prompt ?? '');
|
|
66
|
+
}
|
|
56
67
|
/** Flatten a message `content` (string or content-block array) to plain text. */
|
|
57
68
|
function contentToText(content) {
|
|
58
69
|
if (typeof content === 'string')
|
|
@@ -90,6 +101,7 @@ function isToolFailure(flag, text) {
|
|
|
90
101
|
*/
|
|
91
102
|
function distillTranscript(raw, meta) {
|
|
92
103
|
const userPrompts = [];
|
|
104
|
+
const reSteerSamples = [];
|
|
93
105
|
const errors = [];
|
|
94
106
|
const tools = new Set();
|
|
95
107
|
let turns = 0;
|
|
@@ -143,13 +155,57 @@ function distillTranscript(raw, meta) {
|
|
|
143
155
|
const isToolResultOnly = Array.isArray(content) && content.length > 0 && content.every((b) => b?.type === 'tool_result');
|
|
144
156
|
if (!isToolResultOnly) {
|
|
145
157
|
const text = contentToText(content).trim();
|
|
146
|
-
if (text && !text.startsWith('<')
|
|
147
|
-
userPrompts.
|
|
158
|
+
if (text && !text.startsWith('<')) {
|
|
159
|
+
if (userPrompts.length < MAX_PROMPTS)
|
|
160
|
+
userPrompts.push(text.slice(0, PROMPT_CHARS));
|
|
161
|
+
// Capture corrections verbatim (bounded) — the sharpest coaching signal.
|
|
162
|
+
if (reSteerSamples.length < MAX_RESTEER && isCorrectionPrompt(text)) {
|
|
163
|
+
reSteerSamples.push(text.slice(0, RESTEER_CHARS));
|
|
164
|
+
}
|
|
148
165
|
}
|
|
149
166
|
}
|
|
150
167
|
}
|
|
151
168
|
}
|
|
152
|
-
return { sessionId: meta.sessionId, agentType: meta.agentType, turns, userPrompts, errors, tools: [...tools] };
|
|
169
|
+
return { sessionId: meta.sessionId, agentType: meta.agentType, turns, userPrompts, reSteerSamples, errors, tools: [...tools] };
|
|
170
|
+
}
|
|
171
|
+
// ------------------------------------------------------- context classification
|
|
172
|
+
// Lines that are structural/boilerplate rather than real, custom guidance.
|
|
173
|
+
const BOILERPLATE_MARKERS = [
|
|
174
|
+
'ws-rules:',
|
|
175
|
+
'this workspace was created with',
|
|
176
|
+
'workspace manager',
|
|
177
|
+
'saved context',
|
|
178
|
+
'add your own notes here',
|
|
179
|
+
'common workflows',
|
|
180
|
+
];
|
|
181
|
+
/**
|
|
182
|
+
* Classify an AGENTS.md/CLAUDE.md by how much *real* guidance it carries, so the
|
|
183
|
+
* judge can tell “no context” from “has a file but it's the generated template.”
|
|
184
|
+
* Heuristic + pure.
|
|
185
|
+
*
|
|
186
|
+
* Deliberately **newline-independent**: it measures the volume of non-boilerplate
|
|
187
|
+
* prose (word count) plus heading count via a whitespace-tolerant regex, rather
|
|
188
|
+
* than splitting on lines. A line-anchored version would misclassify any excerpt
|
|
189
|
+
* whose newlines were collapsed to spaces upstream (a real bug class caught in a
|
|
190
|
+
* sibling implementation) — here even a fully single-lined file classifies the
|
|
191
|
+
* same as its multi-line original.
|
|
192
|
+
*/
|
|
193
|
+
function classifyAgentsMd(content) {
|
|
194
|
+
if (!content || !content.trim())
|
|
195
|
+
return 'missing';
|
|
196
|
+
let s = content.replace(/\r\n/g, '\n').toLowerCase();
|
|
197
|
+
s = s.replace(/```[\s\S]*?```/g, ' '); // drop fenced code
|
|
198
|
+
s = s.replace(/<!--[\s\S]*?-->/g, ' '); // drop HTML comments
|
|
199
|
+
for (const m of BOILERPLATE_MARKERS)
|
|
200
|
+
s = s.split(m).join(' '); // drop generated boilerplate (markers are lowercase)
|
|
201
|
+
// Headings: a `#` run at start OR after any whitespace (so a collapsed,
|
|
202
|
+
// single-line excerpt still counts them), followed by a space.
|
|
203
|
+
const headings = (s.match(/(?:^|\s)#{1,6}\s/g) || []).length;
|
|
204
|
+
// Remaining non-boilerplate words (markdown punctuation stripped).
|
|
205
|
+
const words = s.replace(/[#|>*_`~-]/g, ' ').split(/\s+/).filter((w) => w.length > 1).length;
|
|
206
|
+
if (words < 40)
|
|
207
|
+
return 'boilerplate';
|
|
208
|
+
return headings >= 2 || words >= 60 ? 'substantive' : 'boilerplate';
|
|
153
209
|
}
|
|
154
210
|
// --------------------------------------------------------- corpus gathering
|
|
155
211
|
/** Locate the most recent `.jsonl` transcript for a workspace under an agent. */
|
|
@@ -224,18 +280,25 @@ async function listAvailableSkills() {
|
|
|
224
280
|
}
|
|
225
281
|
return [...names].sort();
|
|
226
282
|
}
|
|
283
|
+
/** Which context files exist at the workspace root, plus how substantive the
|
|
284
|
+
* primary one is (missing/boilerplate/substantive). One read per present file. */
|
|
227
285
|
async function contextFilesFor(workspacePath) {
|
|
228
286
|
const present = [];
|
|
287
|
+
let quality = 'missing';
|
|
229
288
|
for (const name of (0, agent_config_1.getAllKnownContextFileNames)()) {
|
|
230
289
|
try {
|
|
231
|
-
await fs.
|
|
290
|
+
const content = await fs.readFile(path.join(workspacePath, name), 'utf-8');
|
|
232
291
|
present.push(name);
|
|
292
|
+
// Classify the first present file, then keep the best classification seen.
|
|
293
|
+
const c = classifyAgentsMd(content);
|
|
294
|
+
if (quality === 'missing' || (quality === 'boilerplate' && c === 'substantive'))
|
|
295
|
+
quality = c;
|
|
233
296
|
}
|
|
234
297
|
catch {
|
|
235
|
-
/* not present */
|
|
298
|
+
/* not present / unreadable */
|
|
236
299
|
}
|
|
237
300
|
}
|
|
238
|
-
return present;
|
|
301
|
+
return { files: present, quality };
|
|
239
302
|
}
|
|
240
303
|
/**
|
|
241
304
|
* Build the corpus the judge reasons over: the `limit` most **recently active**
|
|
@@ -256,11 +319,13 @@ async function gatherReflectionCorpus(limit, onProgress) {
|
|
|
256
319
|
for (let index = 0; index < recent.length; index++) {
|
|
257
320
|
const s = recent[index];
|
|
258
321
|
const meta = metaByName.get(s.workspaceName);
|
|
322
|
+
const context = await contextFilesFor(s.workspacePath);
|
|
259
323
|
const digest = {
|
|
260
324
|
name: s.workspaceName,
|
|
261
325
|
repoCount: meta?.metadata?.repositories?.length ?? 0,
|
|
262
326
|
repos: (meta?.metadata?.repositories ?? []).map((r) => r.name),
|
|
263
|
-
contextFiles:
|
|
327
|
+
contextFiles: context.files,
|
|
328
|
+
contextQuality: context.quality,
|
|
264
329
|
session: await readDigestForSession(s),
|
|
265
330
|
};
|
|
266
331
|
digests.push(digest);
|
|
@@ -292,37 +357,8 @@ exports.REFLECT_SCHEMA = JSON.stringify({
|
|
|
292
357
|
},
|
|
293
358
|
required: ['summary', 'recommendations'],
|
|
294
359
|
});
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
* is asked to recommend concrete improvements to the user's SETUP (skills,
|
|
298
|
-
* AGENTS.md/context rules, connectivity/tests, prompt habits, workflow) — not to
|
|
299
|
-
* redo the tasks. Output is strict JSON matching REFLECT_SCHEMA.
|
|
300
|
-
*/
|
|
301
|
-
function buildJudgePrompt(corpus) {
|
|
302
|
-
const lines = [];
|
|
303
|
-
lines.push('You are an expert reviewer ("LLM as a judge") analyzing an engineer\'s recent AI coding-agent sessions.', 'Goal: recommend concrete improvements to their SETUP so the agent works better next time —', 'which skills to add and WHERE, which AGENTS.md/context rules are missing, missing connectivity/', 'smoke tests, and prompt habits to change. Judge the setup, do NOT redo the tasks.', '', 'Base every recommendation on evidence in the sessions below (repeated failures, retries, vague', 'prompts, missing context). Prefer a few high-signal, actionable items over many generic ones.', 'When you suggest a skill or an AGENTS.md rule, include a short concrete `example` snippet.', '', `Globally installed skills (don't re-suggest these; suggest genuinely missing ones): ${corpus.availableSkills.join(', ') || '(none)'}`, '', `Recent workspaces (${corpus.workspaces.length}):`);
|
|
304
|
-
for (const ws of corpus.workspaces) {
|
|
305
|
-
lines.push(`\n## ${ws.name}`);
|
|
306
|
-
lines.push(`repos: ${ws.repos.join(', ') || '(none)'} | context files: ${ws.contextFiles.join(', ') || 'NONE'}`);
|
|
307
|
-
if (!ws.session) {
|
|
308
|
-
lines.push('session: (no recent agent session found)');
|
|
309
|
-
continue;
|
|
310
|
-
}
|
|
311
|
-
lines.push(`session: ${ws.session.turns} turns, tools used: ${ws.session.tools.join(', ') || '(none)'}`);
|
|
312
|
-
if (ws.session.userPrompts.length) {
|
|
313
|
-
lines.push('user prompts:');
|
|
314
|
-
for (const p of ws.session.userPrompts)
|
|
315
|
-
lines.push(` - ${p.replace(/\n/g, ' ')}`);
|
|
316
|
-
}
|
|
317
|
-
if (ws.session.errors.length) {
|
|
318
|
-
lines.push('errors/failures observed:');
|
|
319
|
-
for (const e of ws.session.errors)
|
|
320
|
-
lines.push(` - ${e.replace(/\n/g, ' ')}`);
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
lines.push('', 'Respond with ONLY a JSON object of this shape (no prose, no markdown fence):', '{"summary": string, "recommendations": [{"kind":"skill|context|test|prompt|connectivity|workflow|other",', '"title": string, "detail": string, "target": string(optional workspace/repo/path),', '"priority":"high|medium|low", "example": string(optional snippet)}]}');
|
|
324
|
-
return lines.join('\n');
|
|
325
|
-
}
|
|
360
|
+
// The judge prompt is now built from pre-computed FACTS (see reflect-analyze.ts
|
|
361
|
+
// `buildAnalysisPrompt`), not raw transcripts, so the LLM call stays small/fast.
|
|
326
362
|
// ----------------------------------------------------------- response parse
|
|
327
363
|
const KINDS = ['skill', 'context', 'test', 'prompt', 'connectivity', 'workflow', 'other'];
|
|
328
364
|
const PRIORITIES = ['high', 'medium', 'low'];
|
package/package.json
CHANGED
package/src/cli/ai-prompt.ts
CHANGED
|
@@ -140,6 +140,10 @@ function runExtraction(cmd: string, fullArgs: string[], fallbackArgs?: string[])
|
|
|
140
140
|
const exec = (args: string[]) =>
|
|
141
141
|
execFileSync(cmd, args, {
|
|
142
142
|
encoding: 'utf-8',
|
|
143
|
+
// `input: ''` closes the child's stdin (EOF) so a stdin-reading agent (pi)
|
|
144
|
+
// can't block this synchronous call forever in a headless/piped context
|
|
145
|
+
// — the same hang the async judge hit, guarded here for `nemus -- "…"`.
|
|
146
|
+
input: '',
|
|
143
147
|
timeout: EXTRACTION_TIMEOUT_MS,
|
|
144
148
|
maxBuffer: 10 * 1024 * 1024,
|
|
145
149
|
});
|