@nemus-cli/nemus 0.2.13 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +17 -0
- package/dist/cli/ai-prompt.js +13 -25
- package/dist/commands/reflect.js +160 -0
- package/dist/program.js +2 -0
- package/dist/utils/agent-judge.js +137 -0
- package/dist/utils/reflect.js +353 -0
- package/package.json +1 -1
- package/src/cli/ai-prompt.ts +12 -22
- package/src/commands/reflect.ts +172 -0
- package/src/program.ts +2 -0
- package/src/utils/agent-judge.test.ts +88 -0
- package/src/utils/agent-judge.ts +168 -0
- package/src/utils/reflect.test.ts +103 -0
- package/src/utils/reflect.ts +405 -0
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.REFLECT_SCHEMA = void 0;
|
|
37
|
+
exports.distillTranscript = distillTranscript;
|
|
38
|
+
exports.findLatestTranscriptFile = findLatestTranscriptFile;
|
|
39
|
+
exports.gatherReflectionCorpus = gatherReflectionCorpus;
|
|
40
|
+
exports.buildJudgePrompt = buildJudgePrompt;
|
|
41
|
+
exports.parseReflectionReport = parseReflectionReport;
|
|
42
|
+
const fs = __importStar(require("fs/promises"));
|
|
43
|
+
const path = __importStar(require("path"));
|
|
44
|
+
const workspace_meta_1 = require("./workspace-meta");
|
|
45
|
+
const agent_config_1 = require("./agent-config");
|
|
46
|
+
const claude_sessions_1 = require("./claude-sessions");
|
|
47
|
+
// --------------------------------------------------------- transcript distill
|
|
48
|
+
// Kept deliberately lean: the judge runs on the user's own (often local, slow)
|
|
49
|
+
// agent, and a 10-workspace corpus at full verbosity produced a ~200KB prompt
|
|
50
|
+
// that timed pi out. These bounds capture the pattern of a session at a
|
|
51
|
+
// fraction of the tokens (~halving the prompt), so the judge actually finishes.
|
|
52
|
+
const MAX_PROMPTS = 12;
|
|
53
|
+
const MAX_ERRORS = 15;
|
|
54
|
+
const PROMPT_CHARS = 400;
|
|
55
|
+
const ERROR_CHARS = 200;
|
|
56
|
+
/** Flatten a message `content` (string or content-block array) to plain text. */
|
|
57
|
+
function contentToText(content) {
|
|
58
|
+
if (typeof content === 'string')
|
|
59
|
+
return content;
|
|
60
|
+
if (Array.isArray(content)) {
|
|
61
|
+
return content
|
|
62
|
+
.map((b) => (b && b.type === 'text' && typeof b.text === 'string' ? b.text : ''))
|
|
63
|
+
.filter(Boolean)
|
|
64
|
+
.join('\n');
|
|
65
|
+
}
|
|
66
|
+
return '';
|
|
67
|
+
}
|
|
68
|
+
// Only used as a FALLBACK when a tool result carries no explicit error flag.
|
|
69
|
+
// Kept to strong failure signals so a successful grep/log line for the word
|
|
70
|
+
// "error", or a passing test named “…error…”, isn't mistaken for a failure.
|
|
71
|
+
const ERROR_RE = /\b(fatal|failed|failure|exception|traceback|denied|not a git repository|timed out|exit code\s+[1-9])\b/i;
|
|
72
|
+
/**
|
|
73
|
+
* Decide whether a tool result is a failure: trust the explicit `isError` flag
|
|
74
|
+
* when present (true => failure, false => success), and only guess from the
|
|
75
|
+
* text when there's no flag at all. This keeps the judge's “evidence” to real
|
|
76
|
+
* failures instead of any output that happens to contain the word “error”.
|
|
77
|
+
*/
|
|
78
|
+
function isToolFailure(flag, text) {
|
|
79
|
+
if (flag === true)
|
|
80
|
+
return true;
|
|
81
|
+
if (flag === false)
|
|
82
|
+
return false;
|
|
83
|
+
return ERROR_RE.test(text);
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Distill a raw `.jsonl` transcript into the signals a judge needs: the human
|
|
87
|
+
* prompts, tool failures, and which tools ran. Pure (operates on file content),
|
|
88
|
+
* defensive about the several line shapes Claude/pi emit, and bounded so a huge
|
|
89
|
+
* transcript can't blow the prompt budget.
|
|
90
|
+
*/
|
|
91
|
+
function distillTranscript(raw, meta) {
|
|
92
|
+
const userPrompts = [];
|
|
93
|
+
const errors = [];
|
|
94
|
+
const tools = new Set();
|
|
95
|
+
let turns = 0;
|
|
96
|
+
for (const line of raw.split('\n')) {
|
|
97
|
+
const trimmed = line.trim();
|
|
98
|
+
if (!trimmed)
|
|
99
|
+
continue;
|
|
100
|
+
let obj;
|
|
101
|
+
try {
|
|
102
|
+
obj = JSON.parse(trimmed);
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
const msg = obj.message ?? obj;
|
|
108
|
+
const role = msg?.role ?? obj?.type;
|
|
109
|
+
const content = msg?.content;
|
|
110
|
+
// Assistant turn + tool uses. Claude uses `tool_use` blocks; pi uses `toolCall`.
|
|
111
|
+
if (role === 'assistant') {
|
|
112
|
+
turns++;
|
|
113
|
+
if (Array.isArray(content)) {
|
|
114
|
+
for (const b of content) {
|
|
115
|
+
if (b && (b.type === 'tool_use' || b.type === 'toolCall') && typeof b.name === 'string')
|
|
116
|
+
tools.add(b.name);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
// Tool results, two shapes:
|
|
121
|
+
// - pi: a top-level message with role 'toolResult' (+ toolName, content).
|
|
122
|
+
// - Claude: a `tool_result` block inside a user message's content array.
|
|
123
|
+
if (role === 'toolResult') {
|
|
124
|
+
if (typeof msg.toolName === 'string')
|
|
125
|
+
tools.add(msg.toolName);
|
|
126
|
+
const text = contentToText(content);
|
|
127
|
+
if (isToolFailure(msg.isError ?? msg.is_error, text) && text.trim() && errors.length < MAX_ERRORS) {
|
|
128
|
+
errors.push(text.trim().slice(0, ERROR_CHARS));
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
if (Array.isArray(content)) {
|
|
132
|
+
for (const b of content) {
|
|
133
|
+
if (b && b.type === 'tool_result') {
|
|
134
|
+
const text = contentToText(b.content);
|
|
135
|
+
if (isToolFailure(b.is_error, text) && text.trim() && errors.length < MAX_ERRORS) {
|
|
136
|
+
errors.push(text.trim().slice(0, ERROR_CHARS));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
// Human prompts: a user message that carries actual text (not a tool_result echo).
|
|
142
|
+
if (role === 'user') {
|
|
143
|
+
const isToolResultOnly = Array.isArray(content) && content.length > 0 && content.every((b) => b?.type === 'tool_result');
|
|
144
|
+
if (!isToolResultOnly) {
|
|
145
|
+
const text = contentToText(content).trim();
|
|
146
|
+
if (text && !text.startsWith('<') && userPrompts.length < MAX_PROMPTS) {
|
|
147
|
+
userPrompts.push(text.slice(0, PROMPT_CHARS));
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return { sessionId: meta.sessionId, agentType: meta.agentType, turns, userPrompts, errors, tools: [...tools] };
|
|
153
|
+
}
|
|
154
|
+
// --------------------------------------------------------- corpus gathering
|
|
155
|
+
/** Locate the most recent `.jsonl` transcript for a workspace under an agent. */
|
|
156
|
+
async function findLatestTranscriptFile(sessionProjectsDir, workspacePath, agentType) {
|
|
157
|
+
if (agentType !== 'claude' && agentType !== 'pi')
|
|
158
|
+
return null;
|
|
159
|
+
const projDir = path.join(sessionProjectsDir, (0, claude_sessions_1.pathToProjectDirName)(workspacePath, agentType));
|
|
160
|
+
let entries;
|
|
161
|
+
try {
|
|
162
|
+
entries = await fs.readdir(projDir);
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
const jsonl = entries.filter((f) => f.endsWith('.jsonl'));
|
|
168
|
+
if (jsonl.length === 0)
|
|
169
|
+
return null;
|
|
170
|
+
const stats = await Promise.all(jsonl.map(async (f) => {
|
|
171
|
+
try {
|
|
172
|
+
return { f, mtime: (await fs.stat(path.join(projDir, f))).mtime.getTime() };
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
}));
|
|
178
|
+
const best = stats.filter((s) => !!s).sort((a, b) => b.mtime - a.mtime)[0];
|
|
179
|
+
return best ? path.join(projDir, best.f) : null;
|
|
180
|
+
}
|
|
181
|
+
const MAX_TRANSCRIPT_BYTES = 4 * 1024 * 1024;
|
|
182
|
+
/** Read + distill the transcript for a specific discovered session. Prefers the
|
|
183
|
+
* exact `<sessionId>.jsonl`; falls back to the latest transcript in that
|
|
184
|
+
* project dir if the exact file has been rotated away. */
|
|
185
|
+
async function readDigestForSession(s) {
|
|
186
|
+
if (s.agentType !== 'claude' && s.agentType !== 'pi')
|
|
187
|
+
return null;
|
|
188
|
+
const projectsDir = (0, agent_config_1.getAgentPaths)(s.agentType).sessionProjectsDir;
|
|
189
|
+
const exact = path.join(projectsDir, (0, claude_sessions_1.pathToProjectDirName)(s.workspacePath, s.agentType), `${s.sessionId}.jsonl`);
|
|
190
|
+
let file = exact;
|
|
191
|
+
try {
|
|
192
|
+
await fs.access(exact);
|
|
193
|
+
}
|
|
194
|
+
catch {
|
|
195
|
+
file = await findLatestTranscriptFile(projectsDir, s.workspacePath, s.agentType);
|
|
196
|
+
}
|
|
197
|
+
if (!file)
|
|
198
|
+
return null;
|
|
199
|
+
let raw;
|
|
200
|
+
try {
|
|
201
|
+
raw = await fs.readFile(file, 'utf-8');
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
return null;
|
|
205
|
+
}
|
|
206
|
+
if (raw.length > MAX_TRANSCRIPT_BYTES)
|
|
207
|
+
raw = raw.slice(raw.length - MAX_TRANSCRIPT_BYTES); // keep the tail (most recent)
|
|
208
|
+
return distillTranscript(raw, { sessionId: s.sessionId, agentType: s.agentType });
|
|
209
|
+
}
|
|
210
|
+
async function listAvailableSkills() {
|
|
211
|
+
const names = new Set();
|
|
212
|
+
for (const dir of (0, agent_config_1.getSkillsTargetDirs)()) {
|
|
213
|
+
try {
|
|
214
|
+
for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
|
|
215
|
+
if (entry.isDirectory())
|
|
216
|
+
names.add(entry.name);
|
|
217
|
+
else if (entry.name.endsWith('.md'))
|
|
218
|
+
names.add(entry.name.replace(/\.md$/, ''));
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
/* dir may not exist */
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return [...names].sort();
|
|
226
|
+
}
|
|
227
|
+
async function contextFilesFor(workspacePath) {
|
|
228
|
+
const present = [];
|
|
229
|
+
for (const name of (0, agent_config_1.getAllKnownContextFileNames)()) {
|
|
230
|
+
try {
|
|
231
|
+
await fs.access(path.join(workspacePath, name));
|
|
232
|
+
present.push(name);
|
|
233
|
+
}
|
|
234
|
+
catch {
|
|
235
|
+
/* not present */
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return present;
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Build the corpus the judge reasons over: the `limit` most **recently active**
|
|
242
|
+
* workspaces (by their latest agent session, not creation date — a retrospective
|
|
243
|
+
* is about recent *work*), each with its repos, context files, and distilled
|
|
244
|
+
* session, plus the globally-available skills. `onProgress` (optional) fires
|
|
245
|
+
* once per workspace as it finishes, for a live progress display.
|
|
246
|
+
*/
|
|
247
|
+
async function gatherReflectionCorpus(limit, onProgress) {
|
|
248
|
+
const [sessions, workspaces, availableSkills] = await Promise.all([
|
|
249
|
+
(0, claude_sessions_1.getWorkspaceSessions)(), // already sorted by last-active, one per workspace
|
|
250
|
+
(0, workspace_meta_1.listWorkspaces)(false),
|
|
251
|
+
listAvailableSkills(),
|
|
252
|
+
]);
|
|
253
|
+
const metaByName = new Map(workspaces.map((w) => [w.name, w]));
|
|
254
|
+
const recent = sessions.slice(0, limit);
|
|
255
|
+
const digests = [];
|
|
256
|
+
for (let index = 0; index < recent.length; index++) {
|
|
257
|
+
const s = recent[index];
|
|
258
|
+
const meta = metaByName.get(s.workspaceName);
|
|
259
|
+
const digest = {
|
|
260
|
+
name: s.workspaceName,
|
|
261
|
+
repoCount: meta?.metadata?.repositories?.length ?? 0,
|
|
262
|
+
repos: (meta?.metadata?.repositories ?? []).map((r) => r.name),
|
|
263
|
+
contextFiles: await contextFilesFor(s.workspacePath),
|
|
264
|
+
session: await readDigestForSession(s),
|
|
265
|
+
};
|
|
266
|
+
digests.push(digest);
|
|
267
|
+
onProgress?.({ index, total: recent.length, digest });
|
|
268
|
+
}
|
|
269
|
+
return { generatedAt: new Date().toISOString(), availableSkills, workspaces: digests };
|
|
270
|
+
}
|
|
271
|
+
// ------------------------------------------------------------- judge prompt
|
|
272
|
+
/** JSON schema for `claude --json-schema` (best-effort; other agents ignore it). */
|
|
273
|
+
exports.REFLECT_SCHEMA = JSON.stringify({
|
|
274
|
+
type: 'object',
|
|
275
|
+
properties: {
|
|
276
|
+
summary: { type: 'string' },
|
|
277
|
+
recommendations: {
|
|
278
|
+
type: 'array',
|
|
279
|
+
items: {
|
|
280
|
+
type: 'object',
|
|
281
|
+
properties: {
|
|
282
|
+
kind: { type: 'string', enum: ['skill', 'context', 'test', 'prompt', 'connectivity', 'workflow', 'other'] },
|
|
283
|
+
title: { type: 'string' },
|
|
284
|
+
detail: { type: 'string' },
|
|
285
|
+
target: { type: 'string' },
|
|
286
|
+
priority: { type: 'string', enum: ['high', 'medium', 'low'] },
|
|
287
|
+
example: { type: 'string' },
|
|
288
|
+
},
|
|
289
|
+
required: ['kind', 'title', 'detail', 'priority'],
|
|
290
|
+
},
|
|
291
|
+
},
|
|
292
|
+
},
|
|
293
|
+
required: ['summary', 'recommendations'],
|
|
294
|
+
});
|
|
295
|
+
/**
|
|
296
|
+
* Build the LLM-as-a-judge prompt. The judge sees distilled recent sessions and
|
|
297
|
+
* is asked to recommend concrete improvements to the user's SETUP (skills,
|
|
298
|
+
* AGENTS.md/context rules, connectivity/tests, prompt habits, workflow) — not to
|
|
299
|
+
* redo the tasks. Output is strict JSON matching REFLECT_SCHEMA.
|
|
300
|
+
*/
|
|
301
|
+
function buildJudgePrompt(corpus) {
|
|
302
|
+
const lines = [];
|
|
303
|
+
lines.push('You are an expert reviewer ("LLM as a judge") analyzing an engineer\'s recent AI coding-agent sessions.', 'Goal: recommend concrete improvements to their SETUP so the agent works better next time —', 'which skills to add and WHERE, which AGENTS.md/context rules are missing, missing connectivity/', 'smoke tests, and prompt habits to change. Judge the setup, do NOT redo the tasks.', '', 'Base every recommendation on evidence in the sessions below (repeated failures, retries, vague', 'prompts, missing context). Prefer a few high-signal, actionable items over many generic ones.', 'When you suggest a skill or an AGENTS.md rule, include a short concrete `example` snippet.', '', `Globally installed skills (don't re-suggest these; suggest genuinely missing ones): ${corpus.availableSkills.join(', ') || '(none)'}`, '', `Recent workspaces (${corpus.workspaces.length}):`);
|
|
304
|
+
for (const ws of corpus.workspaces) {
|
|
305
|
+
lines.push(`\n## ${ws.name}`);
|
|
306
|
+
lines.push(`repos: ${ws.repos.join(', ') || '(none)'} | context files: ${ws.contextFiles.join(', ') || 'NONE'}`);
|
|
307
|
+
if (!ws.session) {
|
|
308
|
+
lines.push('session: (no recent agent session found)');
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
lines.push(`session: ${ws.session.turns} turns, tools used: ${ws.session.tools.join(', ') || '(none)'}`);
|
|
312
|
+
if (ws.session.userPrompts.length) {
|
|
313
|
+
lines.push('user prompts:');
|
|
314
|
+
for (const p of ws.session.userPrompts)
|
|
315
|
+
lines.push(` - ${p.replace(/\n/g, ' ')}`);
|
|
316
|
+
}
|
|
317
|
+
if (ws.session.errors.length) {
|
|
318
|
+
lines.push('errors/failures observed:');
|
|
319
|
+
for (const e of ws.session.errors)
|
|
320
|
+
lines.push(` - ${e.replace(/\n/g, ' ')}`);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
lines.push('', 'Respond with ONLY a JSON object of this shape (no prose, no markdown fence):', '{"summary": string, "recommendations": [{"kind":"skill|context|test|prompt|connectivity|workflow|other",', '"title": string, "detail": string, "target": string(optional workspace/repo/path),', '"priority":"high|medium|low", "example": string(optional snippet)}]}');
|
|
324
|
+
return lines.join('\n');
|
|
325
|
+
}
|
|
326
|
+
// ----------------------------------------------------------- response parse
|
|
327
|
+
const KINDS = ['skill', 'context', 'test', 'prompt', 'connectivity', 'workflow', 'other'];
|
|
328
|
+
const PRIORITIES = ['high', 'medium', 'low'];
|
|
329
|
+
/** Validate + normalize the judge's parsed JSON into a ReflectionReport. */
|
|
330
|
+
function parseReflectionReport(parsed) {
|
|
331
|
+
const obj = (parsed ?? {});
|
|
332
|
+
const summary = typeof obj.summary === 'string' ? obj.summary : '';
|
|
333
|
+
const rawRecs = Array.isArray(obj.recommendations) ? obj.recommendations : [];
|
|
334
|
+
const recommendations = rawRecs
|
|
335
|
+
.map((r) => {
|
|
336
|
+
if (!r || typeof r !== 'object')
|
|
337
|
+
return null;
|
|
338
|
+
const title = typeof r.title === 'string' ? r.title : '';
|
|
339
|
+
const detail = typeof r.detail === 'string' ? r.detail : '';
|
|
340
|
+
if (!title && !detail)
|
|
341
|
+
return null;
|
|
342
|
+
const kind = KINDS.includes(r.kind) ? r.kind : 'other';
|
|
343
|
+
const priority = PRIORITIES.includes(r.priority) ? r.priority : 'medium';
|
|
344
|
+
const rec = { kind, title, detail, priority };
|
|
345
|
+
if (typeof r.target === 'string' && r.target.trim())
|
|
346
|
+
rec.target = r.target.trim();
|
|
347
|
+
if (typeof r.example === 'string' && r.example.trim())
|
|
348
|
+
rec.example = r.example.trim();
|
|
349
|
+
return rec;
|
|
350
|
+
})
|
|
351
|
+
.filter((r) => r !== null);
|
|
352
|
+
return { summary, recommendations };
|
|
353
|
+
}
|
package/package.json
CHANGED
package/src/cli/ai-prompt.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawn, execFile, execFileSync } from 'child_process';
|
|
2
|
+
import { parseAgentJson } from '../utils/agent-judge';
|
|
2
3
|
import { promisify } from 'util';
|
|
3
4
|
import * as fs from 'fs';
|
|
4
5
|
import * as path from 'path';
|
|
@@ -259,32 +260,21 @@ export async function extractIntent(prompt: string): Promise<ExtractedIntent> {
|
|
|
259
260
|
result = runExtraction('pi', [...piLean, ...piCore], piCore);
|
|
260
261
|
}
|
|
261
262
|
|
|
262
|
-
//
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
const parsed = JSON.parse(jsonStr);
|
|
270
|
-
|
|
271
|
-
// Handle different output formats:
|
|
272
|
-
// - Claude: { structured_output: {...} } or { result: "..." }
|
|
273
|
-
// - Pi: may return the object directly or wrap it
|
|
263
|
+
// Unwrap the agent's reply (code fences + the structured_output / result-string
|
|
264
|
+
// / result-object / bare-object envelopes) with the shared parser, so this and
|
|
265
|
+
// the reflect judge can't drift when a new agent shape is learned. The
|
|
266
|
+
// extraction-specific INVOCATION (lean flags + tailored timeout/auth errors)
|
|
267
|
+
// deliberately stays here — those messages are part of the `nemus --` UX.
|
|
274
268
|
let intent: ExtractedIntent | undefined;
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
intent = parsed.result;
|
|
281
|
-
} else if (parsed.workspaceName || parsed.repos || parsed.remainingIntent !== undefined) {
|
|
282
|
-
// Pi may return the extracted object directly
|
|
283
|
-
intent = parsed;
|
|
269
|
+
try {
|
|
270
|
+
const parsed = parseAgentJson(result) as any;
|
|
271
|
+
if (parsed && typeof parsed === 'object') intent = parsed;
|
|
272
|
+
} catch {
|
|
273
|
+
throw new Error(`Could not extract intent from agent response: ${result.slice(0, 200)}`);
|
|
284
274
|
}
|
|
285
275
|
|
|
286
276
|
if (!intent) {
|
|
287
|
-
throw new Error(`Could not extract intent from agent response
|
|
277
|
+
throw new Error(`Could not extract intent from agent response: ${result.slice(0, 200)}`);
|
|
288
278
|
}
|
|
289
279
|
|
|
290
280
|
// Coerce/validate field types so malformed model output can't crash the
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
import { logError, logInfo, logStep } from '../utils/logger';
|
|
3
|
+
import { outputJson, outputJsonError } from '../utils/output';
|
|
4
|
+
import { colorize } from '../utils/colors';
|
|
5
|
+
import {
|
|
6
|
+
gatherReflectionCorpus,
|
|
7
|
+
buildJudgePrompt,
|
|
8
|
+
parseReflectionReport,
|
|
9
|
+
REFLECT_SCHEMA,
|
|
10
|
+
ReflectionReport,
|
|
11
|
+
ReflectProgress,
|
|
12
|
+
Recommendation,
|
|
13
|
+
} from '../utils/reflect';
|
|
14
|
+
import { runAgentJsonAsync } from '../utils/agent-judge';
|
|
15
|
+
|
|
16
|
+
export function registerReflectCommand(parent: Command) {
|
|
17
|
+
parent
|
|
18
|
+
.command('reflect')
|
|
19
|
+
.alias('retro')
|
|
20
|
+
.description('Analyze your recent workspace sessions and suggest skill/prompt/context improvements (LLM-as-a-judge)')
|
|
21
|
+
.option('-n, --limit <n>', 'How many recent workspaces to analyze', '10')
|
|
22
|
+
.option('--json', 'Output the report as JSON')
|
|
23
|
+
.option('--dry-run', 'Print the assembled corpus + judge prompt without calling the agent')
|
|
24
|
+
.action(async (opts) => {
|
|
25
|
+
await handleReflect(opts);
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async function handleReflect(opts: { limit?: string; json?: boolean; dryRun?: boolean }) {
|
|
30
|
+
const limit = Math.max(1, Number.parseInt(opts.limit ?? '10', 10) || 10);
|
|
31
|
+
try {
|
|
32
|
+
const showProgress = !opts.json && !opts.dryRun;
|
|
33
|
+
if (showProgress) {
|
|
34
|
+
logStep(`Analyzing your ${colorize(String(limit), 'cyan')} most recent workspaces…`);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const corpus = await gatherReflectionCorpus(limit, showProgress ? printProgress : undefined);
|
|
38
|
+
const withSessions = corpus.workspaces.filter((w) => w.session).length;
|
|
39
|
+
const prompt = buildJudgePrompt(corpus);
|
|
40
|
+
|
|
41
|
+
if (opts.dryRun) {
|
|
42
|
+
// No LLM call — surface exactly what the judge would see.
|
|
43
|
+
if (opts.json) outputJson({ corpus, prompt });
|
|
44
|
+
else {
|
|
45
|
+
process.stdout.write(prompt + '\n');
|
|
46
|
+
}
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
if (withSessions === 0) {
|
|
51
|
+
const msg = 'No recent agent sessions found to analyze (need Claude/pi session transcripts).';
|
|
52
|
+
if (opts.json) outputJsonError(msg);
|
|
53
|
+
else logError(msg);
|
|
54
|
+
process.exit(1);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// The judge shells the user's own agent and can take minutes; run it async
|
|
58
|
+
// (non-blocking) so a live spinner shows it's alive, not hung. Timeout is
|
|
59
|
+
// overridable for slow local models.
|
|
60
|
+
const timeoutMs = Number.parseInt(process.env.NEMUS_JUDGE_TIMEOUT_MS ?? '', 10) || undefined;
|
|
61
|
+
const stopSpinner = opts.json
|
|
62
|
+
? () => {}
|
|
63
|
+
: startSpinner(`Judging ${withSessions} session(s) with your configured agent (this can take a minute)…`);
|
|
64
|
+
let parsed: unknown;
|
|
65
|
+
try {
|
|
66
|
+
parsed = await runAgentJsonAsync(prompt, { schema: REFLECT_SCHEMA, timeoutMs });
|
|
67
|
+
} finally {
|
|
68
|
+
stopSpinner();
|
|
69
|
+
}
|
|
70
|
+
const report = parseReflectionReport(parsed);
|
|
71
|
+
|
|
72
|
+
if (opts.json) {
|
|
73
|
+
outputJson({ analyzed: withSessions, workspaces: corpus.workspaces.length, ...report });
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
printReport(report, corpus.workspaces.length, withSessions);
|
|
77
|
+
} catch (error) {
|
|
78
|
+
const msg = error instanceof Error ? error.message : 'reflect failed';
|
|
79
|
+
if (opts.json) outputJsonError(msg);
|
|
80
|
+
else {
|
|
81
|
+
logError('Failed to analyze sessions');
|
|
82
|
+
logError(msg);
|
|
83
|
+
}
|
|
84
|
+
process.exit(1);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const KIND_LABEL: Record<Recommendation['kind'], string> = {
|
|
89
|
+
skill: 'Skill',
|
|
90
|
+
context: 'Context/AGENTS.md',
|
|
91
|
+
test: 'Test',
|
|
92
|
+
prompt: 'Prompt',
|
|
93
|
+
connectivity: 'Connectivity',
|
|
94
|
+
workflow: 'Workflow',
|
|
95
|
+
other: 'Other',
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* A minimal stderr spinner with elapsed seconds. Returns a stop() that clears
|
|
100
|
+
* the line. No-op (single log line) when stderr isn't a TTY (piped/CI), so it
|
|
101
|
+
* never pollutes captured output. Kept local + tiny — no new dependency.
|
|
102
|
+
*/
|
|
103
|
+
function startSpinner(text: string): () => void {
|
|
104
|
+
if (!process.stderr.isTTY) {
|
|
105
|
+
logInfo(text);
|
|
106
|
+
return () => {};
|
|
107
|
+
}
|
|
108
|
+
const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
|
|
109
|
+
const start = Date.now();
|
|
110
|
+
let i = 0;
|
|
111
|
+
const render = () => {
|
|
112
|
+
const secs = Math.floor((Date.now() - start) / 1000);
|
|
113
|
+
process.stderr.write(`\r${colorize(frames[(i = (i + 1) % frames.length)], 'cyan')} ${text} ${colorize(`(${secs}s)`, 'dim')}`);
|
|
114
|
+
};
|
|
115
|
+
render();
|
|
116
|
+
const timer = setInterval(render, 100);
|
|
117
|
+
timer.unref?.(); // never keep the process alive on our account
|
|
118
|
+
return () => {
|
|
119
|
+
clearInterval(timer);
|
|
120
|
+
process.stderr.write('\r' + ' '.repeat(text.length + 24) + '\r');
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Live per-workspace line during the gather phase (to stderr — stdout stays
|
|
125
|
+
* reserved for the report / JSON). */
|
|
126
|
+
function printProgress(p: ReflectProgress): void {
|
|
127
|
+
const n = colorize(`${p.index + 1}/${p.total}`, 'dim');
|
|
128
|
+
const d = p.digest.session;
|
|
129
|
+
if (!d) {
|
|
130
|
+
process.stderr.write(` ${colorize('·', 'dim')} ${n} ${p.digest.name} ${colorize('— no session', 'dim')}\n`);
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
const failures = `${d.errors.length} ${d.errors.length === 1 ? 'failure' : 'failures'}`;
|
|
134
|
+
const stats = colorize(`${d.turns} turns · ${d.userPrompts.length} prompts · ${failures}`, 'dim');
|
|
135
|
+
process.stderr.write(` ${colorize('✓', 'green')} ${n} ${p.digest.name} ${stats}\n`);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function priorityBadge(p: Recommendation['priority']): string {
|
|
139
|
+
if (p === 'high') return colorize('● high', 'red');
|
|
140
|
+
if (p === 'medium') return colorize('● med', 'yellow');
|
|
141
|
+
return colorize('● low', 'gray');
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function printReport(report: ReflectionReport, workspaces: number, analyzed: number) {
|
|
145
|
+
console.log('');
|
|
146
|
+
console.log(colorize(' Reflection', 'bright') + colorize(` (${analyzed} sessions across ${workspaces} workspaces)`, 'dim'));
|
|
147
|
+
console.log(colorize(' ' + '─'.repeat(56), 'dim'));
|
|
148
|
+
if (report.summary) {
|
|
149
|
+
console.log('\n ' + report.summary.replace(/\n/g, '\n '));
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (report.recommendations.length === 0) {
|
|
153
|
+
console.log('\n ' + colorize('No specific recommendations — looks solid.', 'green') + '\n');
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// High priority first.
|
|
158
|
+
const order = { high: 0, medium: 1, low: 2 };
|
|
159
|
+
const recs = [...report.recommendations].sort((a, b) => order[a.priority] - order[b.priority]);
|
|
160
|
+
|
|
161
|
+
console.log('');
|
|
162
|
+
for (const r of recs) {
|
|
163
|
+
const target = r.target ? colorize(` [${r.target}]`, 'cyan') : '';
|
|
164
|
+
console.log(` ${priorityBadge(r.priority)} ${colorize(KIND_LABEL[r.kind], 'bright')} ${r.title}${target}`);
|
|
165
|
+
if (r.detail) console.log(` ${r.detail.replace(/\n/g, '\n ')}`);
|
|
166
|
+
if (r.example) {
|
|
167
|
+
console.log(colorize(' example:', 'dim'));
|
|
168
|
+
console.log(colorize(r.example.replace(/^/gm, ' '), 'dim'));
|
|
169
|
+
}
|
|
170
|
+
console.log('');
|
|
171
|
+
}
|
|
172
|
+
}
|
package/src/program.ts
CHANGED
|
@@ -59,6 +59,7 @@ import { registerSaveContextCommand } from './commands/save-context';
|
|
|
59
59
|
import { registerMigrateCommand } from './commands/migrate';
|
|
60
60
|
import { registerReportBugCommand } from './commands/report-bug';
|
|
61
61
|
import { registerCompletionCommand } from './commands/completion';
|
|
62
|
+
import { registerReflectCommand } from './commands/reflect';
|
|
62
63
|
|
|
63
64
|
registerCreateCommand(program);
|
|
64
65
|
registerListCommand(program);
|
|
@@ -84,6 +85,7 @@ registerSaveContextCommand(program);
|
|
|
84
85
|
registerMigrateCommand(program);
|
|
85
86
|
registerReportBugCommand(program);
|
|
86
87
|
registerCompletionCommand(program);
|
|
88
|
+
registerReflectCommand(program);
|
|
87
89
|
|
|
88
90
|
// Register TUI (delegates to existing Ink/React implementation)
|
|
89
91
|
program
|