@roodriigoooo/pi-scrutiny 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -0
- package/extensions/scrutiny/analysis.ts +335 -0
- package/extensions/scrutiny/config.ts +407 -0
- package/extensions/scrutiny/engine.ts +513 -0
- package/extensions/scrutiny/history.ts +566 -0
- package/extensions/scrutiny/packet.ts +188 -0
- package/extensions/scrutiny/palette.ts +413 -0
- package/extensions/scrutiny/preview.ts +261 -0
- package/extensions/scrutiny/registry.ts +48 -0
- package/extensions/scrutiny/runner.ts +128 -0
- package/extensions/scrutiny/scout.ts +314 -0
- package/extensions/scrutiny/summary.ts +270 -0
- package/extensions/scrutiny/types.ts +184 -0
- package/extensions/scrutiny/ui.ts +299 -0
- package/extensions/scrutiny/util.ts +123 -0
- package/extensions/scrutiny.ts +333 -0
- package/package.json +48 -0
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
export type ScrutinySurface = "consult" | "hypotheses" | "criteria" | "repo-map" | "risks" | "verify";
|
|
2
|
+
export type PanelMode = "replicate" | "roles";
|
|
3
|
+
export type ScrutinyStatus = "pending" | "running" | "ready" | "failed";
|
|
4
|
+
export type ThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh";
|
|
5
|
+
|
|
6
|
+
export type ScrutinyUsage = {
|
|
7
|
+
input: number;
|
|
8
|
+
output: number;
|
|
9
|
+
cacheRead: number;
|
|
10
|
+
cacheWrite: number;
|
|
11
|
+
cost: number;
|
|
12
|
+
contextTokens: number;
|
|
13
|
+
turns: number;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export type PanelSpec = {
|
|
17
|
+
model: string;
|
|
18
|
+
role: string;
|
|
19
|
+
thinking?: ThinkingLevel;
|
|
20
|
+
status: ScrutinyStatus;
|
|
21
|
+
startedAt?: number;
|
|
22
|
+
endedAt?: number;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
export type PanelResponse = {
|
|
26
|
+
model: string;
|
|
27
|
+
role: string;
|
|
28
|
+
status: "ok" | "error";
|
|
29
|
+
content: string;
|
|
30
|
+
error?: string;
|
|
31
|
+
usage: ScrutinyUsage;
|
|
32
|
+
durationMs: number;
|
|
33
|
+
exitCode: number;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export type ScrutinyAnalysis = {
|
|
37
|
+
consensus?: string[];
|
|
38
|
+
contradictions?: Array<{ topic: string; stances: Array<{ model: string; stance: string }> }>;
|
|
39
|
+
unique_insights?: Array<{ model: string; insight: string }>;
|
|
40
|
+
risks?: string[];
|
|
41
|
+
coverage?: string[];
|
|
42
|
+
blind_spots?: string[];
|
|
43
|
+
confidence?: "low" | "medium" | "high" | string;
|
|
44
|
+
disagreement_signal?: boolean;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export type ScrutinyRunProgress = {
|
|
48
|
+
runId: string;
|
|
49
|
+
surface: ScrutinySurface;
|
|
50
|
+
panel_mode?: PanelMode;
|
|
51
|
+
packetPath?: string;
|
|
52
|
+
panel: PanelSpec[];
|
|
53
|
+
judge?: PanelSpec;
|
|
54
|
+
startedAt: number;
|
|
55
|
+
updatedAt: number;
|
|
56
|
+
status: "running" | "ok" | "error";
|
|
57
|
+
message?: string;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
export type ScrutinyRunResult = {
|
|
61
|
+
runId: string;
|
|
62
|
+
surface: ScrutinySurface;
|
|
63
|
+
panel_mode?: PanelMode;
|
|
64
|
+
status: "ok" | "error";
|
|
65
|
+
failure_reason?: "missing_panel" | "all_panels_failed" | "judge_failed" | "recursion_capped" | "unexpected_error" | "verify_failed";
|
|
66
|
+
error?: string;
|
|
67
|
+
packetPath?: string;
|
|
68
|
+
packet: string;
|
|
69
|
+
responses: PanelResponse[];
|
|
70
|
+
failed_models: Array<{ model: string; error: string }>;
|
|
71
|
+
judge?: PanelResponse;
|
|
72
|
+
analysis?: ScrutinyAnalysis;
|
|
73
|
+
verify?: VerifyReport;
|
|
74
|
+
startedAt: number;
|
|
75
|
+
endedAt: number;
|
|
76
|
+
durationMs: number;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
export type ScrutinySummary = {
|
|
80
|
+
runId: string;
|
|
81
|
+
surface: ScrutinySurface;
|
|
82
|
+
startedAt: number;
|
|
83
|
+
endedAt: number;
|
|
84
|
+
prompt: string;
|
|
85
|
+
status: "ok" | "error";
|
|
86
|
+
failure_reason?: ScrutinyRunResult["failure_reason"];
|
|
87
|
+
error?: string;
|
|
88
|
+
files: string[];
|
|
89
|
+
symbols: string[];
|
|
90
|
+
keywords: string[];
|
|
91
|
+
signals: string[];
|
|
92
|
+
risks: string[];
|
|
93
|
+
contradictions: string[];
|
|
94
|
+
missingContext: string[];
|
|
95
|
+
sourceRefs: string[];
|
|
96
|
+
fileHashes: Record<string, string>;
|
|
97
|
+
resultPath: string;
|
|
98
|
+
surfaceArtifactPath?: string;
|
|
99
|
+
packetPath?: string;
|
|
100
|
+
responsesPath?: string;
|
|
101
|
+
verifyPath?: string;
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
export type VerifyCheck = {
|
|
105
|
+
name: string;
|
|
106
|
+
command: string;
|
|
107
|
+
status: "pass" | "fail" | "skipped" | "error";
|
|
108
|
+
exitCode?: number;
|
|
109
|
+
output?: string;
|
|
110
|
+
durationMs: number;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
export type VerifyReport = {
|
|
114
|
+
checks: VerifyCheck[];
|
|
115
|
+
diffStat?: string;
|
|
116
|
+
passed: number;
|
|
117
|
+
failed: number;
|
|
118
|
+
skipped: number;
|
|
119
|
+
durationMs: number;
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
export type ScrutinyConfigSource = {
|
|
123
|
+
scope: "global" | "project" | "env";
|
|
124
|
+
path?: string;
|
|
125
|
+
status: "loaded" | "missing" | "skipped" | "error";
|
|
126
|
+
reason?: string;
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
export type ScrutinyConfig = {
|
|
130
|
+
panel: PanelMember[];
|
|
131
|
+
judge?: string;
|
|
132
|
+
maxPanelModels: number;
|
|
133
|
+
maxPanelOutputChars: number;
|
|
134
|
+
maxJudgeOutputChars: number;
|
|
135
|
+
panelTimeoutMs: number;
|
|
136
|
+
judgeTimeoutMs: number;
|
|
137
|
+
verifyTimeoutMs: number;
|
|
138
|
+
includeGitDiff: boolean;
|
|
139
|
+
gitDiffCharLimit: number;
|
|
140
|
+
tools: string[];
|
|
141
|
+
verifyChecks: VerifyCheckSpec[];
|
|
142
|
+
councils: Council[];
|
|
143
|
+
configSources: ScrutinyConfigSource[];
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
export type VerifyCheckSpec = {
|
|
147
|
+
name: string;
|
|
148
|
+
command: string;
|
|
149
|
+
args?: string[];
|
|
150
|
+
timeoutMs?: number;
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
export type PanelMember = {
|
|
154
|
+
model: string;
|
|
155
|
+
lens?: string;
|
|
156
|
+
thinking?: ThinkingLevel;
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
export type CouncilPanelist = PanelMember;
|
|
160
|
+
|
|
161
|
+
export type Council = {
|
|
162
|
+
name: string;
|
|
163
|
+
surface: ScrutinySurface;
|
|
164
|
+
panelists: PanelMember[];
|
|
165
|
+
thinking?: ThinkingLevel;
|
|
166
|
+
judge?: string;
|
|
167
|
+
judgeMode?: "auto" | "off" | "on";
|
|
168
|
+
includeGitDiff?: boolean;
|
|
169
|
+
verify?: boolean;
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
export type ScrutinyParams = {
|
|
173
|
+
prompt: string;
|
|
174
|
+
context?: string;
|
|
175
|
+
surface?: ScrutinySurface;
|
|
176
|
+
panel?: string[];
|
|
177
|
+
panelMembers?: PanelMember[];
|
|
178
|
+
judge?: string;
|
|
179
|
+
judgeMode?: "auto" | "off" | "on";
|
|
180
|
+
maxPanelModels?: number;
|
|
181
|
+
includeGitDiff?: boolean;
|
|
182
|
+
tools?: string[];
|
|
183
|
+
verify?: boolean;
|
|
184
|
+
};
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
import { getMarkdownTheme } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { Box, Markdown, Text } from "@earendil-works/pi-tui";
|
|
3
|
+
import { SURFACE_DEFAULTS } from "./config.js";
|
|
4
|
+
import type { PanelMode, ScrutinyRunProgress, ScrutinyRunResult, PanelResponse } from "./types.js";
|
|
5
|
+
import { formatDuration, formatTokens, truncate } from "./util.js";
|
|
6
|
+
|
|
7
|
+
export function scrutinyStatusText(details: unknown): string {
|
|
8
|
+
if (isResult(details)) {
|
|
9
|
+
const ok = details.responses.filter((response) => response.status === "ok").length;
|
|
10
|
+
const failed = details.failed_models.length;
|
|
11
|
+
const mode = details.panel_mode ? ` ${details.panel_mode}` : "";
|
|
12
|
+
const panel = details.responses.length ? ` ${ok}/${details.responses.length}` : "";
|
|
13
|
+
return `scrutiny ${details.status} ${details.surface}${mode} ${formatDuration(details.durationMs)}${panel}${failed ? ` ${failed} failed` : ""}`;
|
|
14
|
+
}
|
|
15
|
+
if (isProgress(details)) {
|
|
16
|
+
const ready = details.panel.filter((item) => item.status === "ready").length;
|
|
17
|
+
const elapsed = formatDuration(Math.max(0, details.updatedAt - details.startedAt));
|
|
18
|
+
const mode = details.panel_mode ? ` ${details.panel_mode}` : "";
|
|
19
|
+
const panel = details.panel.length ? ` ${ready}/${details.panel.length}` : " verify";
|
|
20
|
+
return `scrutiny ${details.surface}${mode} ${elapsed}${panel} ${progressPhase(details)}`;
|
|
21
|
+
}
|
|
22
|
+
return "scrutiny";
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function renderScrutinyResult(result: any, options: { expanded?: boolean; isPartial?: boolean }, theme: any, context?: any) {
|
|
26
|
+
const details = result.details;
|
|
27
|
+
if (isProgress(details)) return new Text(renderProgress(details, theme), 0, 0);
|
|
28
|
+
if (!isResult(details)) return new Text(result.content?.[0]?.text ?? "scrutiny", 0, 0);
|
|
29
|
+
|
|
30
|
+
if (options.expanded) {
|
|
31
|
+
const markdown = renderExpandedMarkdown(details);
|
|
32
|
+
return new Markdown(markdown, 0, 0, getMarkdownTheme());
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const box = new Box(1, 0, (s: string) => theme.bg(details.status === "ok" ? "toolSuccessBg" : "toolErrorBg", s));
|
|
36
|
+
box.addChild(new Text(renderCompactResult(details, theme), 0, 0));
|
|
37
|
+
return box;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function renderScrutinyCall(args: any, theme: any) {
|
|
41
|
+
const surface = args?.surface ?? "consult";
|
|
42
|
+
const panel = Array.isArray(args?.panel) && args.panel.length ? args.panel : undefined;
|
|
43
|
+
const judgeMode = args?.judgeMode ?? "auto";
|
|
44
|
+
const title = theme.fg("toolTitle", theme.bold("scrutiny_consult"));
|
|
45
|
+
const bits = [chip(theme, surface, "accent"), modeChip(theme, surface), chip(theme, panel ? `${panel.length} models` : "env panel", panel ? "success" : "muted"), chip(theme, `map:${judgeMode}`, judgeMode === "on" ? "warning" : "muted")];
|
|
46
|
+
return new Text(`${title} ${bits.join(" ")}`, 0, 0);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function renderScrutinyMessage(message: any, { expanded }: { expanded?: boolean }, theme: any) {
|
|
50
|
+
const details = message.details;
|
|
51
|
+
if (!isResult(details)) return renderStaticMessage(message, theme);
|
|
52
|
+
if (expanded) return new Markdown(renderExpandedMarkdown(details), 0, 0, getMarkdownTheme());
|
|
53
|
+
const box = new Box(1, 1, (s: string) => theme.bg("customMessageBg", s));
|
|
54
|
+
box.addChild(new Text(renderCompactResult(details, theme), 0, 0));
|
|
55
|
+
return box;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function renderStaticMessage(message: any, theme: any) {
|
|
59
|
+
const content = String(message.content ?? "scrutiny");
|
|
60
|
+
const kind = typeof message.details?.kind === "string" ? message.details.kind : inferStaticKind(content);
|
|
61
|
+
const box = new Box(1, 1, (s: string) => theme.bg("customMessageBg", s));
|
|
62
|
+
const chips = staticChips(kind).map((item) => chip(theme, item, item === "env override" ? "warning" : "muted"));
|
|
63
|
+
box.addChild(new Text(`${theme.fg("accent", "◆")} ${theme.bold("scrutiny")} ${theme.fg("dim", kind)} ${chips.join(" ")}`.trim(), 0, 0));
|
|
64
|
+
box.addChild(new Markdown(stripFirstHeading(content), 0, 0, getMarkdownTheme()));
|
|
65
|
+
return box;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function inferStaticKind(content: string): string {
|
|
69
|
+
const first = content.split(/\r?\n/).find((line) => line.trim())?.trim() ?? "message";
|
|
70
|
+
const match = first.match(/^#\s+scrutiny\s+(\S+)/i) ?? first.match(/^#\s+pi-scrutiny/i);
|
|
71
|
+
if (!match) return "message";
|
|
72
|
+
return match[1]?.toLowerCase() ?? "help";
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function stripFirstHeading(content: string): string {
|
|
76
|
+
return content.replace(/^#\s+[^\n]+\n?/, "").trim() || content;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function staticChips(kind: string): string[] {
|
|
80
|
+
switch (kind) {
|
|
81
|
+
case "help":
|
|
82
|
+
case "pi-scrutiny":
|
|
83
|
+
return ["6 surfaces", "inline", "no patch fusion"];
|
|
84
|
+
case "models":
|
|
85
|
+
return ["panel", "verify", "env override"];
|
|
86
|
+
case "runs":
|
|
87
|
+
return ["session", "artifacts"];
|
|
88
|
+
case "councils":
|
|
89
|
+
return ["presets", "lenses"];
|
|
90
|
+
case "config":
|
|
91
|
+
return ["global", "project", "env override"];
|
|
92
|
+
default:
|
|
93
|
+
return [];
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function chip(theme: any, text: string, color: "accent" | "muted" | "success" | "warning" | "error"): string {
|
|
98
|
+
return theme.fg(color, `[${text}]`);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function modeChip(theme: any, surface: string): string {
|
|
102
|
+
const mode = (SURFACE_DEFAULTS as Partial<Record<string, { panelMode?: PanelMode }>>)[surface]?.panelMode;
|
|
103
|
+
return mode ? chip(theme, mode, mode === "replicate" ? "accent" : "muted") : chip(theme, "no panel", "muted");
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function progressPhase(progress: ScrutinyRunProgress): string {
|
|
107
|
+
if (progress.judge?.status === "running") return "map";
|
|
108
|
+
if (/verify/i.test(progress.message ?? "")) return "verify";
|
|
109
|
+
if (/done/i.test(progress.message ?? "")) return "done";
|
|
110
|
+
if (/failed|unusable|error/i.test(progress.message ?? "")) return "attention";
|
|
111
|
+
return progress.panel.length ? "panel" : "checks";
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export function renderScrutinyDock(progresses: ScrutinyRunProgress[], theme: any): string[] {
|
|
115
|
+
if (progresses.length === 0) return [];
|
|
116
|
+
const lines = [`${theme.fg("accent", "◆")} ${theme.bold("scrutiny")} ${theme.fg("dim", "esc to cancel")}`];
|
|
117
|
+
for (const progress of progresses.slice(0, 1)) {
|
|
118
|
+
const ready = progress.panel.filter((item) => item.status === "ready").length;
|
|
119
|
+
const running = progress.panel.filter((item) => item.status === "running").length;
|
|
120
|
+
const elapsed = formatDuration(Math.max(0, progress.updatedAt - progress.startedAt));
|
|
121
|
+
const mode = progress.panel_mode ? ` ${progress.panel_mode}` : "";
|
|
122
|
+
const status = progress.panel.length ? `${ready}/${progress.panel.length}` : "verify";
|
|
123
|
+
const icon = running ? theme.fg("warning", "◐") : theme.fg("accent", "◆");
|
|
124
|
+
lines.push(` ${icon} ${theme.fg("accent", progress.surface)}${theme.fg("dim", mode)} ${theme.fg("muted", elapsed)} ${theme.fg("dim", status)} ${theme.fg("muted", progressPhase(progress))}`);
|
|
125
|
+
const current = progress.panel.find((item) => item.status === "running");
|
|
126
|
+
if (current) lines.push(` ${theme.fg("warning", "→")} ${theme.fg("toolOutput", current.model)} ${theme.fg("dim", current.role)}`);
|
|
127
|
+
}
|
|
128
|
+
return lines;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function renderProgress(progress: ScrutinyRunProgress, theme: any): string {
|
|
132
|
+
const ready = progress.panel.filter((item) => item.status === "ready").length;
|
|
133
|
+
const elapsed = formatDuration(Math.max(0, progress.updatedAt - progress.startedAt));
|
|
134
|
+
const mode = progress.panel_mode ? ` ${progress.panel_mode}` : "";
|
|
135
|
+
const status = progress.panel.length ? `${ready}/${progress.panel.length}` : "verify";
|
|
136
|
+
const lines: string[] = [];
|
|
137
|
+
lines.push(`${theme.fg("accent", "◐")} ${theme.bold("scrutiny")} ${theme.fg("accent", progress.surface)}${theme.fg("dim", mode)} ${theme.fg("muted", elapsed)} ${theme.fg("dim", status)} ${theme.fg("muted", progressPhase(progress))}`);
|
|
138
|
+
for (const item of progress.panel) {
|
|
139
|
+
const dur = item.endedAt ? formatDuration(item.endedAt - (item.startedAt ?? progress.startedAt)) : "";
|
|
140
|
+
lines.push(` ${statusIcon(item.status, theme)} ${theme.fg("toolOutput", item.model)} ${theme.fg("dim", item.role)}${dur ? ` ${theme.fg("muted", dur)}` : ""}`);
|
|
141
|
+
}
|
|
142
|
+
if (progress.judge) lines.push(` ${statusIcon(progress.judge.status, theme)} ${theme.fg("toolOutput", progress.judge.model)} ${theme.fg("dim", progress.judge.role)}`);
|
|
143
|
+
return lines.join("\n");
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function renderCompactResult(result: ScrutinyRunResult, theme: any): string {
|
|
147
|
+
const ok = result.responses.filter((response) => response.status === "ok");
|
|
148
|
+
const failed = result.responses.filter((response) => response.status === "error");
|
|
149
|
+
const lines: string[] = [];
|
|
150
|
+
const color = result.status === "ok" ? "success" : "error";
|
|
151
|
+
const mode = result.panel_mode ? ` ${result.panel_mode}` : "";
|
|
152
|
+
lines.push(`${theme.fg(color, result.status === "ok" ? "◆" : "✕")} ${theme.bold("scrutiny")} ${theme.fg("accent", result.surface)}${theme.fg("dim", mode)} ${theme.fg("muted", formatDuration(result.durationMs))}`);
|
|
153
|
+
if (result.status === "error" && result.error) {
|
|
154
|
+
lines.push(` ${theme.fg("error", truncate(result.error, 180).replace(/\n/g, " "))}`);
|
|
155
|
+
}
|
|
156
|
+
if (result.responses.length > 0) {
|
|
157
|
+
const judgeBit = result.judge ? ` ${result.judge.status === "ok" ? theme.fg("success", "map") : theme.fg("warning", "map:failed")}` : "";
|
|
158
|
+
lines.push(` ${theme.fg("success", `${ok.length}/${result.responses.length} ready`)}${failed.length ? ` ${theme.fg("warning", `${failed.length} failed`)}` : ""}${judgeBit}`);
|
|
159
|
+
}
|
|
160
|
+
for (const response of result.responses) lines.push(panelLine(response, theme));
|
|
161
|
+
if (result.analysis?.disagreement_signal) lines.push(` ${theme.fg("error", "⚠ disagreement")} ${theme.fg("dim", "stop signal")}`);
|
|
162
|
+
else if (result.panel_mode !== "roles" && result.analysis?.contradictions?.length) lines.push(` ${theme.fg("warning", "contradiction")} ${truncate(result.analysis.contradictions[0]?.topic ?? "", 100).replace(/\n/g, " ")}`);
|
|
163
|
+
else if (result.analysis?.coverage?.length) lines.push(` ${theme.fg("accent", "coverage")} ${truncate(result.analysis.coverage[0] ?? "", 100).replace(/\n/g, " ")}`);
|
|
164
|
+
else if (result.analysis?.consensus?.length) lines.push(` ${theme.fg("accent", "shared")} ${truncate(result.analysis.consensus[0] ?? "", 100).replace(/\n/g, " ")}`);
|
|
165
|
+
if (result.verify) lines.push(` ${theme.fg(result.verify.failed ? "error" : "success", "verify")} ${result.verify.passed} pass ${result.verify.failed} fail ${result.verify.skipped} skip`);
|
|
166
|
+
if (result.packetPath) lines.push(` ${theme.fg("dim", "ctrl+o expand")}`);
|
|
167
|
+
return lines.join("\n");
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function artifactPath(packetPath: string, file: string): string {
|
|
171
|
+
return packetPath.replace(/packet\.md$/, file);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function panelLine(response: PanelResponse, theme: any): string {
|
|
175
|
+
const dur = formatDuration(response.durationMs);
|
|
176
|
+
const usage = response.usage.input || response.usage.output ? ` ${formatTokens(response.usage.input)}↑ ${formatTokens(response.usage.output)}↓` : "";
|
|
177
|
+
const cost = response.usage.cost ? ` $${response.usage.cost.toFixed(4)}` : "";
|
|
178
|
+
return ` ${response.status === "ok" ? theme.fg("success", "●") : theme.fg("error", "×")} ${theme.fg("toolOutput", response.model)} ${theme.fg("dim", response.role)} ${theme.fg("muted", dur)}${theme.fg("dim", usage)}${theme.fg("dim", cost)}`;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function renderExpandedMarkdown(result: ScrutinyRunResult): string {
|
|
182
|
+
const lines: string[] = [];
|
|
183
|
+
lines.push(`# Scrutiny ${result.status}`);
|
|
184
|
+
lines.push(`surface: ${result.surface} `);
|
|
185
|
+
if (result.panel_mode) lines.push(`panel mode: ${result.panel_mode} `);
|
|
186
|
+
lines.push(`duration: ${formatDuration(result.durationMs)} `);
|
|
187
|
+
if (result.packetPath) {
|
|
188
|
+
lines.push(`result: \`${artifactPath(result.packetPath, "result.json")}\``);
|
|
189
|
+
lines.push(`packet: \`${result.packetPath}\``);
|
|
190
|
+
}
|
|
191
|
+
lines.push("");
|
|
192
|
+
if (result.analysis) {
|
|
193
|
+
lines.push("## Evidence map");
|
|
194
|
+
pushList(lines, "Consensus", result.analysis.consensus);
|
|
195
|
+
pushList(lines, "Risks", result.analysis.risks);
|
|
196
|
+
pushList(lines, "Coverage", result.analysis.coverage);
|
|
197
|
+
pushList(lines, "Blind spots", result.analysis.blind_spots);
|
|
198
|
+
if (result.analysis.unique_insights?.length) {
|
|
199
|
+
lines.push("### Unique insights");
|
|
200
|
+
for (const item of result.analysis.unique_insights) lines.push(`- **${item.model}**: ${item.insight}`);
|
|
201
|
+
}
|
|
202
|
+
if (result.panel_mode !== "roles" && result.analysis.contradictions?.length) {
|
|
203
|
+
lines.push("### Contradictions");
|
|
204
|
+
for (const item of result.analysis.contradictions) {
|
|
205
|
+
lines.push(`- ${item.topic}`);
|
|
206
|
+
for (const stance of item.stances) lines.push(` - **${stance.model}**: ${stance.stance}`);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
if (result.analysis.confidence) lines.push(`confidence: ${result.analysis.confidence}`);
|
|
210
|
+
lines.push("");
|
|
211
|
+
}
|
|
212
|
+
const stats = contextStats(result);
|
|
213
|
+
if (stats.hasContext) {
|
|
214
|
+
lines.push("## Context footprint");
|
|
215
|
+
lines.push(`scout candidates: ${stats.candidates} `);
|
|
216
|
+
lines.push(`related memory: ${stats.memory} `);
|
|
217
|
+
lines.push(`missing-context signals: ${stats.gaps}`);
|
|
218
|
+
lines.push("");
|
|
219
|
+
}
|
|
220
|
+
lines.push("## Panel outputs");
|
|
221
|
+
for (const response of result.responses) {
|
|
222
|
+
lines.push(`### ${response.model} (${response.role})`);
|
|
223
|
+
if (response.status === "error") lines.push(`error: ${response.error ?? "unknown"}`);
|
|
224
|
+
else lines.push(response.content);
|
|
225
|
+
lines.push("");
|
|
226
|
+
}
|
|
227
|
+
if (result.judge) {
|
|
228
|
+
lines.push("## Trade-off explainer raw output");
|
|
229
|
+
lines.push(result.judge.status === "ok" ? result.judge.content : result.judge.error ?? "trade-off explainer failed");
|
|
230
|
+
}
|
|
231
|
+
if (result.verify) {
|
|
232
|
+
lines.push("## Verify (objective arbiter)");
|
|
233
|
+
lines.push(`${result.verify.passed} passed · ${result.verify.failed} failed · ${result.verify.skipped} skipped · ${formatDuration(result.verify.durationMs)}`);
|
|
234
|
+
if (result.verify.diffStat) lines.push("```", result.verify.diffStat.trim(), "```");
|
|
235
|
+
for (const check of result.verify.checks) {
|
|
236
|
+
const icon = check.status === "pass" ? "✓" : check.status === "fail" ? "✕" : check.status === "error" ? "!" : "–";
|
|
237
|
+
lines.push(`- ${icon} ${check.name} (${check.status})`);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
return lines.join("\n");
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function contextStats(result: ScrutinyRunResult): { hasContext: boolean; candidates: number; memory: number; gaps: number } {
|
|
244
|
+
const packet = result.packet ?? "";
|
|
245
|
+
const scout = /^## Context scout\b/m.test(packet);
|
|
246
|
+
const candidateLines = section(packet, "Context scout")
|
|
247
|
+
?.split(/\r?\n/)
|
|
248
|
+
.filter((line) => /^-\s+/.test(line.trim())) ?? [];
|
|
249
|
+
const memory = candidateLines.filter((line) => /\[prior;/i.test(line) || /\bscr_[a-z0-9]/i.test(line)).length;
|
|
250
|
+
const scoutGaps = /\b(skipped:|no local candidates found|ask user to choose scope|inspect scope manually)\b/i.test(packet) ? 1 : 0;
|
|
251
|
+
const missing = missingContextSignals(result);
|
|
252
|
+
return {
|
|
253
|
+
hasContext: scout || missing > 0,
|
|
254
|
+
candidates: candidateLines.length,
|
|
255
|
+
memory,
|
|
256
|
+
gaps: scoutGaps + missing,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function missingContextSignals(result: ScrutinyRunResult): number {
|
|
261
|
+
const lines = [
|
|
262
|
+
...(result.analysis?.blind_spots ?? []),
|
|
263
|
+
...result.responses.flatMap((response) => response.content.split(/\r?\n/)),
|
|
264
|
+
]
|
|
265
|
+
.map((line) => line.trim().replace(/^[-*•]\s+/, "").replace(/^\d+[.)]\s+/, ""))
|
|
266
|
+
.filter((line) => line.length >= 20 && line.length <= 500)
|
|
267
|
+
.filter((line) => !/^Deterministic analysis does not infer/i.test(line))
|
|
268
|
+
.filter((line) => /\b(missing|not shown|not in (the )?packet|insufficient|unknown|cannot determine|can't determine|need(?:s)? to inspect|must inspect|would need|need more evidence|not enough evidence)\b/i.test(line));
|
|
269
|
+
return new Set(lines.map((line) => truncate(line, 240))).size;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function section(markdown: string, heading: string): string | undefined {
|
|
273
|
+
const lines = markdown.split(/\r?\n/);
|
|
274
|
+
const start = lines.findIndex((line) => line.trim() === `## ${heading}`);
|
|
275
|
+
if (start < 0) return undefined;
|
|
276
|
+
const next = lines.findIndex((line, index) => index > start && /^##\s+/.test(line));
|
|
277
|
+
return lines.slice(start + 1, next < 0 ? undefined : next).join("\n").trim();
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function pushList(lines: string[], title: string, items: string[] | undefined): void {
|
|
281
|
+
if (!items?.length) return;
|
|
282
|
+
lines.push(`### ${title}`);
|
|
283
|
+
for (const item of items) lines.push(`- ${item}`);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
function statusIcon(status: string, theme: any): string {
|
|
287
|
+
if (status === "ready") return theme.fg("success", "●");
|
|
288
|
+
if (status === "running") return theme.fg("warning", "◐");
|
|
289
|
+
if (status === "failed") return theme.fg("error", "×");
|
|
290
|
+
return theme.fg("dim", "○");
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function isProgress(value: unknown): value is ScrutinyRunProgress {
|
|
294
|
+
return Boolean(value && typeof value === "object" && (value as any).runId && Array.isArray((value as any).panel) && (value as any).status === "running");
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function isResult(value: unknown): value is ScrutinyRunResult {
|
|
298
|
+
return Boolean(value && typeof value === "object" && (value as any).runId && Array.isArray((value as any).responses) && ((value as any).status === "ok" || (value as any).status === "error"));
|
|
299
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import type { ScrutinyAnalysis, ScrutinyUsage } from "./types.js";
|
|
5
|
+
|
|
6
|
+
export function createRunId(): string {
|
|
7
|
+
return `scr_${Date.now().toString(36)}_${randomBytes(3).toString("hex")}`;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export function truncate(text: string, maxChars: number): string {
|
|
11
|
+
if (maxChars <= 0) return "";
|
|
12
|
+
if (text.length <= maxChars) return text;
|
|
13
|
+
return `${text.slice(0, maxChars)}\n\n[truncated ${text.length - maxChars} chars]`;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function formatDuration(ms: number): string {
|
|
17
|
+
if (ms < 1_000) return `${ms}ms`;
|
|
18
|
+
if (ms < 60_000) return `${(ms / 1_000).toFixed(1)}s`;
|
|
19
|
+
return `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1_000)}s`;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function formatTokens(count: number): string {
|
|
23
|
+
if (!count) return "0";
|
|
24
|
+
if (count < 1_000) return String(count);
|
|
25
|
+
if (count < 10_000) return `${(count / 1_000).toFixed(1)}k`;
|
|
26
|
+
return `${Math.round(count / 1_000)}k`;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function usageZero(): ScrutinyUsage {
|
|
30
|
+
return { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0, contextTokens: 0, turns: 0 };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function addUsage(a: ScrutinyUsage, b: Partial<ScrutinyUsage> | undefined): ScrutinyUsage {
|
|
34
|
+
return {
|
|
35
|
+
input: a.input + (b?.input ?? 0),
|
|
36
|
+
output: a.output + (b?.output ?? 0),
|
|
37
|
+
cacheRead: a.cacheRead + (b?.cacheRead ?? 0),
|
|
38
|
+
cacheWrite: a.cacheWrite + (b?.cacheWrite ?? 0),
|
|
39
|
+
cost: a.cost + (b?.cost ?? 0),
|
|
40
|
+
contextTokens: Math.max(a.contextTokens, b?.contextTokens ?? 0),
|
|
41
|
+
turns: a.turns + (b?.turns ?? 0),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function getAssistantText(message: any): string {
|
|
46
|
+
if (!message || message.role !== "assistant") return "";
|
|
47
|
+
const content = message.content;
|
|
48
|
+
if (typeof content === "string") return content;
|
|
49
|
+
if (!Array.isArray(content)) return "";
|
|
50
|
+
return content
|
|
51
|
+
.map((part) => {
|
|
52
|
+
if (!part || typeof part !== "object") return "";
|
|
53
|
+
if (part.type === "text") return String(part.text ?? "");
|
|
54
|
+
return "";
|
|
55
|
+
})
|
|
56
|
+
.filter(Boolean)
|
|
57
|
+
.join("\n");
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function extractUsage(message: any): Partial<ScrutinyUsage> | undefined {
|
|
61
|
+
const usage = message?.usage;
|
|
62
|
+
if (!usage || typeof usage !== "object") return undefined;
|
|
63
|
+
return {
|
|
64
|
+
input: Number(usage.input ?? 0),
|
|
65
|
+
output: Number(usage.output ?? 0),
|
|
66
|
+
cacheRead: Number(usage.cacheRead ?? 0),
|
|
67
|
+
cacheWrite: Number(usage.cacheWrite ?? 0),
|
|
68
|
+
cost: Number(usage.cost?.total ?? usage.cost ?? 0),
|
|
69
|
+
contextTokens: Number(usage.totalTokens ?? 0),
|
|
70
|
+
turns: 1,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function parseAnalysisJson(text: string): ScrutinyAnalysis | undefined {
|
|
75
|
+
const trimmed = text.trim();
|
|
76
|
+
const candidates = [trimmed, stripFence(trimmed), firstJsonObject(trimmed)].filter((value): value is string => Boolean(value));
|
|
77
|
+
for (const candidate of candidates) {
|
|
78
|
+
try {
|
|
79
|
+
const parsed = JSON.parse(candidate) as ScrutinyAnalysis;
|
|
80
|
+
if (parsed && typeof parsed === "object") return parsed;
|
|
81
|
+
} catch {
|
|
82
|
+
// try next
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function safeMkdir(dir: string): void {
|
|
89
|
+
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function scrutinyDataDir(cwd: string): string {
|
|
93
|
+
return path.join(cwd, ".pi", "scrutiny");
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function stripFence(text: string): string | undefined {
|
|
97
|
+
const match = text.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
|
|
98
|
+
return match?.[1]?.trim();
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function firstJsonObject(text: string): string | undefined {
|
|
102
|
+
const start = text.indexOf("{");
|
|
103
|
+
if (start < 0) return undefined;
|
|
104
|
+
let depth = 0;
|
|
105
|
+
let inString = false;
|
|
106
|
+
let escaped = false;
|
|
107
|
+
for (let i = start; i < text.length; i++) {
|
|
108
|
+
const char = text[i];
|
|
109
|
+
if (inString) {
|
|
110
|
+
if (escaped) escaped = false;
|
|
111
|
+
else if (char === "\\") escaped = true;
|
|
112
|
+
else if (char === '"') inString = false;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
if (char === '"') inString = true;
|
|
116
|
+
else if (char === "{") depth++;
|
|
117
|
+
else if (char === "}") {
|
|
118
|
+
depth--;
|
|
119
|
+
if (depth === 0) return text.slice(start, i + 1);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return undefined;
|
|
123
|
+
}
|