@harnessforce/claude-code-plugin 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +34 -13
- package/commands/help.md +31 -0
- package/commands/import.md +18 -0
- package/commands/run.md +25 -0
- package/commands/setup.md +29 -47
- package/commands/status.md +26 -0
- package/commands/tune.md +18 -18
- package/package.json +1 -1
- package/scripts/harnessforce-hook-main.cjs +539 -154
- package/scripts/harnessforce-hook.cjs +2 -1
- package/skills/propose-improvements/SKILL.md +69 -69
- package/skills/propose-improvements/scripts/summarize.mjs +59 -57
- package/skills/record-run/SKILL.md +14 -51
- package/skills/using-harnessforce/SKILL.md +32 -32
|
@@ -14,8 +14,9 @@ if (major >= 18) {
|
|
|
14
14
|
// 繰り返し表示しないよう、session開始のhookだけが出す。
|
|
15
15
|
if (process.argv[2] === "session-start") {
|
|
16
16
|
process.stdout.write(
|
|
17
|
+
// biome-ignore lint/style/useTemplate: 古いNode.jsがES5として解析できる形を保つ。templateリテラルはES2015である。
|
|
17
18
|
JSON.stringify({
|
|
18
|
-
systemMessage: "Harnessforce
|
|
19
|
+
systemMessage: "Harnessforce hooks require Node.js 18 or later",
|
|
19
20
|
}) + "\n"
|
|
20
21
|
);
|
|
21
22
|
}
|
|
@@ -1,85 +1,85 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: propose-improvements
|
|
3
|
-
description: /harnessforce:tune
|
|
3
|
+
description: From the analysis results of /harnessforce:tune, make suggested improvements for human interventions, repeatable loops, MCP servers, and usage (context compaction, cache reuse, model choice), record them with harnessforce tune record, and then show them. Use it only for targets that /harnessforce:tune marks as "a suggestion can be made". Does not modify files.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
#
|
|
6
|
+
# Make improvement suggestions from the analysis results
|
|
7
7
|
|
|
8
|
-
`/harnessforce:tune
|
|
8
|
+
For each target marked "a suggestion can be made" in the "Targets for suggestions" printed by `summarize.mjs` of `/harnessforce:tune`, make a suggestion. Make each suggestion in a form the user can apply as it is, and record it before showing it.
|
|
9
9
|
|
|
10
|
-
##
|
|
10
|
+
## Do not modify anything
|
|
11
11
|
|
|
12
|
-
-
|
|
13
|
-
-
|
|
12
|
+
- Do not modify the user's files, settings, repository, or managed settings. Do not use the Edit or Write tools, `git apply`, `git commit`, creating a branch, or `gh pr create`. The diff and the Pull Request draft are only shown.
|
|
13
|
+
- The only thing written is the record that `harnessforce tune record` writes to `~/.harnessforce/tune/`. Do not create temporary files either.
|
|
14
14
|
|
|
15
|
-
##
|
|
15
|
+
## Conditions for making a suggestion
|
|
16
16
|
|
|
17
|
-
`summarize.mjs
|
|
17
|
+
The result of `summarize.mjs` is authoritative. Make no suggestion for a target marked "Not enough data" or "not measured, so no suggestion is made". The conditions are as follows.
|
|
18
18
|
|
|
19
|
-
|
|
|
19
|
+
| Target | Minimum condition for a suggestion |
|
|
20
20
|
| --- | --- |
|
|
21
|
-
|
|
|
22
|
-
|
|
|
23
|
-
|
|
|
24
|
-
| MCP
|
|
25
|
-
|
|
|
26
|
-
|
|
|
21
|
+
| All | The analyzed range has at least 10 sessions |
|
|
22
|
+
| Human interventions | That `intervention.kind` appears in at least 3 sessions and at least 10 times in total |
|
|
23
|
+
| Repeatable loops | That `loop.kind` appears at least 3 times in total |
|
|
24
|
+
| MCP servers | At least 10 sessions have `configured` |
|
|
25
|
+
| `frequent_compaction` and `low_cache_reuse` of usage | At least 3 sessions match that `usage.kind` |
|
|
26
|
+
| `model_choice` of usage | At least 10 sessions have `models`, and in the total of `models` over the analyzed range, one `message.model` value accounts for at least 90% of the total `output_tokens` |
|
|
27
27
|
|
|
28
|
-
-
|
|
29
|
-
-
|
|
30
|
-
-
|
|
31
|
-
-
|
|
32
|
-
-
|
|
28
|
+
- Do not use a value that was not measured (`not_measured`) as evidence. Do not treat "not measured" as 0.
|
|
29
|
+
- The change type of a usage suggestion is limited to `claude_md`, `rule`, `skill`, `agent`, and `command` (for example, split up a bloated CLAUDE.md, or move exploration to a subagent and specify that agent's `model`). Do not use another change type such as the `model` in settings. `harnessforce tune record` rejects other change types. A change to `claude_md` is not detected when applied, so it does not appear in the before and after of the previous suggestion either.
|
|
30
|
+
- Usage values (`usage`) stay on this device and are not sent to Harnessforce. You may use them as evidence for a suggestion.
|
|
31
|
+
- Do not make a suggestion about the harness configuration itself (differences in results between skills, rules, agents, commands, hooks, MCP servers, and model versions).
|
|
32
|
+
- If you investigate and find no change the user can apply as it is, make no suggestion for that target and say why in one line.
|
|
33
33
|
|
|
34
|
-
##
|
|
34
|
+
## Gather evidence
|
|
35
35
|
|
|
36
|
-
-
|
|
37
|
-
-
|
|
38
|
-
-
|
|
36
|
+
- Read only the parts you need of the record at the `transcript_path` of the target's evidence sessions. For human interventions, read the prompts of that kind and the agent response just before them. For loops, read the steps of one cycle (fetching the Issue, fixing, testing, pushing, creating the Pull Request, and so on) and the points where a person intervened. For MCP servers, read the calls and failures. For usage, read the instructions and the course of events before compaction for `frequent_compaction`, the responses before and after where the cache was not reused for `low_cache_reuse`, and the exploration and routine work done by the main agent for `model_choice`.
|
|
37
|
+
- When you make a suggestion for the same target as a suggestion that "Before and after the previous suggestion" shows as "Did not move in the improving direction", cite that before and after as evidence and do not repeat the same diff as last time.
|
|
38
|
+
- Read the current definition of the target (settings, skill, rule, agent, command, hook, `.mcp.json`, `mcpServers` in `~/.claude.json`, CLAUDE.md) and make the diff against the current content.
|
|
39
39
|
|
|
40
|
-
##
|
|
40
|
+
## Format
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
Give each suggestion the following.
|
|
43
43
|
|
|
44
|
-
|
|
|
44
|
+
| Item | Content |
|
|
45
45
|
| --- | --- |
|
|
46
|
-
|
|
|
47
|
-
|
|
|
48
|
-
|
|
|
49
|
-
|
|
|
50
|
-
|
|
|
51
|
-
|
|
|
52
|
-
|
|
|
46
|
+
| Target category | One of `intervention.kind=<kind>`, `loop.kind=<kind>`, `mcp_server=<identifier>`, `usage.kind=<kind>`. The `<kind>` of `usage.kind` is one of `frequent_compaction`, `low_cache_reuse`, `model_choice` |
|
|
47
|
+
| Evidence | The period, the counts, the times of the matching sessions, and excerpts of the records. Excerpts are shown on this device only |
|
|
48
|
+
| Change type | One of `permissions`, `hook`, `skill`, `rule`, `agent`, `command`, `loop_prompt`, `mcp_config`, `claude_md` |
|
|
49
|
+
| Apply to | The user scope, the repository scope, or managed settings, and the file path |
|
|
50
|
+
| Change | A unified diff that can be applied as it is. For the repository scope, a Pull Request draft made of the branch name, the title, the body, and the diff. For managed settings, the diff that an Owner or Admin pastes |
|
|
51
|
+
| Expected effect | Which category value decreases, and which existing metric (CI failure rate, rework rate, cost per completed Issue) changes |
|
|
52
|
+
| How to measure | The ComponentVersion that changes when applied (kind and identifier), and the metric used for comparison |
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
The change type and the component that changes when applied correspond as follows.
|
|
55
55
|
|
|
56
|
-
|
|
|
56
|
+
| Change type | Component `kind` | Component identifier (`id`) | Expected content |
|
|
57
57
|
| --- | --- | --- | --- |
|
|
58
|
-
| `permissions` | `permissions` | `permissions` |
|
|
59
|
-
| `hook` | `hook` |
|
|
60
|
-
| `skill` | `skill` |
|
|
61
|
-
| `rule` | `rule` | `CLAUDE.md
|
|
62
|
-
| `agent` | `agent` | `agents
|
|
63
|
-
| `command` | `command` | `commands
|
|
64
|
-
| `loop_prompt` |
|
|
65
|
-
| `mcp_config` | `mcp_server
|
|
66
|
-
| `claude_md` |
|
|
58
|
+
| `permissions` | `permissions` | `permissions` | The whole `permissions` value of the settings after applying (`value`) |
|
|
59
|
+
| `hook` | `hook` | The event name of the hook | The value of that event under `hooks` in the settings after applying (`value`) |
|
|
60
|
+
| `skill` | `skill` | The directory name of the skill | The whole content of `SKILL.md` after applying (`content`) |
|
|
61
|
+
| `rule` | `rule` | `CLAUDE.md` or `rules/<path>` | The whole content of the file after applying (`content`) |
|
|
62
|
+
| `agent` | `agent` | The path from `agents/` without `.md`, with `/` replaced by `:` | The whole content of the file after applying (`content`) |
|
|
63
|
+
| `command` | `command` | The path from `commands/` without `.md`, with `/` replaced by `:` | The whole content of the file after applying (`content`) |
|
|
64
|
+
| `loop_prompt` | `command` or `skill`, following the format of the file written | Same as `command` or `skill` above | The whole content of the file after applying (`content`) |
|
|
65
|
+
| `mcp_config` | `mcp_server` (a diff that deletes the server has no component) | The server name | The definition of the server after applying (`value`). For deletion, the whole content of the file after applying (`content`) |
|
|
66
|
+
| `claude_md` | None | None | The whole content of CLAUDE.md after applying (`content`) |
|
|
67
67
|
|
|
68
|
-
component
|
|
68
|
+
The component `source` is the scope it applies to (`user`, `repository`, `local`, `managed`). A server in `.mcp.json` is `repository`, the top-level `mcpServers` of `~/.claude.json` is `user`, and the one under its `projects` is `local`.
|
|
69
69
|
|
|
70
|
-
##
|
|
70
|
+
## Suggestions to reduce permissions
|
|
71
71
|
|
|
72
|
-
-
|
|
73
|
-
-
|
|
72
|
+
- A suggestion to reduce answers to permission prompts takes the tools that asked for confirmation and the breakdown of their permissions as evidence. Allow only the smallest range of tools and commands that appeared in the evidence (for example `Bash(pnpm test:*)`, `Bash(git status)`, `mcp__github__get_issue`).
|
|
73
|
+
- Do not suggest a rule that allows all commands. Do not include in any suggestion a permission for a whole tool or all commands such as `Bash`, `Bash(*)`, `Bash(:*)`, or `*`, nor `"defaultMode": "bypassPermissions"` or `--dangerously-skip-permissions`.
|
|
74
74
|
|
|
75
|
-
##
|
|
75
|
+
## Suggestions for loops
|
|
76
76
|
|
|
77
|
-
-
|
|
78
|
-
- prompt
|
|
77
|
+
- A suggestion for a repeatable loop is a prompt in the form of a command or a skill (`loop_prompt`), and it must include stopping conditions. The stopping conditions are both the maximum number of repetitions and the points where a person checks (for example before creating a Pull Request, or when the same failure happens twice in a row).
|
|
78
|
+
- In the prompt, replace each point where a person intervened in the evidence sessions with either a stopping condition or a step the agent does by itself.
|
|
79
79
|
|
|
80
|
-
##
|
|
80
|
+
## Record, then show
|
|
81
81
|
|
|
82
|
-
|
|
82
|
+
Before showing a suggestion, run `harnessforce tune record` for each suggestion and pass one UTF-8 JSON object on stdin. Pass the input with a heredoc and do not write it to a file. Use a delimiter word that does not appear in the input.
|
|
83
83
|
|
|
84
84
|
```sh
|
|
85
85
|
harnessforce tune record <<'HF_TUNE_RECORD_7f3c'
|
|
@@ -87,19 +87,19 @@ harnessforce tune record <<'HF_TUNE_RECORD_7f3c'
|
|
|
87
87
|
HF_TUNE_RECORD_7f3c
|
|
88
88
|
```
|
|
89
89
|
|
|
90
|
-
|
|
|
90
|
+
| Item | Required | Content |
|
|
91
91
|
| --- | --- | --- |
|
|
92
|
-
| `category` |
|
|
93
|
-
| `change_type` |
|
|
94
|
-
| `scope` |
|
|
95
|
-
| `path` |
|
|
96
|
-
| `project_root` | `scope
|
|
97
|
-
| `component` |
|
|
98
|
-
| `content` | `component
|
|
99
|
-
| `value` | `component
|
|
100
|
-
| `evidence_session_ids` |
|
|
101
|
-
| `body` |
|
|
102
|
-
|
|
103
|
-
- `harnessforce tune record
|
|
104
|
-
-
|
|
105
|
-
-
|
|
92
|
+
| `category` | yes | The target category |
|
|
93
|
+
| `change_type` | yes | The change type |
|
|
94
|
+
| `scope` | yes | One of `user`, `repository`, `local`, `managed` |
|
|
95
|
+
| `path` | yes | The absolute path of the file the change writes |
|
|
96
|
+
| `project_root` | required when `scope` is `repository` or `local` | The absolute path of the root of the repository it applies to. Do not pass it otherwise |
|
|
97
|
+
| `component` | required for a change that has a component in the table above | `kind`, `source`, `id` |
|
|
98
|
+
| `content` | required when the `kind` of `component` is `rule`, `skill`, `agent`, or `command`, and when there is no component | The whole content of the file at `path` after applying |
|
|
99
|
+
| `value` | required when the `kind` of `component` is `hook`, `permissions`, or `mcp_server` | The settings value (JSON) of that component after applying |
|
|
100
|
+
| `evidence_session_ids` | yes | The session IDs of the evidence sessions (at least 1). Use only those shown in the evidence sessions |
|
|
101
|
+
| `body` | yes | The body of the suggestion (markdown): the evidence, the change, the expected effect, and how to measure |
|
|
102
|
+
|
|
103
|
+
- Show only the suggestions for which `harnessforce tune record` ended with exit code 0. Do not show a suggestion that ended with a non-zero code, and tell the user the message on stderr.
|
|
104
|
+
- Record the same suggestion with the same input. With the same `category`, `change_type`, `path`, and content after applying, it counts as one no matter how many times it is recorded.
|
|
105
|
+
- When showing, present the target category, the evidence, the change type, where to apply, the change, the expected effect, and how to measure, in this order.
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
// Node.js 18で依存なしに動かすため、plainなESMで書く。
|
|
4
4
|
import { spawnSync } from "node:child_process";
|
|
5
5
|
|
|
6
|
-
const USAGE = "
|
|
6
|
+
const USAGE = "Usage: /harnessforce:tune [--all] [--no-send]";
|
|
7
7
|
const HF_MISSING =
|
|
8
|
-
"`harnessforce
|
|
8
|
+
"`harnessforce` was not found. Install the CLI with `/harnessforce:setup`";
|
|
9
9
|
const ALLOWED_FLAGS = new Set(["--all", "--no-send"]);
|
|
10
10
|
|
|
11
11
|
const MINIMUM_SESSIONS = 10;
|
|
@@ -54,7 +54,7 @@ const median = (values) => {
|
|
|
54
54
|
: (sorted[middle - 1] + sorted[middle]) / 2;
|
|
55
55
|
};
|
|
56
56
|
|
|
57
|
-
const secondsText = (value) => (value === null ? "—" : `${value.toFixed(1)}
|
|
57
|
+
const secondsText = (value) => (value === null ? "—" : `${value.toFixed(1)}s`);
|
|
58
58
|
|
|
59
59
|
const newestFirst = (a, b) =>
|
|
60
60
|
a.session.started_at < b.session.started_at
|
|
@@ -116,13 +116,13 @@ function mcpServers(sessions) {
|
|
|
116
116
|
|
|
117
117
|
function evidenceLines(hits, unit) {
|
|
118
118
|
const sorted = [...hits].sort(newestFirst);
|
|
119
|
-
const lines = ["
|
|
119
|
+
const lines = [" Evidence sessions (newest first):"];
|
|
120
120
|
for (const { session, count } of sorted.slice(0, MAX_EVIDENCE))
|
|
121
121
|
lines.push(
|
|
122
122
|
` ${session.session_id} ${session.started_at} ${session.repository ?? "-"} sendable=${session.sendable} ${count}${unit} ${session.transcript_path}`,
|
|
123
123
|
);
|
|
124
124
|
if (sorted.length > MAX_EVIDENCE)
|
|
125
|
-
lines.push(`
|
|
125
|
+
lines.push(` and ${sorted.length - MAX_EVIDENCE} more`);
|
|
126
126
|
return lines;
|
|
127
127
|
}
|
|
128
128
|
|
|
@@ -130,7 +130,7 @@ const missing = (parts) =>
|
|
|
130
130
|
parts
|
|
131
131
|
.filter(([amount]) => amount > 0)
|
|
132
132
|
.map(([amount, unit]) => `${amount}${unit}`)
|
|
133
|
-
.join("
|
|
133
|
+
.join(", ");
|
|
134
134
|
|
|
135
135
|
const percent = (share) => `${(share * 100).toFixed(1)}%`;
|
|
136
136
|
|
|
@@ -165,13 +165,13 @@ function modelShares(sessions) {
|
|
|
165
165
|
|
|
166
166
|
const sharesText = (shares) =>
|
|
167
167
|
shares === null
|
|
168
|
-
? "
|
|
168
|
+
? "none"
|
|
169
169
|
: [
|
|
170
|
-
...shares.main.map((m) =>
|
|
170
|
+
...shares.main.map((m) => `main ${m.model} ${percent(m.share)}`),
|
|
171
171
|
...shares.subagent.map(
|
|
172
172
|
(m) => `subagent ${m.model} ${percent(m.share)}`,
|
|
173
173
|
),
|
|
174
|
-
].join("
|
|
174
|
+
].join(", ");
|
|
175
175
|
|
|
176
176
|
// 「作る条件」の`model_choice`: 分析した範囲の`models`の合計で、output_tokensが最も多いmodelとその割合。
|
|
177
177
|
function topMainModel(sessions) {
|
|
@@ -190,7 +190,7 @@ function topMainModel(sessions) {
|
|
|
190
190
|
|
|
191
191
|
const valueText = (value) =>
|
|
192
192
|
value === null
|
|
193
|
-
? "
|
|
193
|
+
? "no value"
|
|
194
194
|
: typeof value === "number"
|
|
195
195
|
? String(Number(value.toFixed(3)))
|
|
196
196
|
: sharesText(value);
|
|
@@ -203,17 +203,19 @@ function improvesBy(category) {
|
|
|
203
203
|
function renderFollowups(followups) {
|
|
204
204
|
if (!Array.isArray(followups) || followups.length === 0) return [];
|
|
205
205
|
const lines = [
|
|
206
|
-
"
|
|
206
|
+
"Before and after the previous suggestion (Before/after difference (does not show causation). The starting point is the time the application was detected):",
|
|
207
207
|
];
|
|
208
208
|
for (const f of followups) {
|
|
209
209
|
lines.push(
|
|
210
|
-
`- ${f.category}
|
|
211
|
-
`
|
|
212
|
-
`
|
|
210
|
+
`- ${f.category} (${f.change_type} ${f.path}) starting point ${f.origin}`,
|
|
211
|
+
` Before ${f.before.from} to ${f.before.to}: ${f.before.sessions} session(s), ${valueText(f.before.value)}`,
|
|
212
|
+
` After ${f.after.from} to ${f.after.to}: ${f.after.sessions} session(s), ${valueText(f.after.value)}`,
|
|
213
213
|
);
|
|
214
|
-
if (f.no_comparison_data) lines.push("
|
|
214
|
+
if (f.no_comparison_data) lines.push(" No comparison data");
|
|
215
215
|
if (f.before_outside_range)
|
|
216
|
-
lines.push(
|
|
216
|
+
lines.push(
|
|
217
|
+
" Part of the earlier period is outside the analysis range",
|
|
218
|
+
);
|
|
217
219
|
const direction = improvesBy(f.category);
|
|
218
220
|
const isImproved =
|
|
219
221
|
direction === "decrease"
|
|
@@ -221,7 +223,7 @@ function renderFollowups(followups) {
|
|
|
221
223
|
: f.after.value > f.before.value;
|
|
222
224
|
if (direction !== undefined && !f.no_comparison_data && !isImproved)
|
|
223
225
|
lines.push(
|
|
224
|
-
"
|
|
226
|
+
" Did not move in the improving direction. Use this as the basis when making a suggestion for the same target",
|
|
225
227
|
);
|
|
226
228
|
}
|
|
227
229
|
lines.push("");
|
|
@@ -235,18 +237,18 @@ function usageTargets(sessions) {
|
|
|
235
237
|
for (const kind of SESSION_USAGE_KINDS) {
|
|
236
238
|
const category = `usage.kind=${kind}`;
|
|
237
239
|
if (withUsage.length === 0) {
|
|
238
|
-
lines.push(`- ${category}:
|
|
240
|
+
lines.push(`- ${category}: not measured, so no suggestion is made`);
|
|
239
241
|
continue;
|
|
240
242
|
}
|
|
241
243
|
const hits = withUsage.filter((s) => s.usage.kinds.includes(kind));
|
|
242
244
|
if (hits.length < USAGE_MIN_SESSIONS) {
|
|
243
245
|
lines.push(
|
|
244
|
-
`- ${category}:
|
|
246
|
+
`- ${category}: Not enough data: needs at least ${USAGE_MIN_SESSIONS} matching sessions (${USAGE_MIN_SESSIONS - hits.length} more session(s) needed)`,
|
|
245
247
|
);
|
|
246
248
|
continue;
|
|
247
249
|
}
|
|
248
250
|
lines.push(
|
|
249
|
-
`- ${category}:
|
|
251
|
+
`- ${category}: a suggestion can be made (${hits.length} session(s))`,
|
|
250
252
|
...evidenceLines(
|
|
251
253
|
hits.map((session) => ({
|
|
252
254
|
session,
|
|
@@ -256,8 +258,8 @@ function usageTargets(sessions) {
|
|
|
256
258
|
: Number(session.usage.cache_reuse_ratio.toFixed(3)),
|
|
257
259
|
})),
|
|
258
260
|
kind === "frequent_compaction"
|
|
259
|
-
? "
|
|
260
|
-
: "
|
|
261
|
+
? " auto-compaction(s)"
|
|
262
|
+
: " (cache reuse ratio)",
|
|
261
263
|
),
|
|
262
264
|
);
|
|
263
265
|
}
|
|
@@ -266,24 +268,24 @@ function usageTargets(sessions) {
|
|
|
266
268
|
(s) => Array.isArray(s.usage.models) && s.usage.models.length > 0,
|
|
267
269
|
);
|
|
268
270
|
if (withUsage.length === 0) {
|
|
269
|
-
lines.push(`- ${category}:
|
|
271
|
+
lines.push(`- ${category}: not measured, so no suggestion is made`);
|
|
270
272
|
return lines;
|
|
271
273
|
}
|
|
272
274
|
if (withModels.length < MODEL_CHOICE_MIN_SESSIONS) {
|
|
273
275
|
lines.push(
|
|
274
|
-
`- ${category}:
|
|
276
|
+
`- ${category}: Not enough data: needs at least ${MODEL_CHOICE_MIN_SESSIONS} sessions with models (${MODEL_CHOICE_MIN_SESSIONS - withModels.length} more session(s) needed)`,
|
|
275
277
|
);
|
|
276
278
|
return lines;
|
|
277
279
|
}
|
|
278
280
|
const top = topMainModel(withModels);
|
|
279
281
|
if (top === undefined || top.share < MODEL_CHOICE_MIN_SHARE) {
|
|
280
282
|
lines.push(
|
|
281
|
-
`- ${category}:
|
|
283
|
+
`- ${category}: not suggested: no single model accounts for ${MODEL_CHOICE_MIN_SHARE * 100}% or more of the main output_tokens${top ? ` (largest: ${top.model} ${percent(top.share)})` : ""}`,
|
|
282
284
|
);
|
|
283
285
|
return lines;
|
|
284
286
|
}
|
|
285
287
|
lines.push(
|
|
286
|
-
`- ${category}:
|
|
288
|
+
`- ${category}: a suggestion can be made (${withModels.length} session(s), ${top.model} ${percent(top.share)})`,
|
|
287
289
|
...evidenceLines(
|
|
288
290
|
withModels.map((session) => ({
|
|
289
291
|
session,
|
|
@@ -291,7 +293,7 @@ function usageTargets(sessions) {
|
|
|
291
293
|
session.usage.models.find((m) => m.model === top.model)
|
|
292
294
|
?.output_tokens ?? 0,
|
|
293
295
|
})),
|
|
294
|
-
` output_tokens
|
|
296
|
+
` output_tokens (${top.model})`,
|
|
295
297
|
),
|
|
296
298
|
);
|
|
297
299
|
return lines;
|
|
@@ -300,9 +302,9 @@ function usageTargets(sessions) {
|
|
|
300
302
|
function render(output) {
|
|
301
303
|
const { sessions } = output;
|
|
302
304
|
const lines = [
|
|
303
|
-
|
|
305
|
+
`Sessions analyzed: ${output.session_count} (analyzer ${output.analyzer_version}, parser ${output.parser_version})`,
|
|
304
306
|
"",
|
|
305
|
-
"
|
|
307
|
+
"Human interventions (count, sessions with interventions, median wait time):",
|
|
306
308
|
];
|
|
307
309
|
const interventions = INTERVENTION_KINDS.map((kind) => ({
|
|
308
310
|
kind,
|
|
@@ -316,13 +318,13 @@ function render(output) {
|
|
|
316
318
|
for (const { kind, result } of interventions)
|
|
317
319
|
lines.push(
|
|
318
320
|
result === undefined
|
|
319
|
-
? ` ${kind}:
|
|
320
|
-
: ` ${kind}: ${result.total}
|
|
321
|
+
? ` ${kind}: not measured`
|
|
322
|
+
: ` ${kind}: ${result.total} time(s), ${result.hits.length} session(s), ${secondsText(result.median)}`,
|
|
321
323
|
);
|
|
322
324
|
|
|
323
325
|
lines.push(
|
|
324
326
|
"",
|
|
325
|
-
"
|
|
327
|
+
"Repeatable loops (count, interventions during them, median duration):",
|
|
326
328
|
);
|
|
327
329
|
const loops = LOOP_KINDS.map((kind) => ({
|
|
328
330
|
kind,
|
|
@@ -335,7 +337,7 @@ function render(output) {
|
|
|
335
337
|
}));
|
|
336
338
|
for (const { kind, result } of loops) {
|
|
337
339
|
if (result === undefined) {
|
|
338
|
-
lines.push(` ${kind}:
|
|
340
|
+
lines.push(` ${kind}: not measured`);
|
|
339
341
|
continue;
|
|
340
342
|
}
|
|
341
343
|
const count = result.measured.reduce(
|
|
@@ -343,98 +345,98 @@ function render(output) {
|
|
|
343
345
|
0,
|
|
344
346
|
);
|
|
345
347
|
lines.push(
|
|
346
|
-
` ${kind}: ${result.total}
|
|
348
|
+
` ${kind}: ${result.total} time(s), ${count} intervention(s), ${secondsText(result.median)}`,
|
|
347
349
|
);
|
|
348
350
|
}
|
|
349
351
|
|
|
350
|
-
lines.push("", "MCP
|
|
352
|
+
lines.push("", "MCP servers (sessions where configured, calls, failures):");
|
|
351
353
|
const servers = mcpServers(sessions);
|
|
352
|
-
if (servers.length === 0) lines.push("
|
|
354
|
+
if (servers.length === 0) lines.push(" (no MCP servers in the records)");
|
|
353
355
|
for (const s of servers)
|
|
354
356
|
lines.push(
|
|
355
|
-
` ${s.server}:
|
|
357
|
+
` ${s.server}: configured in ${s.configured.length} session(s), ${s.calls === undefined ? "calls: not measured" : `${s.calls} call(s)`}, ${s.failures === undefined ? "failures: not measured" : `${s.failures} failure(s)`}`,
|
|
356
358
|
);
|
|
357
359
|
|
|
358
360
|
const withUsage = sessions.filter((s) => s.usage);
|
|
359
|
-
lines.push("", "
|
|
361
|
+
lines.push("", "Usage (matching sessions):");
|
|
360
362
|
for (const kind of SESSION_USAGE_KINDS)
|
|
361
363
|
lines.push(
|
|
362
364
|
withUsage.length === 0
|
|
363
|
-
? ` ${kind}:
|
|
364
|
-
: ` ${kind}: ${withUsage.filter((s) => s.usage.kinds.includes(kind)).length} session`,
|
|
365
|
+
? ` ${kind}: not measured`
|
|
366
|
+
: ` ${kind}: ${withUsage.filter((s) => s.usage.kinds.includes(kind)).length} session(s)`,
|
|
365
367
|
);
|
|
366
368
|
lines.push(
|
|
367
|
-
` model
|
|
369
|
+
` model (share of output_tokens): ${sharesText(modelShares(withUsage))}`,
|
|
368
370
|
);
|
|
369
371
|
|
|
370
372
|
lines.push("", ...renderFollowups(output.followups));
|
|
371
373
|
if (!output.sufficient) {
|
|
372
374
|
lines.push(
|
|
373
|
-
|
|
375
|
+
`Not enough data, so no suggestions are made: ${output.session_count} session(s) analyzed. Suggestions need at least ${MINIMUM_SESSIONS} (${MINIMUM_SESSIONS - output.session_count} more needed)`,
|
|
374
376
|
);
|
|
375
377
|
return lines;
|
|
376
378
|
}
|
|
377
379
|
|
|
378
|
-
lines.push("
|
|
380
|
+
lines.push("Targets for suggestions:");
|
|
379
381
|
for (const { kind, result } of interventions) {
|
|
380
382
|
const category = `intervention.kind=${kind}`;
|
|
381
383
|
if (result === undefined) {
|
|
382
|
-
lines.push(`- ${category}:
|
|
384
|
+
lines.push(`- ${category}: not measured, so no suggestion is made`);
|
|
383
385
|
continue;
|
|
384
386
|
}
|
|
385
387
|
const lacking = missing([
|
|
386
|
-
[INTERVENTION_MIN_SESSIONS - result.hits.length, " session"],
|
|
387
|
-
[INTERVENTION_MIN_COUNT - result.total, "
|
|
388
|
+
[INTERVENTION_MIN_SESSIONS - result.hits.length, " session(s)"],
|
|
389
|
+
[INTERVENTION_MIN_COUNT - result.total, " time(s)"],
|
|
388
390
|
]);
|
|
389
391
|
if (lacking) {
|
|
390
392
|
lines.push(
|
|
391
|
-
`- ${category}:
|
|
393
|
+
`- ${category}: Not enough data: needs at least ${INTERVENTION_MIN_SESSIONS} sessions and at least ${INTERVENTION_MIN_COUNT} times in total (${lacking} more needed)`,
|
|
392
394
|
);
|
|
393
395
|
continue;
|
|
394
396
|
}
|
|
395
397
|
lines.push(
|
|
396
|
-
`- ${category}:
|
|
397
|
-
...evidenceLines(result.hits, "
|
|
398
|
+
`- ${category}: a suggestion can be made (${result.hits.length} session(s), ${result.total} time(s))`,
|
|
399
|
+
...evidenceLines(result.hits, " time(s)"),
|
|
398
400
|
);
|
|
399
401
|
}
|
|
400
402
|
for (const { kind, result } of loops) {
|
|
401
403
|
const category = `loop.kind=${kind}`;
|
|
402
404
|
if (result === undefined) {
|
|
403
|
-
lines.push(`- ${category}:
|
|
405
|
+
lines.push(`- ${category}: not measured, so no suggestion is made`);
|
|
404
406
|
continue;
|
|
405
407
|
}
|
|
406
408
|
if (result.total < LOOP_MIN_OCCURRENCES) {
|
|
407
409
|
lines.push(
|
|
408
|
-
`- ${category}:
|
|
410
|
+
`- ${category}: Not enough data: needs at least ${LOOP_MIN_OCCURRENCES} times in total (${LOOP_MIN_OCCURRENCES - result.total} more needed)`,
|
|
409
411
|
);
|
|
410
412
|
continue;
|
|
411
413
|
}
|
|
412
414
|
lines.push(
|
|
413
|
-
`- ${category}:
|
|
414
|
-
...evidenceLines(result.hits, "
|
|
415
|
+
`- ${category}: a suggestion can be made (${result.total} time(s) in total)`,
|
|
416
|
+
...evidenceLines(result.hits, " time(s)"),
|
|
415
417
|
);
|
|
416
418
|
}
|
|
417
419
|
// unlistedは構成に無いserverをまとめたものであり、設定されていたsessionを持たないため対象にしない。
|
|
418
420
|
for (const s of servers.filter((server) => server.server !== "unlisted")) {
|
|
419
421
|
const category = `mcp_server=${s.server}`;
|
|
420
422
|
if (s.calls === undefined && s.failures === undefined) {
|
|
421
|
-
lines.push(`- ${category}:
|
|
423
|
+
lines.push(`- ${category}: not measured, so no suggestion is made`);
|
|
422
424
|
continue;
|
|
423
425
|
}
|
|
424
426
|
if (s.configured.length < MCP_MIN_CONFIGURED_SESSIONS) {
|
|
425
427
|
lines.push(
|
|
426
|
-
`- ${category}:
|
|
428
|
+
`- ${category}: Not enough data: needs at least ${MCP_MIN_CONFIGURED_SESSIONS} sessions where it was configured (${MCP_MIN_CONFIGURED_SESSIONS - s.configured.length} more needed)`,
|
|
427
429
|
);
|
|
428
430
|
continue;
|
|
429
431
|
}
|
|
430
432
|
lines.push(
|
|
431
|
-
`- ${category}:
|
|
433
|
+
`- ${category}: a suggestion can be made (configured in ${s.configured.length} session(s))`,
|
|
432
434
|
...evidenceLines(
|
|
433
435
|
s.configured.map(({ session, element }) => ({
|
|
434
436
|
session,
|
|
435
437
|
count: element.calls ?? 0,
|
|
436
438
|
})),
|
|
437
|
-
"
|
|
439
|
+
" call(s)",
|
|
438
440
|
),
|
|
439
441
|
);
|
|
440
442
|
}
|