@cydm/pie 2.6.5 → 2.6.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,9 @@
1
1
  import { createRequire as __createRequire } from "node:module"; const require = __createRequire(import.meta.url);
2
2
  import {
3
3
  createAskUserCapability
4
- } from "../../../chunks/chunk-QKSK5K5N.js";
5
- import "../../../chunks/chunk-AETQBDEG.js";
6
- import "../../../chunks/chunk-5OXH6NUH.js";
4
+ } from "../../../chunks/chunk-H6BAM2N4.js";
5
+ import "../../../chunks/chunk-KEARZX6F.js";
6
+ import "../../../chunks/chunk-G5B6D35K.js";
7
7
  import "../../../chunks/chunk-XXGYUYXI.js";
8
8
  import "../../../chunks/chunk-LZQXQOPJ.js";
9
9
 
@@ -24464,7 +24464,6 @@ function redactSecrets(text, secrets) {
24464
24464
  function encodedSecretVariants(secrets) {
24465
24465
  const variants = [...secrets];
24466
24466
  for (const secret of secrets) {
24467
- if (secret.length < 4) continue;
24468
24467
  try {
24469
24468
  const base642 = Buffer.from(secret, "utf-8").toString("base64");
24470
24469
  if (base642.length >= 4) {
@@ -24472,13 +24471,39 @@ function encodedSecretVariants(secrets) {
24472
24471
  const base64url2 = Buffer.from(secret, "utf-8").toString("base64url");
24473
24472
  if (base64url2 !== base642 && base64url2.length >= 4) variants.push(base64url2);
24474
24473
  }
24474
+ } catch {
24475
+ }
24476
+ try {
24475
24477
  const urlEncoded = encodeURIComponent(secret);
24476
- if (urlEncoded !== secret && urlEncoded.length >= 4) variants.push(urlEncoded);
24478
+ if (urlEncoded !== secret && urlEncoded.length >= 4) {
24479
+ variants.push(urlEncoded);
24480
+ const urlLowerHex = urlEncoded.replace(/%[0-9A-F]{2}/g, (m) => `%${m.slice(1).toLowerCase()}`);
24481
+ if (urlLowerHex !== urlEncoded) variants.push(urlLowerHex);
24482
+ }
24477
24483
  } catch {
24478
24484
  }
24479
24485
  }
24480
24486
  return variants;
24481
24487
  }
24488
+ function unicodeEscapeAll(text) {
24489
+ let out = "";
24490
+ for (let index = 0; index < text.length; index++) {
24491
+ out += `\\u${text.charCodeAt(index).toString(16).padStart(4, "0")}`;
24492
+ }
24493
+ return out;
24494
+ }
24495
+ function unicodeEscapeVariants(secrets) {
24496
+ const variants = [];
24497
+ for (const secret of secrets) {
24498
+ const escaped = unicodeEscapeAll(secret);
24499
+ if (escaped !== secret && escaped.length >= 4) {
24500
+ variants.push(escaped);
24501
+ const escapedUpper = escaped.replace(/[0-9a-f]{4}/g, (m) => m.toUpperCase());
24502
+ if (escapedUpper !== escaped) variants.push(escapedUpper);
24503
+ }
24504
+ }
24505
+ return variants;
24506
+ }
24482
24507
  function hexSecretVariants(secrets) {
24483
24508
  const variants = [];
24484
24509
  for (const secret of secrets) {
@@ -24551,15 +24576,45 @@ function redactTrailingSecretPrefix(text, secrets) {
24551
24576
  }
24552
24577
  function redactAllSecrets(text, secrets) {
24553
24578
  const encoded = encodedSecretVariants(secrets);
24554
- const wholeVariants = [...encoded, ...hexSecretVariants(secrets)];
24579
+ const wholeVariants = [...encoded, ...hexSecretVariants(secrets), ...unicodeEscapeVariants(secrets)];
24555
24580
  return redactTrailingSecretPrefix(
24556
24581
  redactLeadingSecretSuffix(
24557
- redactSecretFragments(redactSecrets(text, wholeVariants), encoded),
24558
- secrets
24582
+ redactCaseInsensitiveEncodedSecrets(
24583
+ redactSecretFragments(redactSecrets(text, wholeVariants), encoded),
24584
+ secrets
24585
+ ),
24586
+ encoded
24559
24587
  ),
24560
- secrets
24588
+ encoded
24561
24589
  );
24562
24590
  }
24591
+ function redactCaseInsensitiveEncodedSecrets(text, secrets) {
24592
+ const escapeRegExpLiteral = (literal2) => literal2.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
24593
+ let sanitized = text;
24594
+ for (const secret of secrets) {
24595
+ const patterns = [];
24596
+ try {
24597
+ const urlEncoded = encodeURIComponent(secret);
24598
+ if (urlEncoded !== secret && urlEncoded.length >= 4 && /%[0-9A-F]{2}/i.test(urlEncoded)) {
24599
+ patterns.push(urlEncoded);
24600
+ }
24601
+ } catch {
24602
+ }
24603
+ try {
24604
+ if (secret.length >= 6) {
24605
+ const hex = Buffer.from(secret, "utf-8").toString("hex");
24606
+ if (hex.length >= 12) patterns.push(hex);
24607
+ }
24608
+ } catch {
24609
+ }
24610
+ const escaped = unicodeEscapeAll(secret);
24611
+ if (escaped !== secret && escaped.length >= 4) patterns.push(escaped);
24612
+ for (const pattern of patterns) {
24613
+ sanitized = sanitized.replace(new RegExp(escapeRegExpLiteral(pattern), "gi"), "[redacted]");
24614
+ }
24615
+ }
24616
+ return sanitized;
24617
+ }
24563
24618
  function redactStderrTail(tail, secrets) {
24564
24619
  return redactAllSecrets(tail, secrets);
24565
24620
  }
@@ -7,9 +7,9 @@ import {
7
7
  isPlanModeSafeCommand,
8
8
  markCompletedPlanSteps,
9
9
  restoreExecutionState
10
- } from "../../../chunks/chunk-QKSK5K5N.js";
11
- import "../../../chunks/chunk-AETQBDEG.js";
12
- import "../../../chunks/chunk-5OXH6NUH.js";
10
+ } from "../../../chunks/chunk-H6BAM2N4.js";
11
+ import "../../../chunks/chunk-KEARZX6F.js";
12
+ import "../../../chunks/chunk-G5B6D35K.js";
13
13
  import {
14
14
  Type
15
15
  } from "../../../chunks/chunk-XXGYUYXI.js";
@@ -1,13 +1,13 @@
1
1
  import { createRequire as __createRequire } from "node:module"; const require = __createRequire(import.meta.url);
2
2
  import {
3
3
  createCliHostCapabilities
4
- } from "../../../chunks/chunk-TFRQE7UQ.js";
4
+ } from "../../../chunks/chunk-OWSXALCG.js";
5
5
  import {
6
6
  createSharedFileSystemTools,
7
7
  createSubagentCapability
8
- } from "../../../chunks/chunk-QKSK5K5N.js";
9
- import "../../../chunks/chunk-AETQBDEG.js";
10
- import "../../../chunks/chunk-5OXH6NUH.js";
8
+ } from "../../../chunks/chunk-H6BAM2N4.js";
9
+ import "../../../chunks/chunk-KEARZX6F.js";
10
+ import "../../../chunks/chunk-G5B6D35K.js";
11
11
  import "../../../chunks/chunk-5353DEEC.js";
12
12
  import "../../../chunks/chunk-XXGYUYXI.js";
13
13
  import "../../../chunks/chunk-LZQXQOPJ.js";
@@ -8,9 +8,9 @@ import {
8
8
  executeManageTodoList,
9
9
  executionStateToTodos,
10
10
  restoreExecutionState
11
- } from "../../../chunks/chunk-QKSK5K5N.js";
12
- import "../../../chunks/chunk-AETQBDEG.js";
13
- import "../../../chunks/chunk-5OXH6NUH.js";
11
+ } from "../../../chunks/chunk-H6BAM2N4.js";
12
+ import "../../../chunks/chunk-KEARZX6F.js";
13
+ import "../../../chunks/chunk-G5B6D35K.js";
14
14
  import "../../../chunks/chunk-XXGYUYXI.js";
15
15
  import "../../../chunks/chunk-LZQXQOPJ.js";
16
16
 
@@ -4,10 +4,11 @@ import {
4
4
  BRIDGE_PORT_IN_USE_EXIT_CODE,
5
5
  PAGE_IDENTITY_CHANGED_ERROR,
6
6
  clearSelectedPageId,
7
+ readSelfStartTime,
7
8
  resolveSessionName,
8
9
  resolveSessionPidFile,
9
10
  resolveSessionPort
10
- } from "./chunk-UPATXALW.js";
11
+ } from "./chunk-U75F4DPF.js";
11
12
  import {
12
13
  __commonJS,
13
14
  __export,
@@ -19599,7 +19600,7 @@ async function isBridgeTargetReachable(client) {
19599
19600
  function writePidFile(port) {
19600
19601
  const pidFile = resolveSessionPidFile();
19601
19602
  mkdirSync(dirname(pidFile), { recursive: true });
19602
- writeFileSync(pidFile, JSON.stringify({ pid: process.pid, port }));
19603
+ writeFileSync(pidFile, JSON.stringify({ pid: process.pid, port, startTime: readSelfStartTime() }));
19603
19604
  }
19604
19605
  function removePidFile(pidFile = resolveSessionPidFile(), ownerPid = process.pid) {
19605
19606
  try {
@@ -21,6 +21,6 @@ function tryFastPath(argv, options) {
21
21
 
22
22
  // vendor/chrome-devtools-axi/bin/chrome-devtools-axi.ts
23
23
  if (!tryFastPath(process.argv.slice(2), { version: VERSION })) {
24
- const { main } = await import("./cli-5Z4COIKQ.js");
24
+ const { main } = await import("./cli-QAILYGLM.js");
25
25
  await main(process.argv.slice(2));
26
26
  }
@@ -200,10 +200,70 @@ function resolveBridgeScript(importMetaDir) {
200
200
  var BRIDGE_PORT_IN_USE_EXIT_CODE = 48;
201
201
  var PAGE_IDENTITY_CHANGED_ERROR = "The browser reconnected and every page id changed, so this call did not target the page you selected";
202
202
 
203
+ // vendor/chrome-devtools-axi/src/process-identity.ts
204
+ import { execFileSync } from "node:child_process";
205
+ function inspectProcessIdentity(pid) {
206
+ if (!Number.isInteger(pid) || pid <= 0) return {};
207
+ if (process.platform === "win32") {
208
+ try {
209
+ const output = execFileSync(
210
+ "powershell",
211
+ [
212
+ "-NoProfile",
213
+ "-NonInteractive",
214
+ "-Command",
215
+ `$p = Get-CimInstance Win32_Process -Filter "ProcessId=<pid>"; if ($p) { $p.CommandLine; $p.CreationDate.ToString('o') }`.replace("<pid>", String(pid))
216
+ ],
217
+ { encoding: "utf-8", timeout: 4e3 }
218
+ );
219
+ const lines = output.split(/\r?\n/).filter((line) => line.length > 0);
220
+ if (lines.length === 0) return {};
221
+ const startTime = lines.length >= 2 ? lines[lines.length - 1] : void 0;
222
+ return { command: lines.slice(0, lines.length - 1).join("\n"), startTime };
223
+ } catch {
224
+ return {};
225
+ }
226
+ }
227
+ try {
228
+ const psEnv = { ...process.env, LC_ALL: "C", LANG: "C" };
229
+ const output = execFileSync(
230
+ "ps",
231
+ ["-p", String(pid), "-o", "lstart=", "-o", "command="],
232
+ { encoding: "utf-8", timeout: 1e3, env: psEnv }
233
+ );
234
+ const firstNewline = output.indexOf("\n");
235
+ const firstLine = firstNewline === -1 ? output : output.slice(0, firstNewline);
236
+ const lstart = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{1,2}:\d{2}:\d{2}\s+\d{4}\s+/.exec(firstLine);
237
+ if (lstart) {
238
+ return { startTime: lstart[0].trimEnd(), command: output.slice(lstart[0].length) };
239
+ }
240
+ try {
241
+ const command = execFileSync("ps", ["-p", String(pid), "-o", "command="], {
242
+ encoding: "utf-8",
243
+ timeout: 1e3,
244
+ env: psEnv
245
+ });
246
+ return { command };
247
+ } catch {
248
+ return {};
249
+ }
250
+ } catch {
251
+ return {};
252
+ }
253
+ }
254
+ function readProcessStartTime(pid) {
255
+ return inspectProcessIdentity(pid).startTime;
256
+ }
257
+ function readSelfStartTime() {
258
+ return readProcessStartTime(process.pid);
259
+ }
260
+
203
261
  export {
204
262
  resolveBridgeScript,
205
263
  BRIDGE_PORT_IN_USE_EXIT_CODE,
206
264
  PAGE_IDENTITY_CHANGED_ERROR,
265
+ inspectProcessIdentity,
266
+ readSelfStartTime,
207
267
  resolveSessionName,
208
268
  resolveSessionPort,
209
269
  resolveSessionStateDir,
@@ -7,6 +7,7 @@ import {
7
7
  PAGE_IDENTITY_CHANGED_ERROR,
8
8
  clearSelectedPageId,
9
9
  getSelectedPageId,
10
+ inspectProcessIdentity,
10
11
  overlaySessionSelected,
11
12
  rememberToolRouting,
12
13
  resolveBridgeScript,
@@ -14,7 +15,7 @@ import {
14
15
  resolveSessionPidFile,
15
16
  resolveSessionPort,
16
17
  resolveSessionStateDir
17
- } from "./chunk-UPATXALW.js";
18
+ } from "./chunk-U75F4DPF.js";
18
19
  import "./chunk-OJ6N23UZ.js";
19
20
 
20
21
  // ../../node_modules/chrome-devtools-axi/node_modules/@toon-format/toon/dist/index.mjs
@@ -1973,33 +1974,41 @@ function commandLineLooksLikeBridge(command) {
1973
1974
  }
1974
1975
  return false;
1975
1976
  }
1976
- function isBridgeProcess(pid) {
1977
- if (process.platform === "win32") {
1978
- try {
1979
- const command = execFileSync(
1980
- "powershell",
1981
- [
1982
- "-NoProfile",
1983
- "-NonInteractive",
1984
- "-Command",
1985
- `(Get-CimInstance Win32_Process -Filter "ProcessId=${pid}").CommandLine`
1986
- ],
1987
- { encoding: "utf-8", timeout: 4e3 }
1988
- );
1989
- return commandLineLooksLikeBridge(command);
1990
- } catch {
1991
- return false;
1977
+ function isBridgeProcess(pid, expectedStartTime) {
1978
+ const identity = inspectProcessIdentity(pid);
1979
+ if (identity.command !== void 0) {
1980
+ if (!commandLineLooksLikeBridge(identity.command)) return false;
1981
+ } else {
1982
+ if (process.platform === "win32") {
1983
+ try {
1984
+ const command = execFileSync(
1985
+ "powershell",
1986
+ [
1987
+ "-NoProfile",
1988
+ "-NonInteractive",
1989
+ "-Command",
1990
+ `(Get-CimInstance Win32_Process -Filter "ProcessId=${pid}").CommandLine`
1991
+ ],
1992
+ { encoding: "utf-8", timeout: 4e3 }
1993
+ );
1994
+ if (!commandLineLooksLikeBridge(command)) return false;
1995
+ } catch {
1996
+ return false;
1997
+ }
1998
+ } else {
1999
+ try {
2000
+ const command = execFileSync("ps", ["-p", String(pid), "-o", "command="], {
2001
+ encoding: "utf-8",
2002
+ timeout: 1e3
2003
+ });
2004
+ if (!commandLineLooksLikeBridge(command)) return false;
2005
+ } catch {
2006
+ return false;
2007
+ }
1992
2008
  }
1993
2009
  }
1994
- try {
1995
- const command = execFileSync("ps", ["-p", String(pid), "-o", "command="], {
1996
- encoding: "utf-8",
1997
- timeout: 1e3
1998
- });
1999
- return commandLineLooksLikeBridge(command);
2000
- } catch {
2001
- return false;
2002
- }
2010
+ if (expectedStartTime === void 0) return true;
2011
+ return identity.startTime !== void 0 && identity.startTime === expectedStartTime;
2003
2012
  }
2004
2013
  async function probeBridgeIdentity(port, sessionName) {
2005
2014
  try {
@@ -2022,11 +2031,12 @@ async function judgeRecordedBridge(pidInfo, sessionName) {
2022
2031
  if (probe.identity === "own") {
2023
2032
  if (probe.reportedPid !== void 0) {
2024
2033
  if (probe.reportedPid === pidInfo.pid) {
2025
- const verified = isBridgeProcess(pidInfo.pid);
2034
+ const verified = isBridgeProcess(pidInfo.pid, pidInfo.startTime);
2026
2035
  return {
2027
2036
  terminate: verified,
2028
2037
  killProcessGroup: verified,
2029
- targetPid: pidInfo.pid
2038
+ targetPid: pidInfo.pid,
2039
+ expectedStartTime: pidInfo.startTime
2030
2040
  };
2031
2041
  }
2032
2042
  const reportedIsBridge = isBridgeProcess(probe.reportedPid);
@@ -2036,14 +2046,14 @@ async function judgeRecordedBridge(pidInfo, sessionName) {
2036
2046
  targetPid: probe.reportedPid
2037
2047
  };
2038
2048
  }
2039
- const bridge2 = isBridgeProcess(pidInfo.pid);
2040
- return { terminate: bridge2, killProcessGroup: bridge2, targetPid: pidInfo.pid };
2049
+ const bridge2 = isBridgeProcess(pidInfo.pid, pidInfo.startTime);
2050
+ return { terminate: bridge2, killProcessGroup: bridge2, targetPid: pidInfo.pid, expectedStartTime: pidInfo.startTime };
2041
2051
  }
2042
2052
  if (probe.identity === "other") {
2043
2053
  return { terminate: false, killProcessGroup: false };
2044
2054
  }
2045
- const bridge = isBridgeProcess(pidInfo.pid);
2046
- return { terminate: bridge, killProcessGroup: bridge, targetPid: pidInfo.pid };
2055
+ const bridge = isBridgeProcess(pidInfo.pid, pidInfo.startTime);
2056
+ return { terminate: bridge, killProcessGroup: bridge, targetPid: pidInfo.pid, expectedStartTime: pidInfo.startTime };
2047
2057
  }
2048
2058
  function removePidFile(pidFile) {
2049
2059
  try {
@@ -2065,9 +2075,13 @@ function taskkillTree(pid) {
2065
2075
  async function terminateBridgeProcess(pid, opts = {}) {
2066
2076
  if (!isProcessAlive(pid)) return true;
2067
2077
  const killProcessGroup = opts.killProcessGroup === true;
2078
+ const identityHolds = () => !opts.requireBridgeIdentity || isBridgeProcess(pid, opts.expectedStartTime);
2079
+ const refuseIfUnverified = async () => !await isProcessAlive(pid);
2068
2080
  if (process.platform === "win32") {
2081
+ if (!identityHolds()) return refuseIfUnverified();
2069
2082
  return taskkillTree(pid) && !await isProcessAlive(pid);
2070
2083
  }
2084
+ if (!identityHolds()) return refuseIfUnverified();
2071
2085
  try {
2072
2086
  process.kill(pid, "SIGTERM");
2073
2087
  } catch (error) {
@@ -2082,10 +2096,12 @@ async function terminateBridgeProcess(pid, opts = {}) {
2082
2096
  }
2083
2097
  return true;
2084
2098
  }
2099
+ if (!identityHolds()) return refuseIfUnverified();
2085
2100
  if (killProcessGroup) {
2086
2101
  try {
2087
2102
  process.kill(-pid, "SIGKILL");
2088
2103
  } catch {
2104
+ if (!identityHolds()) return refuseIfUnverified();
2089
2105
  try {
2090
2106
  process.kill(pid, "SIGKILL");
2091
2107
  } catch {
@@ -2188,7 +2204,9 @@ async function ensureBridge(spawnBridge = spawnBridgeProcess, notice) {
2188
2204
  const verdict = await judgeRecordedBridge(pidInfo, sessionName);
2189
2205
  if (verdict.terminate) {
2190
2206
  await terminateBridgeProcess(verdict.targetPid ?? pidInfo.pid, {
2191
- killProcessGroup: verdict.killProcessGroup
2207
+ killProcessGroup: verdict.killProcessGroup,
2208
+ requireBridgeIdentity: true,
2209
+ expectedStartTime: verdict.expectedStartTime
2192
2210
  });
2193
2211
  } else {
2194
2212
  removePidFile(pidFile);
@@ -2466,7 +2484,9 @@ async function stopBridge() {
2466
2484
  const verdict = await judgeRecordedBridge(pidInfo, resolveSessionName());
2467
2485
  if (!verdict.terminate) return "noop";
2468
2486
  const terminated = await terminateBridgeProcess(verdict.targetPid ?? pidInfo.pid, {
2469
- killProcessGroup: verdict.killProcessGroup
2487
+ killProcessGroup: verdict.killProcessGroup,
2488
+ requireBridgeIdentity: true,
2489
+ expectedStartTime: verdict.expectedStartTime
2470
2490
  });
2471
2491
  clearSelectedPageId();
2472
2492
  return terminated ? "stopped" : "failed";
@@ -166,9 +166,9 @@ This section is one continuous sequence — don't stop partway through. Do NOT u
166
166
 
167
167
  Put results in `<skill-name>-workspace/` as a sibling to the skill directory. Within the workspace, organize results by iteration (`iteration-1/`, `iteration-2/`, etc.) and within that, each test case gets a directory (`eval-0/`, `eval-1/`, etc.). Don't create all of this upfront — just create directories as you go.
168
168
 
169
- ### Step 1: Spawn all runs (with-skill AND baseline) in the same turn
169
+ ### Step 1: Schedule paired runs (with-skill AND baseline)
170
170
 
171
- For each test case, spawn two subagents in the same turn — one with the skill, one without. This is important: don't spawn the with-skill runs first and then come back for baselines later. Launch everything at once so it all finishes around the same time.
171
+ For each test case, prepare paired runs — one with the skill, one baseline. Read-only runs may be batched; Pie uses bounded concurrency (default 4), not unlimited simultaneous execution. Default `mode: "read"` tasks cannot write source files: ask them to return outputs and grading evidence, then have the parent persist artifacts, metadata, timing and grades. When evaluation genuinely requires file edits, use explicit `mode: "edit"` with exactly one task per call, wait for it to finish before any parent edit or next edit task, and review its changes. The workspace is shared, not isolated. Use parent-owned persistence or serialized single edit tasks for all evaluation writes; a save-path instruction does not grant a read task write permission.
172
172
 
173
173
  **With-skill run:**
174
174
 
@@ -177,7 +177,7 @@ Execute this task:
177
177
  - Skill path: <path-to-skill>
178
178
  - Task: <eval prompt>
179
179
  - Input files: <eval files if any, or "none">
180
- - Save outputs to: <workspace>/iteration-<N>/eval-<ID>/with_skill/outputs/
180
+ - Return outputs for parent persistence to: <workspace>/iteration-<N>/eval-<ID>/with_skill/outputs/ (or use a serialized single edit task)
181
181
  - Outputs to save: <what the user cares about — e.g., "the .docx file", "the final CSV">
182
182
  ```
183
183
 
@@ -206,29 +206,28 @@ Update the `eval_metadata.json` files and `evals/evals.json` with the assertions
206
206
 
207
207
  ### Step 3: As runs complete, capture timing data
208
208
 
209
- When each subagent task completes, you receive a notification containing `total_tokens` and `duration_ms`. Save this data immediately to `timing.json` in the run directory:
209
+ Pie exposes per-task timing as `task.details.durationMs` when available and aggregate wall time as result `details.totalDuration` (milliseconds). It does not supply `total_tokens` or `duration_ms` notification fields. Have the parent persist available per-run timing to `timing.json`, converting units for the viewer:
210
210
 
211
211
  ```json
212
212
  {
213
- "total_tokens": 84852,
214
213
  "duration_ms": 23332,
215
214
  "total_duration_seconds": 23.3
216
215
  }
217
216
  ```
218
217
 
219
- This is the only opportunity to capture this data — it comes through the task notification and isn't persisted elsewhere. Process each notification as it arrives rather than trying to batch them.
218
+ Here `duration_ms` is an artifact field derived from `task.details.durationMs`, not a native notification field. Keep aggregate `totalDuration` separate from per-run duration; concurrency means it is not the sum of task times. If timing is absent, mark it unavailable rather than inventing a value. Do not report token counts unless separate instrumentation actually provides them; omit `total_tokens` otherwise. Viewer/aggregator default zeros are not measured token usage.
220
219
 
221
220
  ### Step 4: Grade, aggregate, and launch the viewer
222
221
 
223
222
  Once all runs are done:
224
223
 
225
- 1. **Grade each run** — spawn a grader subagent (or grade inline) that reads `agents/grader.md` and evaluates each assertion against the outputs. Save results to `grading.json` in each run directory. The grading.json expectations array must use the fields `text`, `passed`, and `evidence` (not `name`/`met`/`details` or other variants) — the viewer depends on these exact field names. For assertions that can be checked programmatically, write and run a script rather than eyeballing it — scripts are faster, more reliable, and can be reused across iterations.
224
+ 1. **Grade each run** — spawn a grader subagent (or grade inline) that reads `agents/grader.md` and evaluates each assertion against the outputs. Have the parent save returned grades to `grading.json` in each run directory (or use a serialized single edit task); read graders must not write files. The grading.json expectations array must use the fields `text`, `passed`, and `evidence` (not `name`/`met`/`details` or other variants) — the viewer depends on these exact field names. For assertions that can be checked programmatically, write and run a script rather than eyeballing it — scripts are faster, more reliable, and can be reused across iterations.
226
225
 
227
226
  2. **Aggregate into benchmark** — run the aggregation script from the skill-creator directory:
228
227
  ```bash
229
228
  node <skill-creator-path>/scripts/aggregate_benchmark.mjs <workspace>/iteration-N --skill-name <name>
230
229
  ```
231
- This produces `benchmark.json` and `benchmark.md` with pass_rate, time, and tokens for each configuration, with mean ± stddev and the delta. If generating benchmark.json manually, see `references/schemas.md` for the exact schema the viewer expects.
230
+ This produces `benchmark.json` and `benchmark.md` with pass_rate and available timing for each configuration, with mean ± stddev and the delta. Token fields are meaningful only with separate instrumentation; label them unavailable otherwise. If generating benchmark.json manually, see `references/schemas.md` for the exact schema the viewer expects.
232
231
  Put each with_skill version before its baseline counterpart.
233
232
 
234
233
  3. **Do an analyst pass** — read the benchmark data and surface patterns the aggregate stats might hide. See `agents/analyzer.md` (the "Analyzing Benchmark Results" section) for what to look for — things like assertions that always pass regardless of skill (non-discriminating), high-variance evals (possibly flaky), and time/token tradeoffs.