@ishlabs/cli 0.8.3 → 0.8.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,12 +11,108 @@ import { deterministicAlias, getAliasMap, ALIAS_PREFIX } from "./alias-store.js"
11
11
  // --- Lean JSON: strip noise for agent-friendly output ---
12
12
  let _verbose = false;
13
13
  let _fields;
14
+ let _getField;
14
15
  /** Set by withClient() based on global flags. */
15
16
  export function setVerbose(v) { _verbose = v; }
16
17
  export function setFields(fields) { _fields = fields; }
18
+ /**
19
+ * Pattern Ω capture mode: when set, jsonOutput() returns the bare value at
20
+ * the dotted path instead of the full JSON. Cleared between command runs by
21
+ * each invocation of `applyGlobals()`.
22
+ */
23
+ export function setGetField(field) { _getField = field; }
24
+ /**
25
+ * Walk a dotted path through a JSON value. Returns the resolved value or
26
+ * `MISSING` if any step is undefined. Numeric segments index into arrays;
27
+ * non-numeric segments key into objects. When a segment is non-numeric and
28
+ * the current value is an array, the segment is mapped over the array
29
+ * (e.g. `items.alias` on `{items: [...]}` after `items` is unwrapped to the
30
+ * array yields the per-element `alias` values).
31
+ */
32
+ const MISSING = Symbol("missing");
33
+ function walkPath(data, segments) {
34
+ let cur = data;
35
+ for (const seg of segments) {
36
+ if (cur === null || cur === undefined)
37
+ return MISSING;
38
+ if (Array.isArray(cur)) {
39
+ // `seg` could be a numeric index, or a key to apply to each element.
40
+ const asIndex = /^\d+$/.test(seg) ? parseInt(seg, 10) : null;
41
+ if (asIndex !== null) {
42
+ if (asIndex < 0 || asIndex >= cur.length)
43
+ return MISSING;
44
+ cur = cur[asIndex];
45
+ continue;
46
+ }
47
+ // Map across array: pick the key on each element. Skip elements that
48
+ // lack the key so `--get items.alias` on a list with one bad row
49
+ // still returns the rest.
50
+ const mapped = [];
51
+ for (const el of cur) {
52
+ if (el !== null && typeof el === "object" && seg in el) {
53
+ mapped.push(el[seg]);
54
+ }
55
+ }
56
+ if (mapped.length === 0)
57
+ return MISSING;
58
+ cur = mapped;
59
+ continue;
60
+ }
61
+ if (typeof cur !== "object")
62
+ return MISSING;
63
+ const obj = cur;
64
+ if (!(seg in obj))
65
+ return MISSING;
66
+ cur = obj[seg];
67
+ }
68
+ return cur;
69
+ }
70
+ /**
71
+ * Resolve `_getField` against `data`. Auto-descends into a top-level
72
+ * `items: [...]` wrapper when the requested path doesn't start with `items`
73
+ * and the path resolves on items but not at top level — i.e.
74
+ * `--get alias` on a list response acts like `--get items.alias`.
75
+ */
76
+ function extractGetField(data, path) {
77
+ const segments = path.split(".").map((s) => s.trim()).filter(Boolean);
78
+ if (segments.length === 0)
79
+ return MISSING;
80
+ const direct = walkPath(data, segments);
81
+ if (direct !== MISSING)
82
+ return direct;
83
+ // Auto-descend through {items: [...]} wrapper for paginated list responses.
84
+ if (segments[0] !== "items"
85
+ && data !== null
86
+ && typeof data === "object"
87
+ && !Array.isArray(data)
88
+ && Array.isArray(data.items)) {
89
+ const viaItems = walkPath(data, ["items", ...segments]);
90
+ if (viaItems !== MISSING)
91
+ return viaItems;
92
+ }
93
+ return MISSING;
94
+ }
95
+ /**
96
+ * Render an extracted value as a bare string for stdout. Rules:
97
+ * - string/number/boolean: printed as-is (no JSON quotes).
98
+ * - null: empty string.
99
+ * - arrays: one element per line, each element rendered by the same rules
100
+ * (objects within the array are compact JSON).
101
+ * - objects: compact JSON on a single line.
102
+ */
103
+ function renderBare(value) {
104
+ if (value === null || value === undefined)
105
+ return "";
106
+ if (Array.isArray(value)) {
107
+ return value.map((v) => renderBare(v)).join("\n");
108
+ }
109
+ if (typeof value === "object")
110
+ return JSON.stringify(value);
111
+ return String(value);
112
+ }
17
113
  const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
18
114
  const TIMESTAMP_KEYS = new Set(["created_at", "updated_at"]);
19
- const PAGINATION_KEYS = new Set(["items", "total", "limit", "offset"]);
115
+ const PAGINATION_KEYS = new Set(["items", "total", "returned", "limit", "offset", "has_more"]);
20
116
  /**
21
117
  * Strip UUID-valued fields, null/undefined values, and timestamps.
22
118
  * Preserves alias, name, label, status, and other meaningful fields.
@@ -69,7 +165,31 @@ function leanJson(data, keepIds = false) {
69
165
  return Object.keys(result).length > 0 ? result : undefined;
70
166
  }
71
167
  /**
72
- * Detect a paginated list wrapper: `{items: [...], total?, limit?, offset?}`.
168
+ * Standard list envelope: `{items, total, returned, limit, offset, has_more}`.
169
+ * If the backend already returns a wrapper, `total/limit/offset` are passed
170
+ * through; otherwise they're synthesized from the items array. `returned` and
171
+ * `has_more` are always CLI-computed so agents can detect truncation without
172
+ * counting items themselves.
173
+ *
174
+ * The envelope itself bypasses leanJson (`preProjected: true` at the call
175
+ * site) so the wrapper keys are stable even on empty lists — leanJson would
176
+ * otherwise drop `items: []`. Per-item lean-stripping is applied here so
177
+ * agents still get the lean shape inside the envelope, unless the caller has
178
+ * already projected items to a known shape (`preProjectedItems: true`).
179
+ */
180
+ function wrapList(items, existing, opts = {}) {
181
+ const returned = items.length;
182
+ const total = typeof existing?.total === "number" ? existing.total : returned;
183
+ const limit = typeof existing?.limit === "number" ? existing.limit : returned;
184
+ const offset = typeof existing?.offset === "number" ? existing.offset : 0;
185
+ const has_more = total > offset + returned;
186
+ const leanItems = _verbose || opts.preProjectedItems
187
+ ? items
188
+ : leanJson(items) ?? [];
189
+ return { items: leanItems, total, returned, limit, offset, has_more };
190
+ }
191
+ /**
192
+ * Detect a paginated list wrapper: `{items, total?, returned?, limit?, offset?, has_more?}`.
73
193
  * Used so `--fields` filters per-item shape without dropping pagination metadata.
74
194
  */
75
195
  function isListWrapper(data) {
@@ -127,6 +247,19 @@ function jsonOutput(data, options = {}) {
127
247
  if (_fields && _fields.length > 0) {
128
248
  out = pickFields(out, _fields);
129
249
  }
250
+ // Pattern Ω capture mode: --get <field> returns bare values instead of
251
+ // structured JSON. We extract from the post-lean / post-fields data so the
252
+ // path the agent reasons about matches what they'd see on a normal --json
253
+ // call (e.g. UUIDs already replaced by aliases).
254
+ if (_getField) {
255
+ const extracted = extractGetField(out, _getField);
256
+ if (extracted === MISSING) {
257
+ const err = new Error(`--get: field "${_getField}" not found in response.`);
258
+ err.name = "ValidationError";
259
+ throw err;
260
+ }
261
+ return renderBare(extracted);
262
+ }
130
263
  return JSON.stringify(out, null, 2);
131
264
  }
132
265
  /**
@@ -142,9 +275,29 @@ function injectAliases(items, prefix, idField = "id") {
142
275
  }
143
276
  }
144
277
  // --- JSON mode ---
278
+ /**
279
+ * Catch jsonOutput's --get extraction failure (a ValidationError thrown when
280
+ * the requested field is missing) and route it through outputError + exit 2,
281
+ * so commands that don't go through withClient/runInline (e.g. `ish docs *`)
282
+ * still surface a clean usage error instead of an uncaught stack trace.
283
+ */
284
+ function safeJsonOutput(data, options = {}) {
285
+ try {
286
+ return jsonOutput(data, options);
287
+ }
288
+ catch (err) {
289
+ if (err instanceof Error && err.name === "ValidationError") {
290
+ outputError(err, true);
291
+ process.exit(2);
292
+ }
293
+ throw err;
294
+ }
295
+ }
145
296
  export function output(data, json, options = {}) {
146
297
  if (json) {
147
- console.log(jsonOutput(data, options));
298
+ const text = safeJsonOutput(data, options);
299
+ if (text !== undefined)
300
+ console.log(text);
148
301
  return;
149
302
  }
150
303
  if (data === null || data === undefined)
@@ -161,7 +314,9 @@ export function output(data, json, options = {}) {
161
314
  }
162
315
  export function outputList(rows, json) {
163
316
  if (json) {
164
- console.log(jsonOutput(rows));
317
+ const text = safeJsonOutput(rows);
318
+ if (text !== undefined)
319
+ console.log(text);
165
320
  return;
166
321
  }
167
322
  if (rows.length === 0) {
@@ -198,6 +353,21 @@ export class ValidationError extends Error {
198
353
  this.name = "ValidationError";
199
354
  }
200
355
  }
356
+ /**
357
+ * Pull a typed-error detail out of an ApiError body. Backend convention is
358
+ * HTTPException(detail={error_code, ...fields}), which FastAPI serialises as
359
+ * {"detail": {error_code, ...fields}}. Returns undefined when the body isn't
360
+ * shaped that way (e.g. plain string detail, or 422 validation arrays).
361
+ */
362
+ function structuredDetail(err) {
363
+ if (!err.body || typeof err.body !== "object")
364
+ return undefined;
365
+ const detail = err.body.detail;
366
+ if (detail && typeof detail === "object" && !Array.isArray(detail) && "error_code" in detail) {
367
+ return detail;
368
+ }
369
+ return undefined;
370
+ }
201
371
  /**
202
372
  * Map error codes to actionable suggestions so agents can self-recover.
203
373
  */
@@ -215,6 +385,14 @@ function suggestionsForError(err) {
215
385
  ];
216
386
  case "insufficient_credits":
217
387
  return ["Purchase more credits at https://app.ishlabs.io"];
388
+ case "usage_limit_reached": {
389
+ const d = structuredDetail(err);
390
+ const upgradeUrl = typeof d?.upgrade_url === "string" ? d.upgrade_url : "https://app.ishlabs.io/billing";
391
+ return [
392
+ `Upgrade your plan at ${upgradeUrl}`,
393
+ "Run `ish docs get-page reference/billing-limits` for the tier table",
394
+ ];
395
+ }
218
396
  case "validation_error":
219
397
  return ["Check the command help: add --help to see required options"];
220
398
  case "rate_limited":
@@ -262,12 +440,20 @@ export function outputError(err, json) {
262
440
  const mergedSuggestions = bodySuggestions
263
441
  ? Array.from(new Set([...bodySuggestions.map(String), ...suggestions]))
264
442
  : suggestions;
443
+ const limitDetail = err.error_code === "usage_limit_reached" ? structuredDetail(err) : undefined;
265
444
  if (json) {
266
445
  console.error(JSON.stringify({
267
446
  error: err.message,
268
447
  error_code: err.error_code,
269
448
  status: err.status,
270
449
  retryable: err.retryable,
450
+ ...(limitDetail && {
451
+ tier: limitDetail.tier,
452
+ limit: limitDetail.limit,
453
+ current: limitDetail.current,
454
+ max: limitDetail.max,
455
+ upgrade_url: limitDetail.upgrade_url,
456
+ }),
271
457
  ...(bodyErrors !== undefined && { errors: bodyErrors }),
272
458
  ...(mergedSuggestions.length > 0 && { suggestions: mergedSuggestions }),
273
459
  }));
@@ -380,22 +566,20 @@ function projectWorkspace(workspace, options = {}) {
380
566
  return result;
381
567
  }
382
568
  export function formatWorkspaceList(workspaces, json) {
383
- if (workspaces.length === 0) {
384
- if (json)
385
- console.log("[]");
386
- else
387
- console.log("No workspaces.");
388
- return;
389
- }
390
569
  injectAliases(workspaces, ALIAS_PREFIX.workspace);
391
570
  if (json) {
392
- if (_verbose) {
393
- console.log(jsonOutput(workspaces));
394
- }
395
- else {
396
- const projected = workspaces.map((w) => projectWorkspace(w));
397
- console.log(jsonOutput(projected, { preProjected: true }));
398
- }
571
+ // Synthesize pagination metadata: backend returns a flat array, so
572
+ // total/limit/offset reflect what we actually shipped.
573
+ const projected = _verbose
574
+ ? workspaces
575
+ : workspaces.map((w) => projectWorkspace(w));
576
+ // preProjectedItems: workspaces went through projectWorkspace which already
577
+ // chose the field set; skip the inner leanJson so created_at survives.
578
+ console.log(jsonOutput(wrapList(projected, undefined, { preProjectedItems: !_verbose }), { preProjected: true }));
579
+ return;
580
+ }
581
+ if (workspaces.length === 0) {
582
+ console.log("No workspaces.");
399
583
  return;
400
584
  }
401
585
  const aliasMap = getAliasMap(ALIAS_PREFIX.workspace);
@@ -441,16 +625,14 @@ export function formatSiteAccessStatus(summary, json) {
441
625
  }
442
626
  // --- Study formatting ---
443
627
  export function formatStudyList(studies, json) {
444
- if (studies.length === 0) {
445
- if (json)
446
- console.log("[]");
447
- else
448
- console.log("No studies.");
449
- return;
450
- }
451
628
  injectAliases(studies, ALIAS_PREFIX.study);
452
629
  if (json) {
453
- console.log(jsonOutput(studies));
630
+ // Backend returns a flat array; synthesize pagination metadata.
631
+ console.log(jsonOutput(wrapList(studies), { preProjected: true }));
632
+ return;
633
+ }
634
+ if (studies.length === 0) {
635
+ console.log("No studies.");
454
636
  return;
455
637
  }
456
638
  const aliasMap = getAliasMap(ALIAS_PREFIX.study);
@@ -463,9 +645,47 @@ export function formatStudyList(studies, json) {
463
645
  String(s.tester_count ?? "0"),
464
646
  ]));
465
647
  }
648
+ /**
649
+ * CLI-side sanity check for ALL-ISSUES Issue #2 / backend Pattern Bk2.
650
+ *
651
+ * Backend sometimes reports `status: "failed"` even when results are
652
+ * populated (testers completed, interactions present). Until the backend
653
+ * root-cause is fixed, the CLI surfaces the inconsistency rather than
654
+ * letting agents trust a misleading status field:
655
+ * - JSON: adds a `status_inferred` field (e.g. `completed_with_errors`).
656
+ * Original `status` field preserved so existing consumers can still
657
+ * branch on it.
658
+ * - Human / stderr: a one-line warning describing the mismatch.
659
+ *
660
+ * Returns null when status is consistent; no warning emitted.
661
+ */
662
+ function detectStudyStatusInconsistency(study) {
663
+ if (study.status !== "failed")
664
+ return null;
665
+ const allTesters = collectTesters(study);
666
+ const completedCount = allTesters.filter((t) => t.status === "completed" || t.status === "complete").length;
667
+ const totalInteractions = allTesters.reduce((sum, t) => sum + t.interactionCount, 0);
668
+ if (completedCount === 0 && totalInteractions === 0)
669
+ return null;
670
+ return {
671
+ inferred: "completed_with_errors",
672
+ reason: `${completedCount}/${allTesters.length} testers completed, ${totalInteractions} total interactions`,
673
+ };
674
+ }
675
+ function emitStatusInconsistencyWarning(inconsistency) {
676
+ process.stderr.write(`Warning: study reports status="failed" but ${inconsistency.reason}. ` +
677
+ `CLI inferring status_inferred="${inconsistency.inferred}". ` +
678
+ `Backend root-cause tracked as Issue #2 (Pattern Bk2).\n`);
679
+ }
466
680
  export function formatStudyDetail(study, json, options = {}) {
681
+ const inconsistency = detectStudyStatusInconsistency(study);
682
+ if (inconsistency)
683
+ emitStatusInconsistencyWarning(inconsistency);
467
684
  if (json) {
468
- console.log(jsonOutput(study, options));
685
+ const payload = inconsistency
686
+ ? { ...study, status_inferred: inconsistency.inferred }
687
+ : study;
688
+ console.log(jsonOutput(payload, options));
469
689
  return;
470
690
  }
471
691
  // Header
@@ -477,6 +697,12 @@ export function formatStudyDetail(study, json, options = {}) {
477
697
  modalityParts.push(String(study.content_type));
478
698
  modalityParts.push(String(study.status || "draft"), formatDate(study.created_at));
479
699
  console.log(modalityParts.join(" · "));
700
+ // Pattern C-followup: surface the modality rationale on `study generate`
701
+ // so agents (and humans) can spot misclassification without re-reading the
702
+ // brief. The field is only set on the immediate generate response.
703
+ if (study.modality_rationale) {
704
+ console.log(`\n Modality rationale: ${String(study.modality_rationale)}`);
705
+ }
480
706
  // Assignments
481
707
  const assignments = Array.isArray(study.assignments) ? study.assignments : [];
482
708
  if (assignments.length > 0) {
@@ -560,20 +786,41 @@ function buildStudyResultsEnvelope(study) {
560
786
  answers,
561
787
  };
562
788
  });
789
+ // CLI-side sanity check (Pattern E / Issue #2). Surface a status_inferred
790
+ // field when the backend reports failed-with-data; agents can branch on
791
+ // either the original status or status_inferred.
792
+ const inconsistency = detectStudyStatusInconsistency(study);
793
+ // Pattern B2 (cli half): per-tester rows expose status + error_message so
794
+ // agents can act on a failed run without re-fetching every tester.
795
+ const failedCount = allTesters.filter((t) => t.status.toLowerCase() === "failed").length;
796
+ const testerRows = allTesters.map((t) => ({
797
+ alias: t.id ? deterministicAlias(ALIAS_PREFIX.tester, t.id) : null,
798
+ name: t.name,
799
+ iteration: t.iterationLabel,
800
+ status: t.status,
801
+ interaction_count: t.interactionCount,
802
+ ...(t.errorMessage && { error_message: t.errorMessage }),
803
+ }));
563
804
  return {
564
805
  study: {
565
806
  alias: studyAlias,
566
807
  name: study.name || null,
567
808
  status: study.status || null,
809
+ ...(inconsistency && { status_inferred: inconsistency.inferred }),
568
810
  modality: study.modality || null,
569
811
  },
570
812
  tester_count: allTesters.length,
571
813
  completed_count: completedCount,
814
+ failed_count: failedCount,
572
815
  sentiment,
573
816
  interview_answers: interviewAnswers,
817
+ testers: testerRows,
574
818
  };
575
819
  }
576
820
  export function formatStudyResults(study, json) {
821
+ const inconsistency = detectStudyStatusInconsistency(study);
822
+ if (inconsistency)
823
+ emitStatusInconsistencyWarning(inconsistency);
577
824
  if (json) {
578
825
  // preProjected: bypass leanJson so the stable envelope keeps documented
579
826
  // empty defaults (sentiment: null, interview_answers[].answers: []) rather
@@ -628,6 +875,16 @@ export function formatStudyResults(study, json) {
628
875
  parts.length > 0 ? parts.join(", ") : "-",
629
876
  ];
630
877
  }));
878
+ // Pattern B2: list any failure reasons under the table so agents see why
879
+ // a run failed without drilling into `study tester <id>`.
880
+ const failedRows = allTesters.filter((t) => t.status.toLowerCase() === "failed" && t.errorMessage);
881
+ if (failedRows.length > 0) {
882
+ console.log("\nFailed testers:");
883
+ for (const t of failedRows) {
884
+ const alias = t.id ? deterministicAlias(ALIAS_PREFIX.tester, t.id) : t.id;
885
+ console.log(` ${alias} (${t.name}): ${truncate(t.errorMessage, 200)}`);
886
+ }
887
+ }
631
888
  console.log("\nRun `ish tester get <id> --json` for full interaction details.");
632
889
  }
633
890
  }
@@ -657,6 +914,7 @@ function collectTesters(study) {
657
914
  name: String(profile?.name || t.instance_name || "Unknown"),
658
915
  iterationLabel: iterLabel,
659
916
  status: String(t.status || "-"),
917
+ errorMessage: t.error_message ? String(t.error_message) : null,
660
918
  interactionCount: interactions.length,
661
919
  sentimentCounts,
662
920
  interviewAnswers: answers.map((a) => ({
@@ -683,16 +941,14 @@ function truncate(str, maxLen) {
683
941
  }
684
942
  // --- Iteration formatting ---
685
943
  export function formatIterationList(iterations, json) {
686
- if (iterations.length === 0) {
687
- if (json)
688
- console.log("[]");
689
- else
690
- console.log("No iterations.");
691
- return;
692
- }
693
944
  injectAliases(iterations, ALIAS_PREFIX.iteration);
694
945
  if (json) {
695
- console.log(jsonOutput(iterations));
946
+ // Backend returns a flat array; synthesize pagination metadata.
947
+ console.log(jsonOutput(wrapList(iterations), { preProjected: true }));
948
+ return;
949
+ }
950
+ if (iterations.length === 0) {
951
+ console.log("No iterations.");
696
952
  return;
697
953
  }
698
954
  const aliasMap = getAliasMap(ALIAS_PREFIX.iteration);
@@ -727,10 +983,15 @@ export function formatTesterDetail(tester, json) {
727
983
  }
728
984
  }
729
985
  const sentimentParts = Object.entries(sentimentCounts).map(([label, count]) => `${count} ${label.toLowerCase()}`);
986
+ const status = String(tester.status || "-");
987
+ const errorMessage = tester.error_message ? String(tester.error_message) : null;
730
988
  const display = {
731
989
  ID: tester.id || "-",
732
990
  Profile: profileName,
733
- Status: tester.status || "-",
991
+ Status: status,
992
+ ...(errorMessage && status.toLowerCase() === "failed" && {
993
+ Error: errorMessage,
994
+ }),
734
995
  Platform: tester.platform || "-",
735
996
  Language: tester.language || "-",
736
997
  Interactions: `${interactions.length} interactions`,
@@ -742,24 +1003,27 @@ export function formatTesterDetail(tester, json) {
742
1003
  }
743
1004
  // --- Tester Profile formatting ---
744
1005
  export function formatTesterProfileList(profiles, json, limit) {
745
- // The API may return { items: [...], total, limit, offset } or a flat array
1006
+ // The API may return { items: [...], total, limit, offset } or a flat array.
746
1007
  const wrapper = profiles;
1008
+ const wasWrapper = !Array.isArray(profiles)
1009
+ && profiles !== null
1010
+ && typeof profiles === "object"
1011
+ && (Array.isArray(wrapper?.items) || Array.isArray(wrapper?.profiles));
747
1012
  const fullList = Array.isArray(profiles) ? profiles
748
1013
  : Array.isArray(wrapper?.items) ? wrapper.items
749
1014
  : Array.isArray(wrapper?.profiles) ? wrapper.profiles
750
- : null;
751
- if (!Array.isArray(fullList) || fullList.length === 0) {
752
- if (json)
753
- console.log(JSON.stringify(profiles, null, 2));
754
- else
755
- console.log("No tester profiles.");
756
- return;
757
- }
1015
+ : [];
758
1016
  // Client-side limit (server may not enforce it)
759
1017
  const list = limit ? fullList.slice(0, limit) : fullList;
760
1018
  injectAliases(list, ALIAS_PREFIX.testerProfile);
761
1019
  if (json) {
762
- console.log(jsonOutput(profiles));
1020
+ // Pass through server-provided pagination when present; otherwise synthesize.
1021
+ const existing = wasWrapper ? wrapper : undefined;
1022
+ console.log(jsonOutput(wrapList(list, existing), { preProjected: true }));
1023
+ return;
1024
+ }
1025
+ if (list.length === 0) {
1026
+ console.log("No tester profiles.");
763
1027
  return;
764
1028
  }
765
1029
  printTable(["#", "NAME", "OCCUPATION", "COUNTRY", "GENDER", "AGE"], list.map((p) => [
@@ -855,16 +1119,14 @@ function variantLetter(index) {
855
1119
  return `V${index + 1}`;
856
1120
  }
857
1121
  export function formatAskList(asks, json) {
858
- if (asks.length === 0) {
859
- if (json)
860
- console.log("[]");
861
- else
862
- console.log("No asks.");
863
- return;
864
- }
865
1122
  injectAliases(asks, ALIAS_PREFIX.ask);
866
1123
  if (json) {
867
- console.log(jsonOutput(asks));
1124
+ // Backend returns a flat array; synthesize pagination metadata.
1125
+ console.log(jsonOutput(wrapList(asks), { preProjected: true }));
1126
+ return;
1127
+ }
1128
+ if (asks.length === 0) {
1129
+ console.log("No asks.");
868
1130
  return;
869
1131
  }
870
1132
  const aliasMap = getAliasMap(ALIAS_PREFIX.ask);
@@ -877,9 +1139,72 @@ export function formatAskList(asks, json) {
877
1139
  a.is_archived ? "yes" : "no",
878
1140
  ]));
879
1141
  }
1142
+ /**
1143
+ * Add denormalized counts to a round so agents don't have to count
1144
+ * `responses[]` via jq/python:
1145
+ * - responses_total: responses.length
1146
+ * - responses_complete: count where status === "completed"
1147
+ * - responses_errored: count where status === "errored" (only if > 0)
1148
+ */
1149
+ function denormalizeRoundCounts(round) {
1150
+ const responses = Array.isArray(round.responses) ? round.responses : null;
1151
+ if (!responses)
1152
+ return round;
1153
+ let complete = 0;
1154
+ let errored = 0;
1155
+ for (const r of responses) {
1156
+ const status = r.status;
1157
+ if (status === "completed")
1158
+ complete++;
1159
+ else if (status === "errored")
1160
+ errored++;
1161
+ }
1162
+ return {
1163
+ ...round,
1164
+ responses_total: responses.length,
1165
+ responses_complete: complete,
1166
+ ...(errored > 0 && { responses_errored: errored }),
1167
+ };
1168
+ }
1169
+ /**
1170
+ * Layer denormalized counts onto an ask detail so agents reading
1171
+ * `ask get`, `ask create --wait`, `ask run --wait`, etc. don't need to
1172
+ * count nested arrays:
1173
+ * - testers_count: ask.testers.length
1174
+ * - responses_total: sum across rounds (only when > 0)
1175
+ * - responses_complete: sum across rounds
1176
+ * - responses_errored: sum across rounds (only when > 0)
1177
+ * - rounds[i].responses_total / _complete / _errored
1178
+ */
1179
+ function denormalizeAskCounts(ask) {
1180
+ const enriched = { ...ask };
1181
+ const testers = Array.isArray(ask.testers) ? ask.testers : null;
1182
+ if (testers)
1183
+ enriched.testers_count = testers.length;
1184
+ const rounds = Array.isArray(ask.rounds) ? ask.rounds : null;
1185
+ if (rounds) {
1186
+ let total = 0;
1187
+ let complete = 0;
1188
+ let errored = 0;
1189
+ enriched.rounds = rounds.map((r) => {
1190
+ const decorated = denormalizeRoundCounts(r);
1191
+ total += decorated.responses_total ?? 0;
1192
+ complete += decorated.responses_complete ?? 0;
1193
+ errored += decorated.responses_errored ?? 0;
1194
+ return decorated;
1195
+ });
1196
+ if (total > 0) {
1197
+ enriched.responses_total = total;
1198
+ enriched.responses_complete = complete;
1199
+ if (errored > 0)
1200
+ enriched.responses_errored = errored;
1201
+ }
1202
+ }
1203
+ return enriched;
1204
+ }
880
1205
  export function formatAskDetail(ask, json) {
881
1206
  if (json) {
882
- console.log(jsonOutput(ask));
1207
+ console.log(jsonOutput(denormalizeAskCounts(ask)));
883
1208
  return;
884
1209
  }
885
1210
  console.log(`${ask.name || "Untitled"} (${ask.id || ""})`);
@@ -923,7 +1248,7 @@ export function formatAskDetail(ask, json) {
923
1248
  }
924
1249
  export function formatRoundDetail(round, json) {
925
1250
  if (json) {
926
- console.log(jsonOutput(round));
1251
+ console.log(jsonOutput(denormalizeRoundCounts(round)));
927
1252
  return;
928
1253
  }
929
1254
  const variants = Array.isArray(round.variants) ? round.variants : [];
@@ -999,13 +1324,166 @@ function computeVariantStats(round) {
999
1324
  }
1000
1325
  return stats;
1001
1326
  }
1327
+ // When tester_profile and tester_profile_snapshot share all overlapping fields
1328
+ // (the common case — snapshot only diverges if the profile was edited after
1329
+ // dispatch), drop the redundant content from the snapshot and keep only the
1330
+ // snapshot-specific metadata. Saves ~500-1000 bytes per tester in JSON output.
1331
+ function dedupeTesterSnapshot(tester) {
1332
+ const tp = tester.tester_profile;
1333
+ const tps = tester.tester_profile_snapshot;
1334
+ if (!tp || !tps)
1335
+ return tester;
1336
+ const shared = Object.keys(tps).filter((k) => k in tp);
1337
+ if (shared.length === 0)
1338
+ return tester;
1339
+ const isEmpty = (v) => {
1340
+ if (v === null || v === undefined)
1341
+ return true;
1342
+ if (Array.isArray(v))
1343
+ return v.length === 0;
1344
+ if (typeof v === "object")
1345
+ return Object.keys(v).length === 0;
1346
+ return false;
1347
+ };
1348
+ const allMatch = shared.every((k) => {
1349
+ const a = tp[k];
1350
+ const b = tps[k];
1351
+ if (isEmpty(a) && isEmpty(b))
1352
+ return true;
1353
+ return JSON.stringify(a) === JSON.stringify(b);
1354
+ });
1355
+ if (!allMatch)
1356
+ return tester;
1357
+ const snapshotOnly = {};
1358
+ for (const k of Object.keys(tps)) {
1359
+ if (!(k in tp))
1360
+ snapshotOnly[k] = tps[k];
1361
+ }
1362
+ return {
1363
+ ...tester,
1364
+ tester_profile_snapshot: { ...snapshotOnly, _matches_tester_profile: true },
1365
+ };
1366
+ }
1367
+ // Shape per-variant stats into a machine-readable aggregates object so agents
1368
+ // running A/B tests can read the verdict without parsing prose.
1369
+ function buildAggregates(round, stats) {
1370
+ if (stats.length === 0)
1371
+ return undefined;
1372
+ const wantsPick = !!round.wants_pick;
1373
+ const wantsRatings = !!round.wants_ratings;
1374
+ if (!wantsPick && !wantsRatings)
1375
+ return undefined;
1376
+ const out = {};
1377
+ if (wantsPick) {
1378
+ const picks = {};
1379
+ let topCount = -1;
1380
+ let topLetter = "";
1381
+ let tied = false;
1382
+ for (const s of stats) {
1383
+ picks[s.letter] = s.pickCount;
1384
+ if (s.pickCount > topCount) {
1385
+ topCount = s.pickCount;
1386
+ topLetter = s.letter;
1387
+ tied = false;
1388
+ }
1389
+ else if (s.pickCount === topCount && topCount > 0) {
1390
+ tied = true;
1391
+ }
1392
+ }
1393
+ out.picks = picks;
1394
+ if (topCount > 0) {
1395
+ out.winner = { letter: topLetter, count: topCount, tied };
1396
+ }
1397
+ }
1398
+ if (wantsRatings) {
1399
+ const ratings = {};
1400
+ for (const s of stats) {
1401
+ if (s.ratingCount > 0) {
1402
+ ratings[s.letter] = {
1403
+ mean: Number((s.ratingTotal / s.ratingCount).toFixed(3)),
1404
+ n: s.ratingCount,
1405
+ };
1406
+ }
1407
+ }
1408
+ if (Object.keys(ratings).length > 0)
1409
+ out.ratings = ratings;
1410
+ }
1411
+ return out;
1412
+ }
1413
+ function buildCrossRoundSummary(rounds) {
1414
+ if (rounds.length < 2)
1415
+ return undefined;
1416
+ const entries = [];
1417
+ for (const round of rounds) {
1418
+ const idx = typeof round.order_index === "number" ? round.order_index : 0;
1419
+ const stats = computeVariantStats(round);
1420
+ const aggregates = buildAggregates(round, stats);
1421
+ const entry = {
1422
+ round_number: idx + 1,
1423
+ prompt_preview: truncate(String(round.prompt || ""), 80),
1424
+ };
1425
+ if (aggregates?.picks)
1426
+ entry.picks = aggregates.picks;
1427
+ if (aggregates?.winner)
1428
+ entry.winner = aggregates.winner;
1429
+ if (aggregates?.ratings)
1430
+ entry.ratings = aggregates.ratings;
1431
+ entries.push(entry);
1432
+ }
1433
+ // Per-letter delta from first round → last round, when both have picks.
1434
+ const first = entries[0]?.picks;
1435
+ const last = entries[entries.length - 1]?.picks;
1436
+ let picks_delta;
1437
+ if (first && last) {
1438
+ picks_delta = {};
1439
+ const letters = new Set([
1440
+ ...Object.keys(first),
1441
+ ...Object.keys(last),
1442
+ ]);
1443
+ for (const letter of letters) {
1444
+ picks_delta[letter] = (last[letter] ?? 0) - (first[letter] ?? 0);
1445
+ }
1446
+ }
1447
+ return picks_delta ? { rounds: entries, picks_delta } : { rounds: entries };
1448
+ }
1002
1449
  export function formatAskResults(ask, json, roundFilter) {
1003
1450
  const rounds = (Array.isArray(ask.rounds) ? ask.rounds : []);
1004
1451
  const filtered = roundFilter !== undefined
1005
1452
  ? rounds.filter((r) => (typeof r.order_index === "number" ? r.order_index : 0) === roundFilter - 1)
1006
1453
  : rounds;
1007
1454
  if (json) {
1008
- const payload = roundFilter !== undefined ? { ...ask, rounds: filtered } : ask;
1455
+ let total = 0;
1456
+ let complete = 0;
1457
+ let errored = 0;
1458
+ const enrichedRounds = filtered.map((round) => {
1459
+ const stats = computeVariantStats(round);
1460
+ const aggregates = buildAggregates(round, stats);
1461
+ const decorated = denormalizeRoundCounts(round);
1462
+ total += decorated.responses_total ?? 0;
1463
+ complete += decorated.responses_complete ?? 0;
1464
+ errored += decorated.responses_errored ?? 0;
1465
+ return aggregates ? { ...decorated, aggregates } : decorated;
1466
+ });
1467
+ const testers = Array.isArray(ask.testers) ? ask.testers : undefined;
1468
+ const dedupedTesters = testers
1469
+ ? testers.map((t) => dedupeTesterSnapshot(t))
1470
+ : undefined;
1471
+ const payload = { ...ask, rounds: enrichedRounds };
1472
+ if (dedupedTesters)
1473
+ payload.testers = dedupedTesters;
1474
+ if (testers)
1475
+ payload.testers_count = testers.length;
1476
+ if (total > 0) {
1477
+ payload.responses_total = total;
1478
+ payload.responses_complete = complete;
1479
+ if (errored > 0)
1480
+ payload.responses_errored = errored;
1481
+ }
1482
+ // Pattern H2: include cross-round summary when 2+ rounds exist so agents
1483
+ // don't have to diff two `ask results` calls themselves.
1484
+ const crossRound = buildCrossRoundSummary(filtered);
1485
+ if (crossRound)
1486
+ payload.cross_round_summary = crossRound;
1009
1487
  console.log(jsonOutput(payload));
1010
1488
  return;
1011
1489
  }
@@ -1065,19 +1543,46 @@ export function formatAskResults(ask, json, roundFilter) {
1065
1543
  console.log(` ${summary.comment}`);
1066
1544
  }
1067
1545
  }
1546
+ // Pattern H2: cross-round picks comparison when 2+ rounds exist. Saves
1547
+ // agents from re-running results twice and diffing aggregates by hand.
1548
+ const crossRound = buildCrossRoundSummary(filtered);
1549
+ if (crossRound) {
1550
+ console.log("\nCross-round summary:");
1551
+ const letters = new Set();
1552
+ for (const entry of crossRound.rounds) {
1553
+ for (const letter of Object.keys(entry.picks ?? {}))
1554
+ letters.add(letter);
1555
+ }
1556
+ const headers = ["ROUND", "WINNER", ...Array.from(letters).sort()];
1557
+ const rows = crossRound.rounds.map((entry) => {
1558
+ const winnerCell = entry.winner
1559
+ ? entry.winner.tied
1560
+ ? `${entry.winner.letter} (tied)`
1561
+ : entry.winner.letter
1562
+ : "-";
1563
+ return [
1564
+ `R${entry.round_number}`,
1565
+ winnerCell,
1566
+ ...Array.from(letters).sort().map((letter) => String(entry.picks?.[letter] ?? 0)),
1567
+ ];
1568
+ });
1569
+ printTable(headers, rows);
1570
+ if (crossRound.picks_delta) {
1571
+ const deltaParts = Object.entries(crossRound.picks_delta).map(([letter, d]) => `${letter}: ${d > 0 ? "+" : ""}${d}`);
1572
+ console.log(` Δ picks (R1→R${crossRound.rounds.length}): ${deltaParts.join(", ")}`);
1573
+ }
1574
+ }
1068
1575
  }
1069
1576
  // --- Config formatting ---
1070
1577
  export function formatConfigList(configs, json) {
1071
- if (configs.length === 0) {
1072
- if (json)
1073
- console.log("[]");
1074
- else
1075
- console.log("No simulation configs.");
1076
- return;
1077
- }
1078
1578
  injectAliases(configs, ALIAS_PREFIX.config);
1079
1579
  if (json) {
1080
- console.log(jsonOutput(configs));
1580
+ // Backend returns a flat array; synthesize pagination metadata.
1581
+ console.log(jsonOutput(wrapList(configs), { preProjected: true }));
1582
+ return;
1583
+ }
1584
+ if (configs.length === 0) {
1585
+ console.log("No simulation configs.");
1081
1586
  return;
1082
1587
  }
1083
1588
  const aliasMap = getAliasMap(ALIAS_PREFIX.config);