@ishlabs/cli 0.20.0 → 0.21.0

package/dist/lib/output.js

@@ -992,7 +992,7 @@ export function formatStudyDetail(study, json, options = {}, participants) {
  * study state — fields default to `null`, `0`, or `[]` when nothing has run.
  * Agents can rely on the keys always being present (M4).
  */
-function buildStudyResultsEnvelope(study, participants) {
+export function buildStudyResultsEnvelope(study, participants) {
     const allParticipants = collectParticipants(participants, Array.isArray(study.iterations) ? study.iterations : []);
     const studyAlias = study.id
         ? deterministicAlias(ALIAS_PREFIX.study, String(study.id))
@@ -2226,3 +2226,219 @@ function formatDate(value) {
         return str.slice(0, 10);
     }
 }
+const POSITIVE_SENTIMENT = new Set(["satisfied", "curious", "engaged", "confident", "delighted"]);
+const NEGATIVE_SENTIMENT = new Set(["frustrated", "confused", "blocked", "anxious", "disappointed"]);
+function sentimentColor(label) {
+    const l = label.toLowerCase();
+    if (POSITIVE_SENTIMENT.has(l))
+        return c.green;
+    if (NEGATIVE_SENTIMENT.has(l))
+        return c.red;
+    return c.dim;
+}
+function asciiHistogram(hist, options = {}) {
+    const width = options.width ?? 20;
+    const indent = options.indent ?? "  ";
+    const entries = Object.entries(hist).filter(([, v]) => v > 0);
+    if (entries.length === 0)
+        return [];
+    const max = entries.reduce((acc, [, v]) => (v > acc ? v : acc), 0);
+    const labelWidth = entries.reduce((acc, [k]) => (k.length > acc ? k.length : acc), 0);
+    return entries
+        .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+        .map(([label, count]) => {
+        const bars = max > 0 ? Math.max(1, Math.round((count / max) * width)) : 0;
+        const color = sentimentColor(label);
+        return `${indent}${label.padEnd(labelWidth)}  ${color}${"█".repeat(bars)}${c.reset}  ${count}`;
+    });
+}
+function slicesFromProjection(projection) {
+    // Iteration projection wraps `{ study, slices, totals_unfiltered, warnings }`;
+    // all others are bare arrays. Both come through here.
+    if (Array.isArray(projection)) {
+        return projection.filter((s) => Boolean(s) && typeof s === "object" && !Array.isArray(s));
+    }
+    if (projection && typeof projection === "object") {
+        const wrapped = projection;
+        const slices = wrapped.slices;
+        if (Array.isArray(slices)) {
+            return slices.filter((s) => Boolean(s) && typeof s === "object" && !Array.isArray(s));
+        }
+    }
+    return [];
+}
+function totalInteractionsFromSlices(slices) {
+    let total = 0;
+    for (const s of slices) {
+        const n = typeof s.interaction_count === "number" ? s.interaction_count : 0;
+        total += n;
+    }
+    return total;
+}
+function totalsUnfilteredFromProjection(projection) {
+    if (projection && typeof projection === "object" && !Array.isArray(projection)) {
+        const t = projection.totals_unfiltered;
+        if (t && typeof t === "object" && !Array.isArray(t)) {
+            return t;
+        }
+    }
+    return null;
+}
+function renderIterationSlice(slice) {
+    const label = String(slice.iteration_label ?? slice.iteration_id ?? "?");
+    const pCount = Number(slice.participant_count ?? 0);
+    const iCount = Number(slice.interaction_count ?? 0);
+    console.log(`\n  ${c.bold}Iteration ${label}${c.reset}  ${c.dim}${pCount} participant${pCount !== 1 ? "s" : ""} · ${iCount} interaction${iCount !== 1 ? "s" : ""}${c.reset}`);
+    const hist = slice.sentiment ?? {};
+    for (const line of asciiHistogram(hist, { indent: "    " }))
+        console.log(line);
+    const top = Array.isArray(slice.top_actions) ? slice.top_actions : [];
+    if (top.length > 0) {
+        const parts = top.map((a) => `${a.action_type} ×${a.count}`);
+        console.log(`    ${c.dim}Top actions:${c.reset} ${parts.join(", ")}`);
+    }
+    const comments = Array.isArray(slice.sample_comments) ? slice.sample_comments : [];
+    for (const ccomment of comments) {
+        console.log(`    ${c.dim}"${ccomment}"${c.reset}`);
+    }
+}
+function renderFrameSlice(slice) {
+    const label = slice.frame_label ? String(slice.frame_label) : String(slice.frame_id);
+    const iCount = Number(slice.interaction_count ?? 0);
+    const aliases = Array.isArray(slice.participant_aliases) ? slice.participant_aliases : [];
+    console.log(`\n  ${c.bold}${label}${c.reset}  ${c.dim}${iCount} interaction${iCount !== 1 ? "s" : ""} · ${aliases.length} participant${aliases.length !== 1 ? "s" : ""}${c.reset}`);
+    const hist = slice.sentiment_histogram ?? {};
+    for (const line of asciiHistogram(hist, { indent: "    " }))
+        console.log(line);
+    const comments = Array.isArray(slice.sample_comments) ? slice.sample_comments : [];
+    for (const ccomment of comments) {
+        console.log(`    ${c.dim}"${ccomment}"${c.reset}`);
+    }
+}
+function renderSegmentSlice(slice) {
+    const idx = slice.segment_index;
+    const label = slice.segment_label ? String(slice.segment_label) : null;
+    const header = idx !== null && idx !== undefined
+        ? `Segment ${idx}${label ? ` — ${label}` : ""}`
+        : (label ?? "Segment ?");
+    const iCount = Number(slice.interaction_count ?? 0);
+    console.log(`\n  ${c.bold}${header}${c.reset}  ${c.dim}${iCount} interaction${iCount !== 1 ? "s" : ""}${c.reset}`);
+    const hist = slice.sentiment_histogram ?? {};
+    for (const line of asciiHistogram(hist, { indent: "    " }))
+        console.log(line);
+    const engagement = slice.engagement_histogram ?? {};
+    if (Object.keys(engagement).length > 0) {
+        const parts = Object.entries(engagement).map(([k, v]) => `${v} ${k}`);
+        console.log(`    ${c.dim}Engagement:${c.reset} ${parts.join(", ")}`);
+    }
+    const comments = Array.isArray(slice.sample_comments) ? slice.sample_comments : [];
+    for (const ccomment of comments) {
+        console.log(`    ${c.dim}"${ccomment}"${c.reset}`);
+    }
+}
+function renderTurnSlice(slice) {
+    const turn = Number(slice.turn_index ?? 0);
+    const iCount = Number(slice.interaction_count ?? 0);
+    const failures = Number(slice.failures ?? 0);
+    const failPart = failures > 0 ? `  ${c.red}${failures} failure${failures !== 1 ? "s" : ""}${c.reset}` : "";
+    console.log(`\n  ${c.bold}Turn ${turn}${c.reset}  ${c.dim}${iCount} interaction${iCount !== 1 ? "s" : ""}${c.reset}${failPart}`);
+    const hist = slice.sentiment_histogram ?? {};
+    for (const line of asciiHistogram(hist, { indent: "    " }))
+        console.log(line);
+    const replies = Array.isArray(slice.sample_replies) ? slice.sample_replies : [];
+    for (const r of replies) {
+        console.log(`    ${c.dim}"${r}"${c.reset}`);
+    }
+}
+function renderAssignmentSlice(slice) {
+    const name = String(slice.assignment_name ?? slice.assignment_id ?? "?");
+    const iCount = Number(slice.interaction_count ?? 0);
+    console.log(`\n  ${c.bold}${name}${c.reset}  ${c.dim}${iCount} interaction${iCount !== 1 ? "s" : ""}${c.reset}`);
+    const hist = slice.sentiment_histogram ?? {};
+    for (const line of asciiHistogram(hist, { indent: "    " }))
+        console.log(line);
+    const sc = Array.isArray(slice.step_completion) ? slice.step_completion : [];
+    if (sc.length > 0) {
+        const rows = sc.map((s) => [
+            String(s.name ?? s.step_id ?? "?"),
+            String(s.passed ?? 0),
+            String(s.inconclusive ?? 0),
+            String(s.failed ?? 0),
+            typeof s.rate === "number" ? s.rate.toFixed(2) : "-",
+        ]);
+        console.log(`    ${c.dim}Steps:${c.reset}`);
+        printTable(["STEP", "PASSED", "INCONCLUSIVE", "FAILED", "RATE"], rows);
+    }
+}
+function renderStepSlice(slice) {
+    const name = String(slice.step_name ?? slice.step_id ?? "?");
+    const assignment = String(slice.assignment_name ?? "?");
+    const total = Number(slice.total ?? 0);
+    const passed = Number(slice.passed ?? 0);
+    const inconclusive = Number(slice.inconclusive ?? 0);
+    const failed = Number(slice.failed ?? 0);
+    const rate = typeof slice.rate === "number" ? slice.rate.toFixed(2) : "-";
+    const rateColor = failed > passed ? c.red : (passed > failed ? c.green : c.dim);
+    console.log(`\n  ${c.bold}${assignment} › ${name}${c.reset}  ${rateColor}${passed}/${total} passed${c.reset}  ${c.dim}(${inconclusive} inconclusive, ${failed} failed, rate ${rate})${c.reset}`);
+    const verdicts = Array.isArray(slice.participant_verdicts)
+        ? slice.participant_verdicts
+        : [];
+    if (verdicts.length > 0) {
+        const rows = verdicts.map((v) => [
+            String(v.participant_alias ?? "-"),
+            String(v.verdict ?? "-"),
+            v.reason ? truncate(String(v.reason), 60) : "-",
+        ]);
+        printTable(["PARTICIPANT", "VERDICT", "REASON"], rows);
+    }
+}
+/**
+ * Render a `--group-by <kind>` projection. JSON mode is a thin pass-through
+ * to jsonOutput with `preProjected: true` so the lean transform doesn't
+ * strip our stable empties. Human mode renders one section per slice plus
+ * a small ASCII sentiment histogram.
+ *
+ * The renderer accepts both the wrapped `{study, slices, ...}` shape (per-
+ * iteration) and the bare-array shape (every other --group-by); the
+ * surface (T5) doesn't need to know the difference.
+ */
+export function formatStudyResultsGroupBy(projection, kind, json) {
+    if (json) {
+        console.log(jsonOutput(projection, { preProjected: true }));
+        return;
+    }
+    const slices = slicesFromProjection(projection);
+    const totalInteractions = totalInteractionsFromSlices(slices);
+    const unfiltered = totalsUnfilteredFromProjection(projection);
+    const totalUnfiltered = unfiltered && typeof unfiltered.interaction_count === "number"
+        ? unfiltered.interaction_count
+        : null;
+    const headline = `Sliced by ${kind}: ${slices.length} group${slices.length !== 1 ? "s" : ""} (${totalInteractions}${totalUnfiltered !== null ? `/${totalUnfiltered}` : ""} interaction${totalInteractions !== 1 ? "s" : ""})`;
+    console.log(`${c.bold}${headline}${c.reset}`);
+    if (slices.length === 0) {
+        console.log(`  ${c.dim}(no groups matched)${c.reset}`);
+        return;
+    }
+    for (const slice of slices) {
+        switch (kind) {
+            case "iteration":
+                renderIterationSlice(slice);
+                break;
+            case "frame":
+                renderFrameSlice(slice);
+                break;
+            case "segment":
+                renderSegmentSlice(slice);
+                break;
+            case "turn":
+                renderTurnSlice(slice);
+                break;
+            case "assignment":
+                renderAssignmentSlice(slice);
+                break;
+            case "step":
+                renderStepSlice(slice);
+                break;
+        }
+    }
+}

package/dist/lib/skill-content.js

@@ -917,6 +917,70 @@ Rules to remember:
 See \`ish docs get-page concepts/extending-a-simulation\` for the full
 mental model (cancel + extend as a pair, error envelopes, cost model).
 
+## 12. Slice study results by frame / segment / turn / sentiment
+
+Goal: ask narrower questions of a finished run than the kitchen-sink
+\`ish study results\` envelope answers. The canonical use case:
+**"what differed on the login screen across these five iterations?"**.
+
+\`\`\`bash
+# 12a. Across-iterations comparison on one frame (the canonical question).
+#      --frame matches frame names by case-insensitive substring; pass
+#      a full Frame UUID or an f-… alias when the name is ambiguous.
+ish study results s-b2c --frame login --group-by iteration --json
+
+# 12b. Frustrated reactions to one segment of a video study:
+ish study results s-b2c --segment 3 --sentiment Frustrated
+
+# 12c. Who failed the "verify email" step, and why?
+#      --group-by step exposes per-participant verdicts inline so you
+#      don't fan out across participants.
+ish study results s-b2c --assignment "Sign up" --step verify-email \\
+    --group-by step --json
+
+# 12d. Pair-mode chat: only side A turn 4.
+ish study results s-b2c --side a --turn 4
+
+# 12e. Sanity-check coverage when a filter narrows the slice:
+ish study results s-b2c --frame checkout --json \\
+    | jq '{matched: .participant_count, total: .totals_unfiltered.participant_count}'
+
+# 12f. A filter that matches zero interactions still returns the stable
+#      envelope shape — participant_count: 0, totals_unfiltered populated,
+#      exit code 0 (not 4). Never error on no-match.
+ish study results s-b2c --frame doesnotexist --json
+# → ValidationError because "doesnotexist" matches no frame names; pass
+#   --include-unmatched only when --frame DID resolve and you want the
+#   degraded captures (frame_version_id: null) back.
+\`\`\`
+
+Rules to remember:
+- **Filters compose with AND across flags; OR within \`--sentiment\`.**
+  \`--frame login --sentiment Frustrated,Confused\` keeps only login-frame
+  interactions whose sentiment is Frustrated OR Confused.
+- **Modality mismatch is not an error.** \`--segment 0\` on an
+  interactive study emits a stderr warning and is ignored. The
+  exception is **\`--group-by\`** — \`--group-by frame\` on a chat study,
+  \`--group-by turn\` on a video study, etc. error at the router (exit 2).
+- **Empty-slice contract: exit 0, not 4.** Zero matches return a
+  stable envelope with \`participant_count: 0\` and
+  \`totals_unfiltered\` populated. Agents key on
+  \`totals_unfiltered.participant_count\` to ask "is the filter too
+  tight, or did the run not produce data?".
+- \`--frame\` accepts a name substring, a Frame UUID, an \`f-…\` alias,
+  or a \`frame_version_id\` UUID. Ambiguous substring (matches >1
+  frame) errors with the candidate list.
+- \`--summary\` is orthogonal to filters and narrows the summary over
+  the filtered set. \`--transcript\` is single-participant and errors
+  (exit 2) when **any** filter or \`--group-by\` is set.
+- Per-step output exposes \`participant_verdicts: [{participant_alias,
+  verdict, reason, evidence_interaction_ids}]\` — not
+  \`per_participant_verdicts\`. The verdict enum is \`passed\` /
+  \`inconclusive\` / \`failed\`.
+
+See \`ish docs get-page guides/slicing-results\` for the full filter
+table, projection shapes, and the defensive null-handling rules.
+
 ## Tips for chaining commands as an agent
 
 - Capture aliases from JSON: \`ITER=$(ish iteration create --url … --json | jq -r .alias)\`
@@ -1010,6 +1074,10 @@ mental model (cancel + extend as a pair, error envelopes, cost model).
 | List of participants from \`study run\`        | \`--json \\| jq '.participants[].id'\`        | \`--get participant_aliases\` (or \`participant_ids\` for UUIDs)                |
 | Per-answer sentiment                      | \`--json \\| jq '...'\` per participant       | \`ish study results <id> --json\` (sentiment is on every answer row) |
 | "Did this run land?" headline             | \`study results --json\` + jq filtering | \`ish study results <id> --summary --json\`                          |
+| Across-iterations comparison on one frame | \`study results --json\` + jq per iteration | \`ish study results <id> --frame login --group-by iteration --json\` |
+| Per-step pass/fail with reasons inline    | \`study participant --json\` per participant + jq | \`ish study results <id> --step verify-email --group-by step --json\` |
+| Frustrated reactions to one media segment | \`study results --json\` + jq | \`ish study results <id> --segment 3 --sentiment Frustrated --json\` |
+| Sanity-check filter coverage              | hand-count \`.participants\` vs total | \`--get totals_unfiltered.participant_count\` (set on every sliced envelope) |
 | Chat transcript for one participant (external_chatbot) | \`study participant --json\` + jq      | \`ish study results <id> --transcript <participant_id> --json\`           |
 | Pair-mode conversation transcripts        | \`study participant --json\` per participant       | \`ish iteration get <iter-id> --json \\| jq '.conversations[]'\`     |
 | Participant headline only (no action timeline) | \`study participant --json\` + jq            | \`ish study participant <id> --summary --json\`                           |

package/dist/lib/study-participants.d.ts

@@ -6,6 +6,19 @@
  * (person, interactions[], participant_summary, interview_answers, …) that
  * used to be embedded under `study.iterations[*].participants[*]` on the
  * legacy `GET /studies/{id}` response.
+ *
+ * Audit (study-results-slice plan, T4): the flat endpoint already returns
+ * everything the new `ish study results --frame/--segment/--step/...` filter
+ * pipeline needs in a single round-trip — no per-participant fan-out:
+ *   - `interactions[]` (modality-discriminated via `ParticipantWithAttributesPublicResponse`)
+ *   - `participant_assignments[].step_results[]` with `{step_id, name,
+ *     description, verdict, reason, evidence_interaction_ids[]}`, hydrated
+ *     by `attach_participant_step_results_flat` in the study repository
+ *     before serialisation (`ish-backend/app/api/study/repository.py:315`)
+ *   - `participant_summary`, `interview_answers`
+ * If a future filter ever needs `conversation_id` on each interaction (for
+ * `--group-by conversation`), that's a backend-side addition on
+ * `_InteractionResponseBase`, not a CLI change.
  */
 import type { ApiClient } from "./api-client.js";
 import type { Participant } from "./types.js";

package/dist/lib/study-participants.js

@@ -6,6 +6,19 @@
  * (person, interactions[], participant_summary, interview_answers, …) that
  * used to be embedded under `study.iterations[*].participants[*]` on the
  * legacy `GET /studies/{id}` response.
+ *
+ * Audit (study-results-slice plan, T4): the flat endpoint already returns
+ * everything the new `ish study results --frame/--segment/--step/...` filter
+ * pipeline needs in a single round-trip — no per-participant fan-out:
+ *   - `interactions[]` (modality-discriminated via `ParticipantWithAttributesPublicResponse`)
+ *   - `participant_assignments[].step_results[]` with `{step_id, name,
+ *     description, verdict, reason, evidence_interaction_ids[]}`, hydrated
+ *     by `attach_participant_step_results_flat` in the study repository
+ *     before serialisation (`ish-backend/app/api/study/repository.py:315`)
+ *   - `participant_summary`, `interview_answers`
+ * If a future filter ever needs `conversation_id` on each interaction (for
+ * `--group-by conversation`), that's a backend-side addition on
+ * `_InteractionResponseBase`, not a CLI change.
  */
 export async function fetchStudyParticipants(client, studyId, opts) {
     return await client.get(`/studies/${studyId}/participants`, undefined, opts);

package/dist/lib/study-results-filters.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Pure filter pipeline for `ish study results`.
+ *
+ * Input  : the raw `GET /studies/{id}` payload, the raw
+ *          `GET /studies/{id}/participants` payload, the raw
+ *          `GET /studies/{id}/frames` payload (or [] when --frame wasn't
+ *          passed), and a `ResultsFilters` struct from the command surface.
+ * Output : a `FilteredResults` struct — the trimmed participant graph,
+ *          pre-filter counts on `totals_unfiltered`, and a `warnings[]`
+ *          list of modality-mismatch notes for the surface to surface on
+ *          stderr.
+ *
+ * Has no IO and no console side-effects — the caller (study results action)
+ * owns network calls and stderr; we just compute. That keeps the function
+ * trivially unit-testable and lets the projection builders (T3) consume the
+ * same shape without re-walking the graph.
+ *
+ * Defensive null handling is the load-bearing piece. See the plan's
+ * "Defensive handling of nullable fields" section — read it before editing
+ * any predicate.
+ */
+export interface ResultsFilters {
+    /** Frame name (case-insensitive substring), Frame UUID, frame alias `f-...`,
+     *  or a `frame_version_id` UUID. Resolved against the study's frames list. */
+    frame?: string;
+    /** Segment index (parseable int) OR a substring matched against
+     *  `actions[0].data.segment_label` on each interaction. */
+    segment?: string;
+    /** Chat turn index — matched against `actions[0].data.turn_index`. */
+    turn?: number;
+    /** participant_pair side — matched against the parent assignment's `side`. */
+    side?: "a" | "b";
+    /** Assignment UUID, OR a substring matched against
+     *  `study.assignments[].name`. */
+    assignment?: string;
+    /** Step id OR a case-insensitive substring against step `name`. Walks
+     *  `participant_assignments[].step_results[]`. */
+    step?: string;
+    /** Comma-or-repeat list of sentiment labels (case-insensitive). */
+    sentiment?: string[];
+    /** Actor field — case-insensitive match against `interaction.actor`. */
+    actor?: "ai" | "human" | "user";
+    /** Iteration UUID or `label`. */
+    iteration?: string;
+    /** Participant UUID or alias (`pt-...`). */
+    participant?: string;
+    /** When --frame is set, keep interactions with null frame_version_id
+     *  under a synthetic `_unmatched` bucket instead of dropping them. */
+    includeUnmatched?: boolean;
+    /** Pair with --step: also drop interactions whose id is not in any
+     *  surviving `step_results[].evidence_interaction_ids[]`. */
+    includeEvidence?: boolean;
+}
+export interface FilteredResults {
+    /** Shallow copy of the study payload — same shape as the raw response.
+     *  Participants are NOT embedded here; they're carried alongside on
+     *  `participants`. */
+    study: Record<string, unknown>;
+    /** Participants whose interactions[] survived the predicate walk.
+     *  Empty participants are dropped only when an interaction-level filter
+     *  was set (preserves the stable schema when the caller just asked
+     *  "who ran?" without slicing). */
+    participants: Record<string, unknown>[];
+    /** The frame list returned by the surface, with each frame's
+     *  `frame_version_ids[]` flattened onto the row for downstream
+     *  enrichment. Empty when --frame wasn't passed or the modality isn't
+     *  interactive. */
+    frames: Record<string, unknown>[];
+    /** Pre-filter participant + interaction counts, so callers can see
+     *  "matched X / Y". */
+    totals_unfiltered: {
+        participant_count: number;
+        interaction_count: number;
+    };
+    /** Modality-mismatch notes (e.g. "--segment ignored on interactive").
+     *  The surface emits these on stderr. */
+    warnings: string[];
+    /** When --frame was set, the resolved set of frame_version_ids that
+     *  passed. Used by the projection builders (T3) to enrich surviving
+     *  interactions with frame_id / frame_label without re-resolving. */
+    matchedFrameVersionIds: Set<string>;
+    /** Maps frame_version_id → {frame_id, frame_label} for enrichment. */
+    frameVersionLookup: Map<string, {
+        frame_id: string;
+        frame_label: string | null;
+    }>;
+}
+/**
+ * Pure entry point. See file-level comment for input/output contract.
+ */
+export declare function applyResultsFilters(study: Record<string, unknown>, participants: Record<string, unknown>[], rawFrames: Record<string, unknown>[], filters: ResultsFilters): FilteredResults;