@ishlabs/cli 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/chat.js +2 -2
- package/dist/commands/config.js +17 -3
- package/dist/commands/source.js +1 -1
- package/dist/commands/study-analyze.js +15 -2
- package/dist/commands/study-participant.js +19 -0
- package/dist/commands/study.js +96 -34
- package/dist/lib/command-helpers.js +4 -3
- package/dist/lib/docs.js +69 -41
- package/dist/lib/output.d.ts +14 -9
- package/dist/lib/output.js +91 -19
- package/dist/lib/skill-content.js +9 -1
- package/dist/lib/study-results-filters.js +35 -14
- package/dist/lib/study-results-projections.d.ts +47 -17
- package/dist/lib/study-results-projections.js +39 -36
- package/package.json +1 -1
package/dist/commands/chat.js
CHANGED
|
@@ -115,7 +115,7 @@ Examples:
|
|
|
115
115
|
});
|
|
116
116
|
parent
|
|
117
117
|
.command("create")
|
|
118
|
-
.description("Create a chatbot endpoint from a config
|
|
118
|
+
.description("Create a chatbot endpoint from a hand-written ChatbotEndpointConfig JSON (advanced — use `chat endpoint init` to infer the config from a curl sample, JSON, or template).")
|
|
119
119
|
.requiredOption("--endpoint-config <file>", 'Path to JSON file (or "-" for stdin)')
|
|
120
120
|
.option("--name <name>", "Override the name from the config file")
|
|
121
121
|
.option("--workspace <id>", "Workspace ID")
|
|
@@ -342,7 +342,7 @@ endpoint, apply the override, and PUT the merged result. Field flags win over
|
|
|
342
342
|
function attachChatEndpointInit(parent) {
|
|
343
343
|
parent
|
|
344
344
|
.command("init")
|
|
345
|
-
.description("Author an endpoint from a curl/JSON sample via test-and-map, or from a known-good template")
|
|
345
|
+
.description("Author an endpoint from a curl/JSON sample via test-and-map, or from a known-good template (recommended for most users — use `chat endpoint create` only when you already have a hand-written ChatbotEndpointConfig).")
|
|
346
346
|
.option("--from-curl <file>", 'Path to a curl example file (or "-" for stdin)')
|
|
347
347
|
.option("--from-json <file>", 'Path to a JSON request/response sample (or "-" for stdin)')
|
|
348
348
|
.option("--template <name>", `Start from a known-good template (one of: ${TEMPLATE_NAMES.join(", ")})`)
|
package/dist/commands/config.js
CHANGED
|
@@ -78,11 +78,25 @@ Run \`ish docs overview\` for the full mental model.`);
|
|
|
78
78
|
});
|
|
79
79
|
config
|
|
80
80
|
.command("schema")
|
|
81
|
-
.description("Get simulation config schema with defaults")
|
|
81
|
+
.description("Get simulation config schema with defaults (admin-only — non-admin accounts: ask an admin to share an existing config ID and pass it via `ish study run --config <id>`).")
|
|
82
82
|
.action(async (_opts, cmd) => {
|
|
83
83
|
await withClient(cmd, async (client, globals) => {
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
try {
|
|
85
|
+
const data = await client.get("/dev/simulation-configs/schema");
|
|
86
|
+
output(data, globals.json);
|
|
87
|
+
}
|
|
88
|
+
catch (err) {
|
|
89
|
+
// Pattern Z: re-throw with a hint pointing non-admin agents at the
|
|
90
|
+
// workaround (use a shared config ID via `study run --config`).
|
|
91
|
+
if (err instanceof Error && err.status === 403) {
|
|
92
|
+
const tagged = err;
|
|
93
|
+
const extra = "Non-admin accounts cannot introspect the simulation-config schema. To still use a config, ask an admin to share an existing config ID and pass it via `ish study run --config <id>` (`config --help` for the full workflow).";
|
|
94
|
+
tagged.suggestions = Array.isArray(tagged.suggestions)
|
|
95
|
+
? [...tagged.suggestions, extra]
|
|
96
|
+
: [extra];
|
|
97
|
+
}
|
|
98
|
+
throw err;
|
|
99
|
+
}
|
|
86
100
|
});
|
|
87
101
|
});
|
|
88
102
|
config
|
package/dist/commands/source.js
CHANGED
|
@@ -27,7 +27,7 @@ same attachment across multiple generation runs; otherwise pass a local path dir
|
|
|
27
27
|
to \`person generate --source\` and it auto-uploads.
|
|
28
28
|
|
|
29
29
|
Concept pages: ish docs get-page concepts/source
|
|
30
|
-
ish docs get-page concepts/
|
|
30
|
+
ish docs get-page concepts/person`);
|
|
31
31
|
source
|
|
32
32
|
.command("upload")
|
|
33
33
|
.description("Upload a file as a participant attachment and wait for processing")
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* about latency than load).
|
|
15
15
|
*/
|
|
16
16
|
import { withClient, resolveStudy, parseWaitTimeout } from "../lib/command-helpers.js";
|
|
17
|
-
import { resolveId } from "../lib/alias-store.js";
|
|
17
|
+
import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
18
18
|
import { output, printTable } from "../lib/output.js";
|
|
19
19
|
import { WaitTimeoutError } from "./study-run.js";
|
|
20
20
|
const POLL_INTERVAL_MS = 5_000;
|
|
@@ -160,11 +160,24 @@ Trigger a new run with \`ish study analyze --wait\`.`)
|
|
|
160
160
|
const history = await client.get(`/studies/${studyId}/results`);
|
|
161
161
|
const latest = history[0] ?? null;
|
|
162
162
|
if (globals.json) {
|
|
163
|
+
// Pattern K: never emit empty stdout. When no analyses have run,
|
|
164
|
+
// ship a stable envelope with a hint pointing at the verb that
|
|
165
|
+
// populates it. Mirrors the `study results` empty-envelope contract.
|
|
166
|
+
if (!latest) {
|
|
167
|
+
const studyAlias = tagAlias(ALIAS_PREFIX.study, studyId);
|
|
168
|
+
output({
|
|
169
|
+
latest: null,
|
|
170
|
+
history: [],
|
|
171
|
+
hint: `No analyses run yet. Trigger one with \`ish study analyze ${studyAlias}\`.`,
|
|
172
|
+
}, true, { preProjected: true });
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
163
175
|
output({ latest, history }, true);
|
|
164
176
|
return;
|
|
165
177
|
}
|
|
166
178
|
if (!latest) {
|
|
167
|
-
|
|
179
|
+
const studyAlias = tagAlias(ALIAS_PREFIX.study, studyId);
|
|
180
|
+
console.log(`No analysis runs yet. Trigger one with \`ish study analyze ${studyAlias}\`.`);
|
|
168
181
|
return;
|
|
169
182
|
}
|
|
170
183
|
if (opts.all) {
|
|
@@ -76,6 +76,25 @@ Tips:
|
|
|
76
76
|
const result = data;
|
|
77
77
|
if (result.id)
|
|
78
78
|
result.alias = tagAlias(ALIAS_PREFIX.participant, String(result.id));
|
|
79
|
+
// Pattern L: enrich with parent-graph aliases so agents can traverse
|
|
80
|
+
// from a participant straight to its study without hopping through
|
|
81
|
+
// `iteration get`. The participant response carries `iteration_id` but
|
|
82
|
+
// not `study_id`; one iteration fetch supplies both.
|
|
83
|
+
const iterationId = typeof result.iteration_id === "string" ? result.iteration_id : null;
|
|
84
|
+
if (iterationId) {
|
|
85
|
+
result.iteration_alias = tagAlias(ALIAS_PREFIX.iteration, iterationId);
|
|
86
|
+
try {
|
|
87
|
+
const iter = await client.get(`/iterations/${iterationId}`);
|
|
88
|
+
if (typeof iter.study_id === "string") {
|
|
89
|
+
result.study_id = iter.study_id;
|
|
90
|
+
result.study_alias = tagAlias(ALIAS_PREFIX.study, iter.study_id);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// Best-effort enrichment; if the iteration fetch fails (deleted,
|
|
95
|
+
// permission), keep going with the alias we already injected.
|
|
96
|
+
}
|
|
97
|
+
}
|
|
79
98
|
if (opts.summary) {
|
|
80
99
|
output(buildParticipantSummary(result), globals.json, { preProjected: true });
|
|
81
100
|
return;
|
package/dist/commands/study.js
CHANGED
|
@@ -8,7 +8,7 @@ import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
|
8
8
|
import { loadConfig, saveConfig } from "../config.js";
|
|
9
9
|
import { formatStudyList, formatStudyDetail, formatStudyResults, buildStudyResultsEnvelope, buildStudyResultsSummary, buildChatTranscript, formatStudyResultsGroupBy, output, ValidationError, } from "../lib/output.js";
|
|
10
10
|
import { applyResultsFilters } from "../lib/study-results-filters.js";
|
|
11
|
-
import { buildStudyResultsPerIteration, buildStudyResultsPerFrame, buildStudyResultsPerSegment, buildStudyResultsPerTurn, buildStudyResultsPerAssignment, buildStudyResultsPerStep, } from "../lib/study-results-projections.js";
|
|
11
|
+
import { buildStudyResultsPerIteration, buildStudyResultsPerFrame, buildStudyResultsPerSegment, buildStudyResultsPerTurn, buildStudyResultsPerAssignment, buildStudyResultsPerStep, wrapSliceProjection, } from "../lib/study-results-projections.js";
|
|
12
12
|
import { VALID_CONTENT_TYPES } from "../lib/types.js";
|
|
13
13
|
import { fetchStudyParticipants } from "../lib/study-participants.js";
|
|
14
14
|
import { parseAssignment, loadAssignmentsFile, validateAssignmentsArray, parseQuestion } from "../lib/study-inputs.js";
|
|
@@ -611,7 +611,7 @@ Next: configure a run with \`ish iteration create --study <id>\`,
|
|
|
611
611
|
});
|
|
612
612
|
study
|
|
613
613
|
.command("get")
|
|
614
|
-
.description("Get study
|
|
614
|
+
.description("Get the full study payload — iterations (with run details), assignments, interview questions, sentiment + status counts. Accepts multiple IDs for batched lookup. NOTE: this is the full payload, not a roll-up — for a compact cross-study comparison view use `study results <id> --summary`.")
|
|
615
615
|
.argument("<ids...>", "Study ID(s) — one or more aliases/UUIDs (space- or comma-separated)")
|
|
616
616
|
.addHelpText("after", `
|
|
617
617
|
Examples:
|
|
@@ -636,6 +636,18 @@ list table layout in human mode.`)
|
|
|
636
636
|
const result = data;
|
|
637
637
|
if (result.id)
|
|
638
638
|
result.alias = tagAlias(ALIAS_PREFIX.study, String(result.id));
|
|
639
|
+
// Pattern I-r3-1: inline iterations carry only label/name/details
|
|
640
|
+
// from the wire; tag each with its `alias` (computed from id via
|
|
641
|
+
// the local alias-store) so agents can drill from `study get` into
|
|
642
|
+
// `iteration get <alias>` / `study results --iteration <alias>`
|
|
643
|
+
// without a separate `iteration list` round-trip.
|
|
644
|
+
if (Array.isArray(result.iterations)) {
|
|
645
|
+
for (const iter of result.iterations) {
|
|
646
|
+
if (typeof iter.id === "string") {
|
|
647
|
+
iter.alias = tagAlias(ALIAS_PREFIX.iteration, iter.id);
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
}
|
|
639
651
|
if (data.product_id) {
|
|
640
652
|
result.url = getWebUrl(globals, `/${data.product_id}/${rid}/overview`);
|
|
641
653
|
}
|
|
@@ -655,6 +667,13 @@ list table layout in human mode.`)
|
|
|
655
667
|
const r = data;
|
|
656
668
|
if (r.id)
|
|
657
669
|
r.alias = tagAlias(ALIAS_PREFIX.study, String(r.id));
|
|
670
|
+
if (Array.isArray(r.iterations)) {
|
|
671
|
+
for (const iter of r.iterations) {
|
|
672
|
+
if (typeof iter.id === "string") {
|
|
673
|
+
iter.alias = tagAlias(ALIAS_PREFIX.iteration, iter.id);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
}
|
|
658
677
|
if (data.product_id) {
|
|
659
678
|
r.url = getWebUrl(globals, `/${data.product_id}/${rid}/overview`);
|
|
660
679
|
}
|
|
@@ -671,7 +690,7 @@ list table layout in human mode.`)
|
|
|
671
690
|
});
|
|
672
691
|
study
|
|
673
692
|
.command("results")
|
|
674
|
-
.description("View aggregated results: participant counts, sentiment, interview answers. Returns a stable envelope with empty fields when no runs have completed. Slice with filter flags (--frame, --segment, --turn, --side, --assignment, --step, --sentiment, --actor, --iteration, --participant) or project with --group-by (iteration|frame|segment|turn|assignment|step).")
|
|
693
|
+
.description("View aggregated results: participant counts, sentiment, interview answers. Returns a stable envelope with empty fields when no runs have completed. Slice with filter flags (--frame [interactive], --segment [video/audio/text/document], --turn [chat], --side [chat participant_pair], --assignment, --step, --sentiment, --actor, --iteration, --participant) or project with --group-by <axis> (iteration | frame [interactive] | segment [media] | turn [chat] | assignment | step).")
|
|
675
694
|
.argument("<id>", "Study ID")
|
|
676
695
|
.option("--workspace <id>", "Workspace ID; accepted for consistency (workspace is inferred from the study)")
|
|
677
696
|
.option("--summary", "Lean summary projection: counts + sentiment + per-participant {alias, status, sentiment, comment}. Drops interview_answers + per-interaction breakdowns. Composes with filters: `--summary --frame login` narrows the summary to the login-screen interactions.")
|
|
@@ -688,7 +707,7 @@ list table layout in human mode.`)
|
|
|
688
707
|
.option("--side <a|b>", "Filter participant_pair chat interactions by assignment side. Other modalities: warned and ignored.")
|
|
689
708
|
.option("--assignment <ref>", "Filter to a single assignment by UUID or name (substring, case-insensitive).")
|
|
690
709
|
.option("--step <ref>", "Filter `participant_assignments[].step_results[]` to a single step by step-id or name (substring). Pair with --include-evidence to also drop non-evidence interactions.")
|
|
691
|
-
.option("--sentiment <labels>", "Filter to interactions whose sentiment.label is in the comma-separated list (case-insensitive; repeatable). Drops null
|
|
710
|
+
.option("--sentiment <labels>", "Filter to interactions whose sentiment.label is in the comma-separated list (case-insensitive; repeatable). Drops interactions whose sentiment is null. A participant is kept when at least one of their interactions matches, even if their aggregate session sentiment is null (e.g. failed runs with a pre-error matching interaction).", collectIds, [])
|
|
692
711
|
.option("--actor <actor>", "Filter to interactions whose actor is `ai`, `human`, or `user` (case-insensitive).")
|
|
693
712
|
.option("--iteration <ref>", "Restrict to a single iteration by UUID or label.")
|
|
694
713
|
.option("--participant <ref>", "Restrict to a single participant by UUID or `pt-…` alias.")
|
|
@@ -713,6 +732,7 @@ Default --json envelope (M10: per-answer sentiment now included):
|
|
|
713
732
|
"participant_count": 12,
|
|
714
733
|
"completed_count": 8,
|
|
715
734
|
"failed_count": 0,
|
|
735
|
+
"participant_status_counts": { "completed": 8, "running": 3, "draft": 1 },
|
|
716
736
|
"sentiment": { "counts": { "Satisfied": 5, "Frustrated": 2 }, "total": 7 },
|
|
717
737
|
"interview_answers": [
|
|
718
738
|
{ "question": "...", "type": "text",
|
|
@@ -733,6 +753,12 @@ When any filter flag is passed, the envelope gains a \`totals_unfiltered\` field
|
|
|
733
753
|
("matched 12 / 80 participants"). A zero-match filter returns the stable
|
|
734
754
|
envelope with participant_count=0 and exit code 0 (not 4).
|
|
735
755
|
|
|
756
|
+
Filtered count semantics: \`participant_count\` is the matched-set total (every
|
|
757
|
+
participant whose interactions matched the filter — including running and
|
|
758
|
+
failed). The unfiltered denominator is \`totals_unfiltered.participant_count\`,
|
|
759
|
+
and the same envelope still carries \`completed_count\` / \`failed_count\` so
|
|
760
|
+
agents can compute "completed AND matched" without a second call.
|
|
761
|
+
|
|
736
762
|
--summary projection (M2-friction-7: drops the interview_answers payload):
|
|
737
763
|
{ study, participant_count, completed_count, failed_count, sentiment, participants: [...] }
|
|
738
764
|
|
|
@@ -749,23 +775,34 @@ envelope with participant_count=0 and exit code 0 (not 4).
|
|
|
749
775
|
"participant_summary": { "comment": "...", "sentiment": {...} }
|
|
750
776
|
}
|
|
751
777
|
|
|
752
|
-
--group-by
|
|
753
|
-
{
|
|
778
|
+
--group-by projections share one envelope (uniform across all six axes):
|
|
779
|
+
{ axis, rows, totals_unfiltered, modality_warnings, study_id, modality }
|
|
754
780
|
|
|
755
|
-
|
|
756
|
-
|
|
781
|
+
axis echoes the requested axis (iteration|frame|segment|turn|assignment|step)
|
|
782
|
+
study_id the \`s-…\` alias
|
|
783
|
+
modality the study's modality
|
|
784
|
+
totals_unfiltered { participant_count, interaction_count } — pre-filter counts
|
|
785
|
+
modality_warnings any filter-flag mismatches (e.g. --turn on a non-chat study)
|
|
757
786
|
|
|
758
|
-
|
|
759
|
-
[{ segment_index, segment_label, interaction_count, sentiment_histogram, engagement_histogram, sample_comments }, ...]
|
|
787
|
+
Per-axis row shape (one element of \`rows[]\`):
|
|
760
788
|
|
|
761
|
-
--group-by
|
|
762
|
-
|
|
789
|
+
--group-by iteration:
|
|
790
|
+
{ iteration_id, iteration_label, participant_count, interaction_count, sentiment, sample_comments, top_actions }
|
|
763
791
|
|
|
764
|
-
--group-by
|
|
765
|
-
|
|
792
|
+
--group-by frame (interactive only):
|
|
793
|
+
{ frame_id, frame_label, interaction_count, sentiment_histogram, sample_comments, participant_aliases }
|
|
766
794
|
|
|
767
|
-
--group-by
|
|
768
|
-
|
|
795
|
+
--group-by segment (video/audio/text/document):
|
|
796
|
+
{ segment_index, segment_label, interaction_count, sentiment_histogram, engagement_histogram, sample_comments }
|
|
797
|
+
|
|
798
|
+
--group-by turn (chat only):
|
|
799
|
+
{ turn_index, interaction_count, sentiment_histogram, sample_replies, failures }
|
|
800
|
+
|
|
801
|
+
--group-by assignment:
|
|
802
|
+
{ assignment_id, assignment_name, interaction_count, sentiment_histogram, step_completion }
|
|
803
|
+
|
|
804
|
+
--group-by step:
|
|
805
|
+
{ assignment_id, assignment_name, step_id, step_name, total, passed, inconclusive, failed, rate, participant_verdicts: [{ participant_alias, verdict, reason, evidence_interaction_ids }] }
|
|
769
806
|
|
|
770
807
|
Tips:
|
|
771
808
|
Use \`--get <path>\` for a single value (e.g. \`--get participant_count\`),
|
|
@@ -794,17 +831,24 @@ Common --get paths (--transcript <participant_id> envelope):
|
|
|
794
831
|
--get participant_summary.sentiment # aggregate sentiment map
|
|
795
832
|
--get unique_bot_replies # bot-side message count
|
|
796
833
|
|
|
797
|
-
Common --get paths (--group-by
|
|
798
|
-
--get
|
|
799
|
-
--get
|
|
800
|
-
--get
|
|
801
|
-
--get
|
|
802
|
-
--get
|
|
803
|
-
--get
|
|
804
|
-
|
|
805
|
-
--get
|
|
806
|
-
--get 0.
|
|
807
|
-
--get 0.
|
|
834
|
+
Common --get paths (--group-by envelope — uniform across axes):
|
|
835
|
+
--get axis # echoes the requested axis
|
|
836
|
+
--get study_id # s-… alias
|
|
837
|
+
--get modality # study's modality
|
|
838
|
+
--get modality_warnings # filter-flag mismatches (one warning per line)
|
|
839
|
+
--get totals_unfiltered.participant_count # pre-filter participant count
|
|
840
|
+
--get totals_unfiltered.interaction_count # pre-filter interaction count
|
|
841
|
+
|
|
842
|
+
--get rows.iteration_label # per-iteration: one label per line
|
|
843
|
+
--get rows.0.participant_count # per-iteration: first row's count
|
|
844
|
+
--get rows.0.frame_label # per-frame: first row's label
|
|
845
|
+
--get rows.0.sentiment_histogram # per-frame/segment/turn: first row's sentiment map
|
|
846
|
+
--get rows.0.segment_index # per-segment: first row's index
|
|
847
|
+
--get rows.0.turn_index # per-turn: first row's index
|
|
848
|
+
--get rows.0.assignment_name # per-assignment/step: first row's assignment
|
|
849
|
+
--get rows.0.step_name # per-step: first row's step
|
|
850
|
+
--get rows.0.rate # per-step: first row's pass-rate
|
|
851
|
+
--get rows.0.participant_verdicts.verdict # per-step: verdict per participant
|
|
808
852
|
|
|
809
853
|
When no runs have completed, the default envelope is returned with zero counts and empty arrays.`)
|
|
810
854
|
.action(async (id, opts, cmd) => {
|
|
@@ -930,15 +974,27 @@ When no runs have completed, the default envelope is returned with zero counts a
|
|
|
930
974
|
// (devon's T7 note: projection builders are intentionally
|
|
931
975
|
// modality-agnostic and bucket non-matching rows into `_unmatched`;
|
|
932
976
|
// the surface is responsible for refusing nonsensical axes up front).
|
|
977
|
+
// Pattern B: modality-mismatched --group-by names the offending axis's
|
|
978
|
+
// domain AND suggests the axis that DOES apply to the study's current
|
|
979
|
+
// modality, so a cold-start agent can retry productively in one hop.
|
|
980
|
+
const axisHint = (mod) => {
|
|
981
|
+
if (mod === "interactive")
|
|
982
|
+
return "use --group-by frame";
|
|
983
|
+
if (["video", "audio", "text", "document"].includes(mod))
|
|
984
|
+
return "use --group-by segment";
|
|
985
|
+
if (mod === "chat")
|
|
986
|
+
return "use --group-by turn";
|
|
987
|
+
return undefined;
|
|
988
|
+
};
|
|
933
989
|
if (groupByKind === "frame" && modality !== "interactive") {
|
|
934
|
-
throw new ValidationError(`--group-by frame requires modality=interactive; this study is "${modality}".`, ["interactive"]);
|
|
990
|
+
throw new ValidationError(`--group-by frame requires modality=interactive; this study is "${modality}".`, ["interactive"], axisHint(modality));
|
|
935
991
|
}
|
|
936
992
|
const SEGMENT_MODALITIES = ["video", "audio", "text", "document"];
|
|
937
993
|
if (groupByKind === "segment" && !SEGMENT_MODALITIES.includes(modality)) {
|
|
938
|
-
throw new ValidationError(`--group-by segment requires modality ∈ {${SEGMENT_MODALITIES.join(", ")}}; this study is "${modality}".`, SEGMENT_MODALITIES);
|
|
994
|
+
throw new ValidationError(`--group-by segment requires modality ∈ {${SEGMENT_MODALITIES.join(", ")}}; this study is "${modality}".`, SEGMENT_MODALITIES, axisHint(modality));
|
|
939
995
|
}
|
|
940
996
|
if (groupByKind === "turn" && modality !== "chat") {
|
|
941
|
-
throw new ValidationError(`--group-by turn requires modality=chat; this study is "${modality}".`, ["chat"]);
|
|
997
|
+
throw new ValidationError(`--group-by turn requires modality=chat; this study is "${modality}".`, ["chat"], axisHint(modality));
|
|
942
998
|
}
|
|
943
999
|
// Coerce the frames payload to a plain array of records (the API
|
|
944
1000
|
// returns a bare array). Tolerate `{items: [...]}` shape in case the
|
|
@@ -995,7 +1051,8 @@ When no runs have completed, the default envelope is returned with zero counts a
|
|
|
995
1051
|
projection = buildStudyResultsPerStep(filtered);
|
|
996
1052
|
break;
|
|
997
1053
|
}
|
|
998
|
-
|
|
1054
|
+
const envelope = wrapSliceProjection(filtered, groupByKind, projection, rid, modality);
|
|
1055
|
+
formatStudyResultsGroupBy(envelope, groupByKind, globals.json);
|
|
999
1056
|
return;
|
|
1000
1057
|
}
|
|
1001
1058
|
if (wantsSummary) {
|
|
@@ -1011,13 +1068,18 @@ When no runs have completed, the default envelope is returned with zero counts a
|
|
|
1011
1068
|
return;
|
|
1012
1069
|
}
|
|
1013
1070
|
// Default (no --group-by, no --summary) but filters set: stable
|
|
1014
|
-
// envelope on the filtered participants + totals_unfiltered
|
|
1015
|
-
//
|
|
1016
|
-
//
|
|
1071
|
+
// envelope on the filtered participants + totals_unfiltered + the
|
|
1072
|
+
// modality_warnings array (Pattern U). Without `modality_warnings`
|
|
1073
|
+
// on this envelope, agents who pipe stderr to /dev/null lose the
|
|
1074
|
+
// filter-mismatch signal entirely; the `--group-by` envelope
|
|
1075
|
+
// already carries it (see wrapSliceProjection), so this is just
|
|
1076
|
+
// closing the asymmetry. Empty slice contract: zero matches still
|
|
1077
|
+
// yields participant_count=0 and exit 0, never a 4/not-found.
|
|
1017
1078
|
const envelope = buildStudyResultsEnvelope(filtered.study, filtered.participants);
|
|
1018
1079
|
const envelopeOut = {
|
|
1019
1080
|
...envelope,
|
|
1020
1081
|
totals_unfiltered: filtered.totals_unfiltered,
|
|
1082
|
+
modality_warnings: filtered.warnings,
|
|
1021
1083
|
};
|
|
1022
1084
|
output(envelopeOut, globals.json, { preProjected: true });
|
|
1023
1085
|
});
|
|
@@ -294,15 +294,16 @@ function enforceParticipantCap(ids, flags, opts) {
|
|
|
294
294
|
*/
|
|
295
295
|
export function addPersonFilterFlags(cmd, opts = {}) {
|
|
296
296
|
const allFlag = opts.allFlagName ?? "--all";
|
|
297
|
-
const allDesc = opts.allFlagDescription ?? "Use every person matching the filters"
|
|
297
|
+
const allDesc = (opts.allFlagDescription ?? "Use every person matching the filters")
|
|
298
|
+
+ " (capped at 20 per dispatch — split into multiple slices for larger cohorts)";
|
|
298
299
|
return cmd
|
|
299
300
|
.option("--person <ids>", "Person IDs/aliases (comma-separated or repeatable)", collectIds, [])
|
|
300
|
-
.option("--sample <N>", "Randomly sample N people from the matching pool")
|
|
301
|
+
.option("--sample <N>", "Randomly sample N people from the matching pool (max 20 per dispatch — split into multiple slices for larger cohorts)")
|
|
301
302
|
.option(allFlag, allDesc)
|
|
302
303
|
.option("--search <text>", "Substring match against person name")
|
|
303
304
|
.option("--bio <text>", "Substring match against person bio")
|
|
304
305
|
.option("--occupation <text>", "Substring match against person occupation (repeatable)", collectRepeatable, [])
|
|
305
|
-
.option("--gender <gender>", "Filter by gender (repeatable)", collectRepeatable, [])
|
|
306
|
+
.option("--gender <gender>", "Filter by gender (female, male, nonbinary; repeatable, OR semantics)", collectRepeatable, [])
|
|
306
307
|
.option("--country <code>", "Filter by 2-letter country code (repeatable)", collectRepeatable, [])
|
|
307
308
|
.option("--min-age <n>", "Minimum age (inclusive)")
|
|
308
309
|
.option("--max-age <n>", "Maximum age (inclusive)")
|
package/dist/lib/docs.js
CHANGED
|
@@ -635,7 +635,7 @@ Tunables (both modes):
|
|
|
635
635
|
the parties signal the conversation is over.
|
|
636
636
|
|
|
637
637
|
Pair-mode rules:
|
|
638
|
-
- Each side needs **either** \`--
|
|
638
|
+
- Each side needs **either** \`--group-a\` / \`--group-b\` (explicit IDs) **or**
|
|
639
639
|
\`--role-criteria-*\` (filter the backend resolves). The two can also
|
|
640
640
|
be combined — criteria then acts as validation on the explicit list.
|
|
641
641
|
- When both sides use explicit \`--group-a\` / \`--group-b\`, they
|
|
@@ -657,7 +657,7 @@ Pair-mode rules:
|
|
|
657
657
|
\`type\` field in \`--questionnaire\` / \`--questions\` manifests
|
|
658
658
|
(\`single-choice\` ↔ \`single_choice\`).
|
|
659
659
|
- Audiences are pinned to the iteration. \`ish study run\` refuses
|
|
660
|
-
run-time people overrides (\`--
|
|
660
|
+
run-time people overrides (\`--person\` / \`--sample\` / \`--all\` /
|
|
661
661
|
filters) on a pair iteration — change the peoples via
|
|
662
662
|
\`ish iteration update <id> --details-json '{...}'\` instead.
|
|
663
663
|
- \`--max-turns\` / \`--early-termination\` on \`ish study run\` override
|
|
@@ -1174,7 +1174,7 @@ const CONCEPT_PROFILE = `# concept: person
|
|
|
1174
1174
|
A **person** is a reusable persona — the simulated
|
|
1175
1175
|
human whose behaviour drives a participant instance during a study or ask.
|
|
1176
1176
|
|
|
1177
|
-
- Alias prefix: \`
|
|
1177
|
+
- Alias prefix: \`p-\`
|
|
1178
1178
|
- Lives at the workspace level, reusable across studies and asks.
|
|
1179
1179
|
- Distinct from a "participant" (\`pt-\`) — a participant is one *instance* of a
|
|
1180
1180
|
profile inside one iteration.
|
|
@@ -1336,7 +1336,7 @@ A **source** is an input to \`ish person generate\`: a transcript,
|
|
|
1336
1336
|
audio file, image, or PDF that an LLM reads to ground generated profiles
|
|
1337
1337
|
in real customer evidence.
|
|
1338
1338
|
|
|
1339
|
-
- Alias prefix: \`
|
|
1339
|
+
- Alias prefix: \`ps-\`
|
|
1340
1340
|
- Source kinds: \`text_file | audio | image\` (auto-detected from extension; \`text-file\` is accepted as a hyphen variant).
|
|
1341
1341
|
- Audio supports speaker diarization via \`--diarize\`.
|
|
1342
1342
|
|
|
@@ -1406,7 +1406,7 @@ flags. Two ways to select:
|
|
|
1406
1406
|
\`platform\` until the next release with a server-side
|
|
1407
1407
|
deprecation warning)
|
|
1408
1408
|
|
|
1409
|
-
The two modes are **mutually exclusive** — pass either \`--
|
|
1409
|
+
The two modes are **mutually exclusive** — pass either \`--person\` or
|
|
1410
1410
|
the filter set, not both.
|
|
1411
1411
|
|
|
1412
1412
|
## Empty-pool suggestions
|
|
@@ -1658,7 +1658,7 @@ and what they target differ.
|
|
|
1658
1658
|
| Default | latest iteration of the active study | append a round to the active ask |
|
|
1659
1659
|
| Fresh setup | \`ish iteration create …\` first, then run | \`--new\` (creates ask + round 1 in one shot) |
|
|
1660
1660
|
| Specific target| \`--iteration <id>\` | positional ask id (\`a-6ec\`) |
|
|
1661
|
-
| Audience | \`--
|
|
1661
|
+
| Audience | \`--person\` OR filters with \`--sample\`/\`--all\` — else reuse iteration's participants | only at \`--new\`; fixed for the ask afterwards |
|
|
1662
1662
|
| Output unit | per-participant interactions + questionnaire answers | per-participant reactions per round |
|
|
1663
1663
|
|
|
1664
1664
|
## Decision rule
|
|
@@ -1746,7 +1746,7 @@ When extend is **not** the right verb:
|
|
|
1746
1746
|
- Source participant is still RUNNING. \`cancel\` it first, then extend.
|
|
1747
1747
|
Extend refuses non-terminal sources server-side.
|
|
1748
1748
|
- You want a fresh cohort with new people flags. Use \`study run\`
|
|
1749
|
-
with \`--
|
|
1749
|
+
with \`--person\` / \`--sample\` / \`--all\` instead — extend is a
|
|
1750
1750
|
per-participant resume, not a batch op.
|
|
1751
1751
|
- You want to change the iteration's URL or content. Edit the iteration
|
|
1752
1752
|
itself (\`iteration update\` or a fresh iteration) — extend always
|
|
@@ -1906,8 +1906,8 @@ time the CLI sees an entity.
|
|
|
1906
1906
|
- \`s-\` study
|
|
1907
1907
|
- \`i-\` iteration
|
|
1908
1908
|
- \`pt-\` participant (instance of a person in an iteration)
|
|
1909
|
-
- \`
|
|
1910
|
-
- \`
|
|
1909
|
+
- \`p-\` person
|
|
1910
|
+
- \`ps-\` person source
|
|
1911
1911
|
- \`a-\` ask
|
|
1912
1912
|
- \`r-\` ask round
|
|
1913
1913
|
- \`c-\` config (simulation config)
|
|
@@ -2423,7 +2423,7 @@ not branch on \`status: 0\` — that value is never emitted as of 0.20.
|
|
|
2423
2423
|
- Lists print as JSON arrays (or paginated wrappers). Single resources
|
|
2424
2424
|
as JSON objects.
|
|
2425
2425
|
- Field names match the underlying API resource (snake_case).
|
|
2426
|
-
- Aliases (\`s-…\`, \`a-…\`, \`
|
|
2426
|
+
- Aliases (\`s-…\`, \`a-…\`, \`p-…\`, …) appear alongside UUIDs in
|
|
2427
2427
|
\`--verbose\` mode and replace UUIDs in default lean mode.
|
|
2428
2428
|
|
|
2429
2429
|
## Examples
|
|
@@ -2473,11 +2473,14 @@ reshaping output.
|
|
|
2473
2473
|
\`--turn\`, \`--side\`, \`--assignment\`, \`--step\`, \`--sentiment\`,
|
|
2474
2474
|
\`--actor\`, \`--iteration\`, \`--participant\`) and projection flags
|
|
2475
2475
|
(\`--group-by iteration|frame|segment|turn|assignment|step\`). When any
|
|
2476
|
-
filter is passed
|
|
2477
|
-
(\`{participant_count,
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
(not 4) — slicing
|
|
2476
|
+
filter is passed on the default \`study results\` envelope, the envelope
|
|
2477
|
+
gains a \`totals_unfiltered\` field (\`{participant_count,
|
|
2478
|
+
interaction_count}\`) so an agent can sanity-check coverage: "matched
|
|
2479
|
+
12 / 80 participants". A zero-match filter returns the stable envelope
|
|
2480
|
+
with \`participant_count: 0\` and exit code **0** (not 4) — slicing
|
|
2481
|
+
never errors on no-match. \`--group-by\` returns a different shape — a
|
|
2482
|
+
uniform envelope \`{axis, rows, totals_unfiltered, modality_warnings,
|
|
2483
|
+
study_id, modality}\` (see \`guides/slicing-results\`).
|
|
2481
2484
|
|
|
2482
2485
|
\`--group-by\` is **router-gated by modality**: \`frame\` requires
|
|
2483
2486
|
interactive, \`segment\` requires media (video / audio / text / document),
|
|
@@ -2509,7 +2512,7 @@ client-side; no extra round trip beyond the standard study fetch.
|
|
|
2509
2512
|
| \`--step <ref>\` | Filters \`participant_assignments[].step_results[]\` to verdicts matching the step id or name. | interactive + external_chatbot chat (steps live there) |
|
|
2510
2513
|
| \`--sentiment <labels>\` | Comma-separated, case-insensitive label list (repeatable). Drops null-sentiment rows. | all |
|
|
2511
2514
|
| \`--actor <ai\|human\|user>\` | Restrict by actor. | all |
|
|
2512
|
-
| \`--iteration <ref>\` | Iteration UUID or label (\`A\`, \`B\`, … case-insensitive).
|
|
2515
|
+
| \`--iteration <ref>\` | Iteration UUID, iteration alias (\`i-…\`), or label (\`A\`, \`B\`, … case-insensitive). | all |
|
|
2513
2516
|
| \`--participant <ref>\` | Participant UUID or \`pt-…\` alias. | all |
|
|
2514
2517
|
| \`--include-unmatched\` | With \`--frame\`, keep degraded captures (\`frame_version_id: null\`) under a synthetic \`_unmatched\` bucket instead of dropping them. | interactive |
|
|
2515
2518
|
| \`--include-evidence\` | With \`--step\`, also drop interactions not listed in any surviving \`step_results[].evidence_interaction_ids[]\`. | interactive + external_chatbot chat |
|
|
@@ -2520,33 +2523,52 @@ The exception is \`--group-by\` — see below.
|
|
|
2520
2523
|
|
|
2521
2524
|
## Projection flags (--group-by)
|
|
2522
2525
|
|
|
2523
|
-
|
|
2526
|
+
Every \`--group-by\` axis returns the same envelope:
|
|
2527
|
+
\`{axis, rows, totals_unfiltered, modality_warnings, study_id, modality}\`.
|
|
2528
|
+
Top-level \`axis\` echoes the requested axis; \`study_id\` is the \`s-…\`
|
|
2529
|
+
alias; \`modality\` echoes the study's modality. \`rows\` is an
|
|
2530
|
+
axis-specific array of slice objects (see the table below for the per-row
|
|
2531
|
+
shape). \`modality_warnings\` carries any filter-flag mismatches
|
|
2532
|
+
(e.g. \`--turn\` on a non-chat study); empty array when none.
|
|
2533
|
+
|
|
2534
|
+
| Axis | Row shape (one element of \`rows[]\`) | Modality |
|
|
2524
2535
|
|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|
|
|
2525
|
-
| \`iteration\` | \`{
|
|
2526
|
-
| \`frame\` | \`
|
|
2527
|
-
| \`segment\` | \`
|
|
2528
|
-
| \`turn\` | \`
|
|
2529
|
-
| \`assignment\` | \`
|
|
2530
|
-
| \`step\` | \`
|
|
2536
|
+
| \`iteration\` | \`{iteration_id, iteration_label, participant_count, interaction_count, sentiment, sample_comments, top_actions}\` | all |
|
|
2537
|
+
| \`frame\` | \`{frame_id, frame_label, interaction_count, sentiment_histogram, sample_comments, participant_aliases}\` | interactive (router errors on non-interactive) |
|
|
2538
|
+
| \`segment\` | \`{segment_index, segment_label, interaction_count, sentiment_histogram, engagement_histogram, sample_comments}\` | media (router errors on non-media) |
|
|
2539
|
+
| \`turn\` | \`{turn_index, interaction_count, sentiment_histogram, sample_replies, failures}\` | chat (router errors on non-chat) |
|
|
2540
|
+
| \`assignment\` | \`{assignment_id, assignment_name, interaction_count, sentiment_histogram, step_completion}\` | all |
|
|
2541
|
+
| \`step\` | \`{assignment_id, assignment_name, step_id, step_name, total, passed, inconclusive, failed, rate, participant_verdicts: [{participant_alias, verdict, reason, evidence_interaction_ids}]}\` | interactive + external_chatbot chat |
|
|
2531
2542
|
|
|
2532
2543
|
\`--group-by\` is **mutually exclusive with \`--summary\` and
|
|
2533
2544
|
\`--transcript\`**. \`--group-by frame\` on a chat study, \`--group-by
|
|
2534
2545
|
turn\` on a video study, etc. error at the surface (exit 2) with a
|
|
2535
|
-
clear message before any IO.
|
|
2546
|
+
clear message before any IO. The error envelope includes a \`hint\`
|
|
2547
|
+
field naming the axis that DOES apply to the study's modality
|
|
2548
|
+
(\`use --group-by segment\` on audio/video/text/document, \`use --group-by
|
|
2549
|
+
turn\` on chat, \`use --group-by frame\` on interactive) — agents can
|
|
2550
|
+
branch on it to retry productively in one hop.
|
|
2536
2551
|
|
|
2537
2552
|
## The empty-slice contract
|
|
2538
2553
|
|
|
2539
2554
|
A filter combination that matches zero interactions returns the
|
|
2540
|
-
**
|
|
2555
|
+
**uniform envelope** with:
|
|
2541
2556
|
|
|
2542
|
-
- \`
|
|
2557
|
+
- \`rows: []\`
|
|
2543
2558
|
- \`totals_unfiltered: {participant_count: <N>, interaction_count: <M>}\` populated
|
|
2559
|
+
- \`axis\`, \`study_id\`, \`modality\` still populated
|
|
2544
2560
|
- exit code **0** (not 4)
|
|
2545
2561
|
|
|
2546
2562
|
\`totals_unfiltered\` is the agent's sanity check: *"my filter matched
|
|
2547
2563
|
0 of 80 participants — is the filter too tight, or did the run not
|
|
2548
2564
|
produce data?"*. The shape never collapses to \`null\` or a different
|
|
2549
|
-
envelope; \`--get participant_count\` is always safe
|
|
2565
|
+
envelope; \`--get participant_count\` is always safe on the default
|
|
2566
|
+
(non-\`--group-by\`) envelope.
|
|
2567
|
+
|
|
2568
|
+
The default+filter envelope (no \`--group-by\`) also carries
|
|
2569
|
+
\`modality_warnings: string[]\` — any filter flags that were dropped as
|
|
2570
|
+
off-modality (e.g. \`--turn 1\` on an interactive study) appear here.
|
|
2571
|
+
Agents piping stderr to \`/dev/null\` get the same signal on stdout.
|
|
2550
2572
|
|
|
2551
2573
|
## Worked examples
|
|
2552
2574
|
|
|
@@ -2617,22 +2639,26 @@ No match at all errors and lists the available frame names.
|
|
|
2617
2639
|
|
|
2618
2640
|
\`\`\`
|
|
2619
2641
|
# Sanity-check coverage:
|
|
2642
|
+
--get axis
|
|
2643
|
+
--get study_id
|
|
2644
|
+
--get modality
|
|
2620
2645
|
--get totals_unfiltered.participant_count
|
|
2621
2646
|
--get totals_unfiltered.interaction_count
|
|
2647
|
+
--get modality_warnings
|
|
2622
2648
|
|
|
2623
|
-
# Per-iteration projection:
|
|
2624
|
-
--get
|
|
2625
|
-
--get
|
|
2626
|
-
--get
|
|
2649
|
+
# Per-iteration projection rows:
|
|
2650
|
+
--get rows.iteration_label # one label per line
|
|
2651
|
+
--get rows.0.participant_count
|
|
2652
|
+
--get rows.0.sentiment
|
|
2627
2653
|
|
|
2628
|
-
# Per-frame / per-segment / per-turn (
|
|
2629
|
-
--get 0.frame_label
|
|
2630
|
-
--get 0.segment_index
|
|
2631
|
-
--get 0.sentiment_histogram
|
|
2654
|
+
# Per-frame / per-segment / per-turn (rows[] is the axis array):
|
|
2655
|
+
--get rows.0.frame_label
|
|
2656
|
+
--get rows.0.segment_index
|
|
2657
|
+
--get rows.0.sentiment_histogram
|
|
2632
2658
|
|
|
2633
2659
|
# Per-step:
|
|
2634
|
-
--get 0.rate
|
|
2635
|
-
--get 0.participant_verdicts.verdict
|
|
2660
|
+
--get rows.0.rate
|
|
2661
|
+
--get rows.0.participant_verdicts.verdict
|
|
2636
2662
|
\`\`\`
|
|
2637
2663
|
|
|
2638
2664
|
## Related
|
|
@@ -3013,6 +3039,8 @@ free credits before re-dispatch.
|
|
|
3013
3039
|
estimate at preview time — the CLI prints the shape (\`N × … × 2\`)
|
|
3014
3040
|
instead of a number.
|
|
3015
3041
|
|
|
3042
|
+
**Naming note:** "tier" in ish means **billing** tier (FREE / STARTER / PRO / ENTERPRISE — a credit-budget knob). It is NOT a simulation-quality dial. Per-run simulation behaviour (model, timing, retries) is controlled via \`ish config\` — see \`ish config --help\`. \`docs search tier\` returns billing results by design.
|
|
3043
|
+
|
|
3016
3044
|
## Related
|
|
3017
3045
|
|
|
3018
3046
|
- \`reference/billing-limits\` — per-tier *entity* caps (max
|
|
@@ -3447,13 +3475,13 @@ Optional \`--max-turns <n>\` (default 12) caps the chat per participant.
|
|
|
3447
3475
|
|
|
3448
3476
|
Audience size is set at run time for **external_chatbot** chat
|
|
3449
3477
|
studies. Use \`--sample <N>\` to pick N random simulatable profiles,
|
|
3450
|
-
or \`--all\` for the full pool. \`--
|
|
3478
|
+
or \`--all\` for the full pool. \`--person <ids>\` is also supported
|
|
3451
3479
|
for explicit selection:
|
|
3452
3480
|
\`\`\`
|
|
3453
3481
|
ish study run stu-xyz --sample 5 --wait
|
|
3454
3482
|
\`\`\`
|
|
3455
3483
|
|
|
3456
|
-
> **Pair-mode is different.** \`--sample\` / \`--
|
|
3484
|
+
> **Pair-mode is different.** \`--sample\` / \`--person\` / demographic
|
|
3457
3485
|
> filters on \`study run\` are **refused** for participant_pair iterations
|
|
3458
3486
|
> — pair groups live on the iteration itself. Set them at
|
|
3459
3487
|
> iteration-create time via \`--group-a/-b\` (with 1×N broadcast)
|
|
@@ -3609,7 +3637,7 @@ Keys (all optional): \`occupation\`, \`min_age\`, \`max_age\`,
|
|
|
3609
3637
|
\`requires_captions\`, \`uses_screen_reader\`, \`prefers_reduced_motion\`,
|
|
3610
3638
|
\`prefers_high_contrast\`, \`has_any_accessibility_need\`. The five \`*_in\`
|
|
3611
3639
|
arrays accept snake_case spec values; the five accessibility filters are
|
|
3612
|
-
booleans. Combine \`--
|
|
3640
|
+
booleans. Combine \`--group-a\` / \`--group-b\` and \`--role-criteria-*\` on the same side
|
|
3613
3641
|
to make criteria validate an explicit list (mismatch blocks the run).
|
|
3614
3642
|
|
|
3615
3643
|
MECE notes for the list filters:
|
|
@@ -3995,7 +4023,7 @@ cap at 40 entries.
|
|
|
3995
4023
|
- \`concepts/person\` — what a person is; structured fields.
|
|
3996
4024
|
- \`concepts/source\` — interview transcripts / audio / PDF inputs
|
|
3997
4025
|
for the people-generation flow.
|
|
3998
|
-
- \`reference/aliases\` — \`
|
|
4026
|
+
- \`reference/aliases\` — \`p-…\` is the person alias prefix.
|
|
3999
4027
|
`;
|
|
4000
4028
|
const GUIDE_MCP_ADD = `# guide: wire ish into your AI clients (\`ish mcp add\`)
|
|
4001
4029
|
|
package/dist/lib/output.d.ts
CHANGED
|
@@ -35,10 +35,16 @@ export declare function outputList(rows: unknown[], json: boolean): void;
|
|
|
35
35
|
/**
|
|
36
36
|
* Error with valid options — used for content_type and similar validation.
|
|
37
37
|
* Surfaces valid_options in JSON so agents can self-correct.
|
|
38
|
+
*
|
|
39
|
+
* Optional `hint` is the agent's *actionable next step* (e.g. for a wrong
|
|
40
|
+
* --group-by axis on the current modality, the axis that DOES apply). Distinct
|
|
41
|
+
* from `valid_options`, which describes where the supplied value WOULD be
|
|
42
|
+
* valid. Both serialize into the error envelope when present.
|
|
38
43
|
*/
|
|
39
44
|
export declare class ValidationError extends Error {
|
|
40
45
|
valid_options: string[];
|
|
41
|
-
|
|
46
|
+
hint?: string | undefined;
|
|
47
|
+
constructor(message: string, valid_options: string[], hint?: string | undefined);
|
|
42
48
|
}
|
|
43
49
|
export declare function outputError(err: unknown, json: boolean): void;
|
|
44
50
|
export declare function printTable(headers: string[], rows: string[][]): void;
|
|
@@ -110,13 +116,12 @@ export declare function formatAskResults(ask: Record<string, unknown>, json: boo
|
|
|
110
116
|
export declare function formatConfigList(configs: Record<string, unknown>[], json: boolean): void;
|
|
111
117
|
export type StudyResultsGroupByKind = "iteration" | "frame" | "segment" | "turn" | "assignment" | "step";
|
|
112
118
|
/**
|
|
113
|
-
* Render a `--group-by <kind>` projection
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
* surface (T5) doesn't need to know the difference.
|
|
119
|
+
* Render a `--group-by <kind>` projection wrapped in the uniform
|
|
120
|
+
* `SliceResponse` envelope (`{ axis, rows, totals_unfiltered,
|
|
121
|
+
* modality_warnings, study_id, modality }`). JSON mode is a thin
|
|
122
|
+
* pass-through to jsonOutput with `preProjected: true` so the lean
|
|
123
|
+
* transform doesn't strip our stable empties. Human mode pulls slices
|
|
124
|
+
* out of `rows` and renders one section per slice plus a small ASCII
|
|
125
|
+
* sentiment histogram.
|
|
121
126
|
*/
|
|
122
127
|
export declare function formatStudyResultsGroupBy(projection: unknown, kind: StudyResultsGroupByKind, json: boolean): void;
|
package/dist/lib/output.js
CHANGED
|
@@ -278,6 +278,53 @@ function pickFields(data, fields) {
|
|
|
278
278
|
}
|
|
279
279
|
return data;
|
|
280
280
|
}
|
|
281
|
+
/**
|
|
282
|
+
* Pattern A: when an agent passes `--fields foo,bar` and one of those names
|
|
283
|
+
* doesn't exist on the response, emit a one-line stderr warning naming the
|
|
284
|
+
* missing fields plus a sample of what IS available. Otherwise unknown names
|
|
285
|
+
* silently drop and the agent assumes the field doesn't exist on the wire,
|
|
286
|
+
* when the more common cause is a typo or the wrong projection.
|
|
287
|
+
*
|
|
288
|
+
* Probes the response shape: for an object response, the top-level keys;
|
|
289
|
+
* for a list-wrapper response, the keys of `items[0]`; for a bare array,
|
|
290
|
+
* the keys of element 0. Warns at most once per command invocation
|
|
291
|
+
* (the caller invokes this from jsonOutput before pickFields).
|
|
292
|
+
*/
|
|
293
|
+
function warnOnUnknownFields(data, fields) {
|
|
294
|
+
let probe = null;
|
|
295
|
+
if (Array.isArray(data) && data.length > 0 && typeof data[0] === "object" && data[0] !== null) {
|
|
296
|
+
probe = data[0];
|
|
297
|
+
}
|
|
298
|
+
else if (data && typeof data === "object" && !Array.isArray(data)) {
|
|
299
|
+
const obj = data;
|
|
300
|
+
if (isListWrapper(obj) && Array.isArray(obj.items) && obj.items.length > 0
|
|
301
|
+
&& typeof obj.items[0] === "object" && obj.items[0] !== null) {
|
|
302
|
+
probe = obj.items[0];
|
|
303
|
+
}
|
|
304
|
+
else {
|
|
305
|
+
probe = obj;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
if (!probe)
|
|
309
|
+
return;
|
|
310
|
+
const missing = fields.filter((f) => !(f in probe));
|
|
311
|
+
if (missing.length === 0)
|
|
312
|
+
return;
|
|
313
|
+
// Pattern DD: surface↔backend rename hints. The agent-friendly noun is
|
|
314
|
+
// "workspace" but the backend stores `product_id`; agents who guess the
|
|
315
|
+
// surface name need a did-you-mean to find the actual response key.
|
|
316
|
+
const RENAME_MAP = {
|
|
317
|
+
workspace_id: "product_id",
|
|
318
|
+
workspace: "product",
|
|
319
|
+
};
|
|
320
|
+
const renameHints = missing
|
|
321
|
+
.filter((m) => RENAME_MAP[m] && RENAME_MAP[m] in probe)
|
|
322
|
+
.map((m) => `${m} → ${RENAME_MAP[m]}`);
|
|
323
|
+
const available = Object.keys(probe).slice(0, 12).join(", ");
|
|
324
|
+
const more = Object.keys(probe).length > 12 ? `, … (${Object.keys(probe).length - 12} more)` : "";
|
|
325
|
+
const didYouMean = renameHints.length > 0 ? ` Did you mean: ${renameHints.join(", ")}?` : "";
|
|
326
|
+
console.error(`warning: --fields requested ${missing.length === 1 ? "name" : "names"} not on the response: ${missing.join(", ")}.${didYouMean} Available: ${available}${more}.`);
|
|
327
|
+
}
|
|
281
328
|
/** Serialize data as JSON, applying lean transform and field selection. */
|
|
282
329
|
function jsonOutput(data, options = {}) {
|
|
283
330
|
let out;
|
|
@@ -297,6 +344,7 @@ function jsonOutput(data, options = {}) {
|
|
|
297
344
|
out = leanJson(data, options.writePath);
|
|
298
345
|
}
|
|
299
346
|
if (_fields && _fields.length > 0) {
|
|
347
|
+
warnOnUnknownFields(out, _fields);
|
|
300
348
|
out = pickFields(out, _fields);
|
|
301
349
|
}
|
|
302
350
|
// Pattern Ω capture mode: --get <field> returns bare values instead of
|
|
@@ -396,12 +444,19 @@ export function outputList(rows, json) {
|
|
|
396
444
|
/**
|
|
397
445
|
* Error with valid options — used for content_type and similar validation.
|
|
398
446
|
* Surfaces valid_options in JSON so agents can self-correct.
|
|
447
|
+
*
|
|
448
|
+
* Optional `hint` is the agent's *actionable next step* (e.g. for a wrong
|
|
449
|
+
* --group-by axis on the current modality, the axis that DOES apply). Distinct
|
|
450
|
+
* from `valid_options`, which describes where the supplied value WOULD be
|
|
451
|
+
* valid. Both serialize into the error envelope when present.
|
|
399
452
|
*/
|
|
400
453
|
export class ValidationError extends Error {
|
|
401
454
|
valid_options;
|
|
402
|
-
|
|
455
|
+
hint;
|
|
456
|
+
constructor(message, valid_options, hint) {
|
|
403
457
|
super(message);
|
|
404
458
|
this.valid_options = valid_options;
|
|
459
|
+
this.hint = hint;
|
|
405
460
|
this.name = "ValidationError";
|
|
406
461
|
}
|
|
407
462
|
}
|
|
@@ -434,6 +489,11 @@ function suggestionsForError(err) {
|
|
|
434
489
|
return [
|
|
435
490
|
"Run a list command to see available resources",
|
|
436
491
|
"Check that the alias or ID is correct",
|
|
492
|
+
// Pattern R: an active workspace / study / ask saved in config can
|
|
493
|
+
// outlive the resource on the server. Implicit lookups then 404
|
|
494
|
+
// with no indication that the ID came from config. `ish status`
|
|
495
|
+
// flags orphans; `<entity> use --clear` resets the active value.
|
|
496
|
+
"If you didn't pass the resource explicitly, your saved active workspace/study/ask may be stale — run `ish status` to check, then `ish workspace use --clear` (or `ish study use --clear` / `ish ask use --clear`) to reset.",
|
|
437
497
|
];
|
|
438
498
|
case "insufficient_credits":
|
|
439
499
|
return ["Purchase more credits at https://app.ishlabs.io"];
|
|
@@ -593,11 +653,14 @@ export function outputError(err, json) {
|
|
|
593
653
|
error_code: "validation_error",
|
|
594
654
|
retryable: false,
|
|
595
655
|
valid_options: err.valid_options,
|
|
656
|
+
...(err.hint && { hint: err.hint }),
|
|
596
657
|
...(suggestions.length > 0 && { suggestions }),
|
|
597
658
|
}));
|
|
598
659
|
}
|
|
599
660
|
else {
|
|
600
661
|
console.error(`Error: ${err.message}`);
|
|
662
|
+
if (err.hint)
|
|
663
|
+
console.error(` hint: ${err.hint}`);
|
|
601
664
|
for (const s of suggestions)
|
|
602
665
|
console.error(` → ${s}`);
|
|
603
666
|
}
|
|
@@ -635,6 +698,9 @@ export function outputError(err, json) {
|
|
|
635
698
|
? tagged.suggestions.filter((s) => typeof s === "string")
|
|
636
699
|
: [];
|
|
637
700
|
const mergedSuggestions = [...new Set([...suggestions, ...taggedSuggestions])];
|
|
701
|
+
const availableValues = Array.isArray(tagged.available_values)
|
|
702
|
+
? tagged.available_values.filter((s) => typeof s === "string")
|
|
703
|
+
: undefined;
|
|
638
704
|
if (json) {
|
|
639
705
|
console.error(JSON.stringify({
|
|
640
706
|
// Generic Error: CLI-thrown (we control the message), so we don't
|
|
@@ -647,6 +713,7 @@ export function outputError(err, json) {
|
|
|
647
713
|
...(errorKind && { error_kind: errorKind }),
|
|
648
714
|
...(example && { example }),
|
|
649
715
|
...(progress !== undefined && { progress }),
|
|
716
|
+
...(availableValues && availableValues.length > 0 && { available_values: availableValues }),
|
|
650
717
|
...(seededIds && { seeded_but_not_dispatched_ids: seededIds }),
|
|
651
718
|
...(seededAliases && { seeded_but_not_dispatched_aliases: seededAliases }),
|
|
652
719
|
...(mergedSuggestions.length > 0 && { suggestions: mergedSuggestions }),
|
|
@@ -998,6 +1065,14 @@ export function buildStudyResultsEnvelope(study, participants) {
|
|
|
998
1065
|
? deterministicAlias(ALIAS_PREFIX.study, String(study.id))
|
|
999
1066
|
: null;
|
|
1000
1067
|
const completedCount = allParticipants.filter((t) => t.status === "completed" || t.status === "complete").length;
|
|
1068
|
+
// Pattern N: per-status breakdown so callers can distinguish running /
|
|
1069
|
+
// pending / cancelled from terminal completed/failed. Additive — the
|
|
1070
|
+
// aggregate counts (`completed_count` / `failed_count`) stay alongside.
|
|
1071
|
+
const participantStatusCounts = {};
|
|
1072
|
+
for (const t of allParticipants) {
|
|
1073
|
+
const key = (t.status || "unknown").toLowerCase();
|
|
1074
|
+
participantStatusCounts[key] = (participantStatusCounts[key] || 0) + 1;
|
|
1075
|
+
}
|
|
1001
1076
|
// Aggregate sentiment across all interactions on all participants.
|
|
1002
1077
|
const sentimentCounts = {};
|
|
1003
1078
|
let sentimentTotal = 0;
|
|
@@ -1066,6 +1141,7 @@ export function buildStudyResultsEnvelope(study, participants) {
|
|
|
1066
1141
|
participant_count: allParticipants.length,
|
|
1067
1142
|
completed_count: completedCount,
|
|
1068
1143
|
failed_count: failedCount,
|
|
1144
|
+
participant_status_counts: participantStatusCounts,
|
|
1069
1145
|
sentiment,
|
|
1070
1146
|
interview_answers: interviewAnswers,
|
|
1071
1147
|
participants: participantRows,
|
|
@@ -2253,16 +2329,13 @@ function asciiHistogram(hist, options = {}) {
|
|
|
2253
2329
|
});
|
|
2254
2330
|
}
|
|
2255
2331
|
function slicesFromProjection(projection) {
|
|
2256
|
-
//
|
|
2257
|
-
//
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
const slices = wrapped.slices;
|
|
2264
|
-
if (Array.isArray(slices)) {
|
|
2265
|
-
return slices.filter((s) => Boolean(s) && typeof s === "object" && !Array.isArray(s));
|
|
2332
|
+
// Surface wraps every --group-by axis in the uniform SliceResponse envelope
|
|
2333
|
+
// `{ axis, rows, totals_unfiltered, modality_warnings, study_id, modality }`;
|
|
2334
|
+
// slices live under `rows`.
|
|
2335
|
+
if (projection && typeof projection === "object" && !Array.isArray(projection)) {
|
|
2336
|
+
const rows = projection.rows;
|
|
2337
|
+
if (Array.isArray(rows)) {
|
|
2338
|
+
return rows.filter((s) => Boolean(s) && typeof s === "object" && !Array.isArray(s));
|
|
2266
2339
|
}
|
|
2267
2340
|
}
|
|
2268
2341
|
return [];
|
|
@@ -2393,14 +2466,13 @@ function renderStepSlice(slice) {
|
|
|
2393
2466
|
}
|
|
2394
2467
|
}
|
|
2395
2468
|
/**
|
|
2396
|
-
* Render a `--group-by <kind>` projection
|
|
2397
|
-
*
|
|
2398
|
-
*
|
|
2399
|
-
*
|
|
2400
|
-
*
|
|
2401
|
-
*
|
|
2402
|
-
*
|
|
2403
|
-
* surface (T5) doesn't need to know the difference.
|
|
2469
|
+
* Render a `--group-by <kind>` projection wrapped in the uniform
|
|
2470
|
+
* `SliceResponse` envelope (`{ axis, rows, totals_unfiltered,
|
|
2471
|
+
* modality_warnings, study_id, modality }`). JSON mode is a thin
|
|
2472
|
+
* pass-through to jsonOutput with `preProjected: true` so the lean
|
|
2473
|
+
* transform doesn't strip our stable empties. Human mode pulls slices
|
|
2474
|
+
* out of `rows` and renders one section per slice plus a small ASCII
|
|
2475
|
+
* sentiment histogram.
|
|
2404
2476
|
*/
|
|
2405
2477
|
export function formatStudyResultsGroupBy(projection, kind, json) {
|
|
2406
2478
|
if (json) {
|
|
@@ -954,6 +954,12 @@ ish study results s-b2c --frame doesnotexist --json
|
|
|
954
954
|
# degraded captures (frame_version_id: null) back.
|
|
955
955
|
\`\`\`
|
|
956
956
|
|
|
957
|
+
Every \`--group-by <axis>\` call returns the same envelope:
|
|
958
|
+
\`{axis, rows, totals_unfiltered, modality_warnings, study_id, modality}\`.
|
|
959
|
+
The \`rows\` array holds axis-specific slice objects. The envelope is
|
|
960
|
+
uniform across all six axes — agents can code one shape and key on
|
|
961
|
+
\`axis\` / \`modality\` to dispatch on what's inside \`rows\`.
|
|
962
|
+
|
|
957
963
|
Rules to remember:
|
|
958
964
|
- **Filters compose with AND across flags; OR within \`--sentiment\`.**
|
|
959
965
|
\`--frame login --sentiment Frustrated,Confused\` keeps only login-frame
|
|
@@ -974,7 +980,8 @@ Rules to remember:
|
|
|
974
980
|
the filtered set. \`--transcript\` is single-participant and errors
|
|
975
981
|
(exit 2) when **any** filter or \`--group-by\` is set.
|
|
976
982
|
- Per-step output exposes \`participant_verdicts: [{participant_alias,
|
|
977
|
-
verdict, reason, evidence_interaction_ids}]\`
|
|
983
|
+
verdict, reason, evidence_interaction_ids}]\` on **each row of
|
|
984
|
+
\`rows[]\`** (one per \`(assignment, step)\` pair) — not
|
|
978
985
|
\`per_participant_verdicts\`. The verdict enum is \`passed\` /
|
|
979
986
|
\`inconclusive\` / \`failed\`.
|
|
980
987
|
|
|
@@ -1078,6 +1085,7 @@ table, projection shapes, and the defensive null-handling rules.
|
|
|
1078
1085
|
| Per-step pass/fail with reasons inline | \`study participant --json\` per participant + jq | \`ish study results <id> --step verify-email --group-by step --json\` |
|
|
1079
1086
|
| Frustrated reactions to one media segment | \`study results --json\` + jq | \`ish study results <id> --segment 3 --sentiment Frustrated --json\` |
|
|
1080
1087
|
| Sanity-check filter coverage | hand-count \`.participants\` vs total | \`--get totals_unfiltered.participant_count\` (set on every sliced envelope) |
|
|
1088
|
+
| Know the sliced-results envelope shape | guess per axis | \`{axis, rows[], totals_unfiltered, modality_warnings, study_id, modality}\` — every \`--group-by\` axis |
|
|
1081
1089
|
| Chat transcript for one participant (external_chatbot) | \`study participant --json\` + jq | \`ish study results <id> --transcript <participant_id> --json\` |
|
|
1082
1090
|
| Pair-mode conversation transcripts | \`study participant --json\` per participant | \`ish iteration get <iter-id> --json \\| jq '.conversations[]'\` |
|
|
1083
1091
|
| Participant headline only (no action timeline) | \`study participant --json\` + jq | \`ish study participant <id> --summary --json\` |
|
|
@@ -79,10 +79,16 @@ function normaliseIterations(rawIterations) {
|
|
|
79
79
|
}
|
|
80
80
|
return out;
|
|
81
81
|
}
|
|
82
|
-
/** Throw a ValidationError-tagged Error (the wrapper maps it to exit 2).
|
|
83
|
-
|
|
82
|
+
/** Throw a ValidationError-tagged Error (the wrapper maps it to exit 2).
|
|
83
|
+
* Optional `availableValues` rides along as a structured field in the
|
|
84
|
+
* emitted JSON envelope so agents can parse the recovery set without
|
|
85
|
+
* string-extracting it from the message (Pattern V from round 3). */
|
|
86
|
+
function validationError(message, availableValues) {
|
|
84
87
|
const err = new Error(message);
|
|
85
88
|
err.name = "ValidationError";
|
|
89
|
+
if (availableValues && availableValues.length > 0) {
|
|
90
|
+
err.available_values = availableValues;
|
|
91
|
+
}
|
|
86
92
|
return err;
|
|
87
93
|
}
|
|
88
94
|
/** Resolve `--frame <ref>` into a set of frame_version_ids.
|
|
@@ -143,19 +149,21 @@ function resolveFrameRef(ref, frames) {
|
|
|
143
149
|
const needle = ref.toLowerCase();
|
|
144
150
|
const candidates = frames.filter((f) => f.name && f.name.toLowerCase().includes(needle));
|
|
145
151
|
if (candidates.length === 0) {
|
|
146
|
-
|
|
152
|
+
// Pattern V: dedupe + cap the frame list so duplicates (workspace-side
|
|
153
|
+
// data hazards) don't double-list in the error prose. Carry the
|
|
154
|
+
// deduped list as a structured `available_values` field for agents.
|
|
155
|
+
const dedupedNames = Array.from(new Set(frames
|
|
147
156
|
.map((f) => f.name)
|
|
148
|
-
.filter((n) => typeof n === "string")
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
? ` Available frames: ${available.join(", ")}.`
|
|
157
|
+
.filter((n) => typeof n === "string"))).slice(0, 10);
|
|
158
|
+
const hint = dedupedNames.length > 0
|
|
159
|
+
? ` Available frames: ${dedupedNames.join(", ")}.`
|
|
152
160
|
: " This study has no named frames yet.";
|
|
153
|
-
throw validationError(`--frame "${ref}" matched no frames on this study.${hint}
|
|
161
|
+
throw validationError(`--frame "${ref}" matched no frames on this study.${hint}`, dedupedNames);
|
|
154
162
|
}
|
|
155
163
|
if (candidates.length > 1) {
|
|
156
|
-
const names = candidates.map((c) => c.name).filter((n) => !!n);
|
|
164
|
+
const names = Array.from(new Set(candidates.map((c) => c.name).filter((n) => !!n)));
|
|
157
165
|
throw validationError(`--frame "${ref}" is ambiguous — matched ${candidates.length} frames: ${names.join(", ")}. ` +
|
|
158
|
-
`Use a more specific substring, a full Frame UUID, or an \`f-…\` alias
|
|
166
|
+
`Use a more specific substring, a full Frame UUID, or an \`f-…\` alias.`, names);
|
|
159
167
|
}
|
|
160
168
|
indexFrame(candidates[0]);
|
|
161
169
|
return { matchedFrameVersionIds: matched, frameVersionLookup: lookup };
|
|
@@ -192,10 +200,23 @@ function resolveAssignmentRef(ref, assignments) {
|
|
|
192
200
|
return out;
|
|
193
201
|
}
|
|
194
202
|
/** Resolve `--iteration <ref>` to a single iteration_id, or null if no
|
|
195
|
-
* match (caller errors). UUID
|
|
203
|
+
* match (caller errors). Accepts UUID, iteration alias (`i-…`), or label. */
|
|
196
204
|
function resolveIterationRef(ref, iterations) {
|
|
197
|
-
|
|
198
|
-
|
|
205
|
+
// Pattern M: iteration aliases (`i-…`) are the canonical short ID
|
|
206
|
+
// everywhere else in the CLI; accept them here too. Try alias resolution
|
|
207
|
+
// first, then fall through to UUID-direct, then label match.
|
|
208
|
+
let candidate = ref;
|
|
209
|
+
if (ref.startsWith("i-")) {
|
|
210
|
+
try {
|
|
211
|
+
candidate = resolveId(ref);
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
// Unknown alias — let the downstream "matched no iterations" branch
|
|
215
|
+
// emit the labels hint; the resolveId error doesn't add value here.
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
if (UUID_RE.test(candidate)) {
|
|
219
|
+
const m = iterations.find((i) => i.id === candidate);
|
|
199
220
|
if (!m) {
|
|
200
221
|
throw validationError(`No iteration matches "${ref}" on this study.`);
|
|
201
222
|
}
|
|
@@ -383,7 +404,7 @@ export function applyResultsFilters(study, participants, rawFrames, filters) {
|
|
|
383
404
|
warnings.push(`--turn is only meaningful on chat studies; ignored on modality "${modality}".`);
|
|
384
405
|
}
|
|
385
406
|
if (filters.side !== undefined && !isParticipantPair(study)) {
|
|
386
|
-
warnings.push(`--side is only meaningful on participant_pair chat studies; ignored.`);
|
|
407
|
+
warnings.push(`--side is only meaningful on participant_pair chat studies; ignored on modality "${modality}".`);
|
|
387
408
|
}
|
|
388
409
|
// Normalise sentiment labels to lowercase up-front (case-insensitive).
|
|
389
410
|
const sentimentFilter = filters.sentiment
|
|
@@ -3,14 +3,10 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Each `buildStudyResultsPer<Kind>` consumes a `FilteredResults` (the output
|
|
5
5
|
* of `applyResultsFilters` in `study-results-filters.ts`) and returns a
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* `
|
|
9
|
-
*
|
|
10
|
-
* Per-iteration is the only projection that wraps a `{study, slices, ...}`
|
|
11
|
-
* envelope; the others return plain arrays of slice objects. The surface
|
|
12
|
-
* attaches `totals_unfiltered` and `warnings` from the same `FilteredResults`
|
|
13
|
-
* for the array projections.
|
|
6
|
+
* bare array of slice objects. The surface (`commands/study.ts`) wraps the
|
|
7
|
+
* array uniformly in a `SliceResponse` envelope alongside `totals_unfiltered`,
|
|
8
|
+
* `modality_warnings`, `study_id`, and `modality` before handing it off to
|
|
9
|
+
* `formatStudyResultsGroupBy` for JSON or human rendering.
|
|
14
10
|
*
|
|
15
11
|
* Conventions mirror `buildStudyResultsEnvelope` (`output.ts:1081`) and
|
|
16
12
|
* `buildStudyResultsSummary` (`output.ts:1292`):
|
|
@@ -23,18 +19,52 @@
|
|
|
23
19
|
* Has no IO and no console side-effects.
|
|
24
20
|
*/
|
|
25
21
|
import type { FilteredResults } from "./study-results-filters.js";
|
|
22
|
+
import type { StudyResultsGroupByKind } from "./output.js";
|
|
26
23
|
export type { FilteredResults } from "./study-results-filters.js";
|
|
27
24
|
/**
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
25
|
+
* Uniform envelope emitted for every `ish study results --group-by <axis>`
|
|
26
|
+
* call, mirroring the MCP backend's `SliceResponse[T]`. Six top-level keys,
|
|
27
|
+
* stable across all six axes — `rows` carries the bare slice array returned
|
|
28
|
+
* by the matching `buildStudyResultsPer<Kind>` builder.
|
|
29
|
+
*/
|
|
30
|
+
export interface SliceResponse<T> {
|
|
31
|
+
axis: StudyResultsGroupByKind;
|
|
32
|
+
rows: T[];
|
|
33
|
+
totals_unfiltered: {
|
|
34
|
+
participant_count: number;
|
|
35
|
+
interaction_count: number;
|
|
36
|
+
};
|
|
37
|
+
modality_warnings: string[];
|
|
38
|
+
study_id: string;
|
|
39
|
+
modality: string;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Wrap a bare projection array in the uniform `SliceResponse` envelope.
|
|
43
|
+
* The surface calls this once after dispatching to one of the six
|
|
44
|
+
* `buildStudyResultsPer<Kind>` builders, then hands the envelope to
|
|
45
|
+
* `formatStudyResultsGroupBy`.
|
|
46
|
+
*/
|
|
47
|
+
export declare function wrapSliceProjection<T>(filtered: FilteredResults, axis: StudyResultsGroupByKind, rows: T[], studyId: string, modality: string): SliceResponse<T>;
|
|
48
|
+
interface IterationSlice {
|
|
49
|
+
iteration_id: string;
|
|
50
|
+
iteration_label: string | null;
|
|
51
|
+
participant_count: number;
|
|
52
|
+
interaction_count: number;
|
|
53
|
+
sentiment: Record<string, number>;
|
|
54
|
+
sample_comments: string[];
|
|
55
|
+
top_actions: Array<{
|
|
56
|
+
action_type: string;
|
|
57
|
+
count: number;
|
|
58
|
+
}>;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* `--group-by iteration` — one slice per declared iteration, in the same
|
|
62
|
+
* order as `ish study get`. Iterations with zero surviving participants
|
|
63
|
+
* still appear with `participant_count: 0` so the consumer sees the full
|
|
64
|
+
* matrix at stable size. Returns a bare array; the surface wraps it in
|
|
65
|
+
* the uniform `SliceResponse` envelope.
|
|
36
66
|
*/
|
|
37
|
-
export declare function buildStudyResultsPerIteration(filtered: FilteredResults):
|
|
67
|
+
export declare function buildStudyResultsPerIteration(filtered: FilteredResults): IterationSlice[];
|
|
38
68
|
interface FrameSlice {
|
|
39
69
|
frame_id: string;
|
|
40
70
|
frame_label: string | null;
|
|
@@ -3,14 +3,10 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Each `buildStudyResultsPer<Kind>` consumes a `FilteredResults` (the output
|
|
5
5
|
* of `applyResultsFilters` in `study-results-filters.ts`) and returns a
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* `
|
|
9
|
-
*
|
|
10
|
-
* Per-iteration is the only projection that wraps a `{study, slices, ...}`
|
|
11
|
-
* envelope; the others return plain arrays of slice objects. The surface
|
|
12
|
-
* attaches `totals_unfiltered` and `warnings` from the same `FilteredResults`
|
|
13
|
-
* for the array projections.
|
|
6
|
+
* bare array of slice objects. The surface (`commands/study.ts`) wraps the
|
|
7
|
+
* array uniformly in a `SliceResponse` envelope alongside `totals_unfiltered`,
|
|
8
|
+
* `modality_warnings`, `study_id`, and `modality` before handing it off to
|
|
9
|
+
* `formatStudyResultsGroupBy` for JSON or human rendering.
|
|
14
10
|
*
|
|
15
11
|
* Conventions mirror `buildStudyResultsEnvelope` (`output.ts:1081`) and
|
|
16
12
|
* `buildStudyResultsSummary` (`output.ts:1292`):
|
|
@@ -23,6 +19,22 @@
|
|
|
23
19
|
* Has no IO and no console side-effects.
|
|
24
20
|
*/
|
|
25
21
|
import { deterministicAlias, ALIAS_PREFIX } from "./alias-store.js";
|
|
22
|
+
/**
|
|
23
|
+
* Wrap a bare projection array in the uniform `SliceResponse` envelope.
|
|
24
|
+
* The surface calls this once after dispatching to one of the six
|
|
25
|
+
* `buildStudyResultsPer<Kind>` builders, then hands the envelope to
|
|
26
|
+
* `formatStudyResultsGroupBy`.
|
|
27
|
+
*/
|
|
28
|
+
export function wrapSliceProjection(filtered, axis, rows, studyId, modality) {
|
|
29
|
+
return {
|
|
30
|
+
axis,
|
|
31
|
+
rows,
|
|
32
|
+
totals_unfiltered: filtered.totals_unfiltered,
|
|
33
|
+
modality_warnings: filtered.warnings,
|
|
34
|
+
study_id: deterministicAlias(ALIAS_PREFIX.study, studyId),
|
|
35
|
+
modality,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
26
38
|
const SAMPLE_COMMENT_CAP = 5;
|
|
27
39
|
const SAMPLE_COMMENT_MAX_LEN = 200;
|
|
28
40
|
const PARTICIPANT_ALIAS_CAP = 10;
|
|
@@ -44,21 +56,24 @@ function asString(v) {
|
|
|
44
56
|
function truncate(str, maxLen) {
|
|
45
57
|
if (str.length <= maxLen)
|
|
46
58
|
return str;
|
|
47
|
-
|
|
59
|
+
// Pattern H: trim at the last word boundary before the cap so quoted
|
|
60
|
+
// `sample_comments` / `sample_replies` don't end mid-word ("…koncentrera").
|
|
61
|
+
// Prefer the last sentence terminator if one exists in the kept range;
|
|
62
|
+
// otherwise fall back to the last whitespace. If neither is found
|
|
63
|
+
// (a single long unbroken token), hard-cut at the cap.
|
|
64
|
+
const head = str.slice(0, maxLen - 3);
|
|
65
|
+
const sentenceBreak = Math.max(head.lastIndexOf(". "), head.lastIndexOf("! "), head.lastIndexOf("? "));
|
|
66
|
+
if (sentenceBreak >= maxLen / 2)
|
|
67
|
+
return head.slice(0, sentenceBreak + 1) + "…";
|
|
68
|
+
const spaceBreak = head.lastIndexOf(" ");
|
|
69
|
+
if (spaceBreak >= maxLen / 2)
|
|
70
|
+
return head.slice(0, spaceBreak) + " …";
|
|
71
|
+
return head + "…";
|
|
48
72
|
}
|
|
49
73
|
function participantAlias(participant) {
|
|
50
74
|
const id = asString(participant.id);
|
|
51
75
|
return id ? deterministicAlias(ALIAS_PREFIX.participant, id) : null;
|
|
52
76
|
}
|
|
53
|
-
function studyHeader(filtered) {
|
|
54
|
-
const study = filtered.study;
|
|
55
|
-
const id = asString(study.id);
|
|
56
|
-
return {
|
|
57
|
-
alias: id ? deterministicAlias(ALIAS_PREFIX.study, id) : null,
|
|
58
|
-
name: asString(study.name) ?? null,
|
|
59
|
-
modality: asString(study.modality) ?? null,
|
|
60
|
-
};
|
|
61
|
-
}
|
|
62
77
|
function readSentimentLabel(interaction) {
|
|
63
78
|
const s = asRecord(interaction.sentiment);
|
|
64
79
|
return s ? asString(s.label) : null;
|
|
@@ -111,14 +126,11 @@ function collectParticipantAlias(bucket, seen, participant) {
|
|
|
111
126
|
bucket.push(alias);
|
|
112
127
|
}
|
|
113
128
|
/**
|
|
114
|
-
* `--group-by iteration` — one slice per iteration
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
* with `totals_unfiltered` + `warnings` because per-iteration is the
|
|
120
|
-
* "default agent slice" — the one most likely to be piped directly without
|
|
121
|
-
* the surface re-wrapping.
|
|
129
|
+
* `--group-by iteration` — one slice per declared iteration, in the same
|
|
130
|
+
* order as `ish study get`. Iterations with zero surviving participants
|
|
131
|
+
* still appear with `participant_count: 0` so the consumer sees the full
|
|
132
|
+
* matrix at stable size. Returns a bare array; the surface wraps it in
|
|
133
|
+
* the uniform `SliceResponse` envelope.
|
|
122
134
|
*/
|
|
123
135
|
export function buildStudyResultsPerIteration(filtered) {
|
|
124
136
|
const iterations = asArray(filtered.study.iterations);
|
|
@@ -176,16 +188,7 @@ export function buildStudyResultsPerIteration(filtered) {
|
|
|
176
188
|
.map(([action_type, count]) => ({ action_type, count }));
|
|
177
189
|
byIteration.get(iterId).top_actions = rows;
|
|
178
190
|
}
|
|
179
|
-
|
|
180
|
-
// matrix at stable size. Iterations with zero surviving rows still appear
|
|
181
|
-
// with `participant_count: 0` — useful for "matched X / Y" framing.
|
|
182
|
-
const slices = order.map((o) => byIteration.get(o.id));
|
|
183
|
-
return {
|
|
184
|
-
study: studyHeader(filtered),
|
|
185
|
-
slices,
|
|
186
|
-
totals_unfiltered: filtered.totals_unfiltered,
|
|
187
|
-
warnings: filtered.warnings,
|
|
188
|
-
};
|
|
191
|
+
return order.map((o) => byIteration.get(o.id));
|
|
189
192
|
}
|
|
190
193
|
/**
|
|
191
194
|
* `--group-by frame` — one slice per Frame that had a surviving interaction.
|