@ishlabs/cli 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/chat.js +2 -2
- package/dist/commands/config.js +17 -3
- package/dist/commands/source.js +1 -1
- package/dist/commands/study-analyze.js +15 -2
- package/dist/commands/study-participant.js +19 -0
- package/dist/commands/study.js +313 -14
- package/dist/lib/alias-store.d.ts +1 -0
- package/dist/lib/alias-store.js +2 -0
- package/dist/lib/command-helpers.js +4 -3
- package/dist/lib/docs.js +232 -15
- package/dist/lib/output.d.ts +24 -1
- package/dist/lib/output.js +290 -2
- package/dist/lib/skill-content.js +76 -0
- package/dist/lib/study-participants.d.ts +13 -0
- package/dist/lib/study-participants.js +13 -0
- package/dist/lib/study-results-filters.d.ts +91 -0
- package/dist/lib/study-results-filters.js +559 -0
- package/dist/lib/study-results-projections.d.ts +152 -0
- package/dist/lib/study-results-projections.js +580 -0
- package/package.json +1 -1
package/dist/commands/chat.js
CHANGED
|
@@ -115,7 +115,7 @@ Examples:
|
|
|
115
115
|
});
|
|
116
116
|
parent
|
|
117
117
|
.command("create")
|
|
118
|
-
.description("Create a chatbot endpoint from a config
|
|
118
|
+
.description("Create a chatbot endpoint from a hand-written ChatbotEndpointConfig JSON (advanced — use `chat endpoint init` to infer the config from a curl sample, JSON, or template).")
|
|
119
119
|
.requiredOption("--endpoint-config <file>", 'Path to JSON file (or "-" for stdin)')
|
|
120
120
|
.option("--name <name>", "Override the name from the config file")
|
|
121
121
|
.option("--workspace <id>", "Workspace ID")
|
|
@@ -342,7 +342,7 @@ endpoint, apply the override, and PUT the merged result. Field flags win over
|
|
|
342
342
|
function attachChatEndpointInit(parent) {
|
|
343
343
|
parent
|
|
344
344
|
.command("init")
|
|
345
|
-
.description("Author an endpoint from a curl/JSON sample via test-and-map, or from a known-good template")
|
|
345
|
+
.description("Author an endpoint from a curl/JSON sample via test-and-map, or from a known-good template (recommended for most users — use `chat endpoint create` only when you already have a hand-written ChatbotEndpointConfig).")
|
|
346
346
|
.option("--from-curl <file>", 'Path to a curl example file (or "-" for stdin)')
|
|
347
347
|
.option("--from-json <file>", 'Path to a JSON request/response sample (or "-" for stdin)')
|
|
348
348
|
.option("--template <name>", `Start from a known-good template (one of: ${TEMPLATE_NAMES.join(", ")})`)
|
package/dist/commands/config.js
CHANGED
|
@@ -78,11 +78,25 @@ Run \`ish docs overview\` for the full mental model.`);
|
|
|
78
78
|
});
|
|
79
79
|
config
|
|
80
80
|
.command("schema")
|
|
81
|
-
.description("Get simulation config schema with defaults")
|
|
81
|
+
.description("Get simulation config schema with defaults (admin-only — non-admin accounts: ask an admin to share an existing config ID and pass it via `ish study run --config <id>`).")
|
|
82
82
|
.action(async (_opts, cmd) => {
|
|
83
83
|
await withClient(cmd, async (client, globals) => {
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
try {
|
|
85
|
+
const data = await client.get("/dev/simulation-configs/schema");
|
|
86
|
+
output(data, globals.json);
|
|
87
|
+
}
|
|
88
|
+
catch (err) {
|
|
89
|
+
// Pattern Z: re-throw with a hint pointing non-admin agents at the
|
|
90
|
+
// workaround (use a shared config ID via `study run --config`).
|
|
91
|
+
if (err instanceof Error && err.status === 403) {
|
|
92
|
+
const tagged = err;
|
|
93
|
+
const extra = "Non-admin accounts cannot introspect the simulation-config schema. To still use a config, ask an admin to share an existing config ID and pass it via `ish study run --config <id>` (`config --help` for the full workflow).";
|
|
94
|
+
tagged.suggestions = Array.isArray(tagged.suggestions)
|
|
95
|
+
? [...tagged.suggestions, extra]
|
|
96
|
+
: [extra];
|
|
97
|
+
}
|
|
98
|
+
throw err;
|
|
99
|
+
}
|
|
86
100
|
});
|
|
87
101
|
});
|
|
88
102
|
config
|
package/dist/commands/source.js
CHANGED
|
@@ -27,7 +27,7 @@ same attachment across multiple generation runs; otherwise pass a local path dir
|
|
|
27
27
|
to \`person generate --source\` and it auto-uploads.
|
|
28
28
|
|
|
29
29
|
Concept pages: ish docs get-page concepts/source
|
|
30
|
-
ish docs get-page concepts/
|
|
30
|
+
ish docs get-page concepts/person`);
|
|
31
31
|
source
|
|
32
32
|
.command("upload")
|
|
33
33
|
.description("Upload a file as a participant attachment and wait for processing")
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* about latency than load).
|
|
15
15
|
*/
|
|
16
16
|
import { withClient, resolveStudy, parseWaitTimeout } from "../lib/command-helpers.js";
|
|
17
|
-
import { resolveId } from "../lib/alias-store.js";
|
|
17
|
+
import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
18
18
|
import { output, printTable } from "../lib/output.js";
|
|
19
19
|
import { WaitTimeoutError } from "./study-run.js";
|
|
20
20
|
const POLL_INTERVAL_MS = 5_000;
|
|
@@ -160,11 +160,24 @@ Trigger a new run with \`ish study analyze --wait\`.`)
|
|
|
160
160
|
const history = await client.get(`/studies/${studyId}/results`);
|
|
161
161
|
const latest = history[0] ?? null;
|
|
162
162
|
if (globals.json) {
|
|
163
|
+
// Pattern K: never emit empty stdout. When no analyses have run,
|
|
164
|
+
// ship a stable envelope with a hint pointing at the verb that
|
|
165
|
+
// populates it. Mirrors the `study results` empty-envelope contract.
|
|
166
|
+
if (!latest) {
|
|
167
|
+
const studyAlias = tagAlias(ALIAS_PREFIX.study, studyId);
|
|
168
|
+
output({
|
|
169
|
+
latest: null,
|
|
170
|
+
history: [],
|
|
171
|
+
hint: `No analyses run yet. Trigger one with \`ish study analyze ${studyAlias}\`.`,
|
|
172
|
+
}, true, { preProjected: true });
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
163
175
|
output({ latest, history }, true);
|
|
164
176
|
return;
|
|
165
177
|
}
|
|
166
178
|
if (!latest) {
|
|
167
|
-
|
|
179
|
+
const studyAlias = tagAlias(ALIAS_PREFIX.study, studyId);
|
|
180
|
+
console.log(`No analysis runs yet. Trigger one with \`ish study analyze ${studyAlias}\`.`);
|
|
168
181
|
return;
|
|
169
182
|
}
|
|
170
183
|
if (opts.all) {
|
|
@@ -76,6 +76,25 @@ Tips:
|
|
|
76
76
|
const result = data;
|
|
77
77
|
if (result.id)
|
|
78
78
|
result.alias = tagAlias(ALIAS_PREFIX.participant, String(result.id));
|
|
79
|
+
// Pattern L: enrich with parent-graph aliases so agents can traverse
|
|
80
|
+
// from a participant straight to its study without hopping through
|
|
81
|
+
// `iteration get`. The participant response carries `iteration_id` but
|
|
82
|
+
// not `study_id`; one iteration fetch supplies both.
|
|
83
|
+
const iterationId = typeof result.iteration_id === "string" ? result.iteration_id : null;
|
|
84
|
+
if (iterationId) {
|
|
85
|
+
result.iteration_alias = tagAlias(ALIAS_PREFIX.iteration, iterationId);
|
|
86
|
+
try {
|
|
87
|
+
const iter = await client.get(`/iterations/${iterationId}`);
|
|
88
|
+
if (typeof iter.study_id === "string") {
|
|
89
|
+
result.study_id = iter.study_id;
|
|
90
|
+
result.study_alias = tagAlias(ALIAS_PREFIX.study, iter.study_id);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// Best-effort enrichment; if the iteration fetch fails (deleted,
|
|
95
|
+
// permission), keep going with the alias we already injected.
|
|
96
|
+
}
|
|
97
|
+
}
|
|
79
98
|
if (opts.summary) {
|
|
80
99
|
output(buildParticipantSummary(result), globals.json, { preProjected: true });
|
|
81
100
|
return;
|
package/dist/commands/study.js
CHANGED
|
@@ -3,10 +3,12 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { readFileSync } from "node:fs";
|
|
5
5
|
import { Option } from "commander";
|
|
6
|
-
import { withClient, getWebUrl, terminalLink, resolveWorkspace, confirmDestructive, readFileOrStdin } from "../lib/command-helpers.js";
|
|
6
|
+
import { withClient, getWebUrl, terminalLink, resolveWorkspace, confirmDestructive, readFileOrStdin, collectIds } from "../lib/command-helpers.js";
|
|
7
7
|
import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
8
8
|
import { loadConfig, saveConfig } from "../config.js";
|
|
9
|
-
import { formatStudyList, formatStudyDetail, formatStudyResults, buildStudyResultsSummary, buildChatTranscript, output, ValidationError, } from "../lib/output.js";
|
|
9
|
+
import { formatStudyList, formatStudyDetail, formatStudyResults, buildStudyResultsEnvelope, buildStudyResultsSummary, buildChatTranscript, formatStudyResultsGroupBy, output, ValidationError, } from "../lib/output.js";
|
|
10
|
+
import { applyResultsFilters } from "../lib/study-results-filters.js";
|
|
11
|
+
import { buildStudyResultsPerIteration, buildStudyResultsPerFrame, buildStudyResultsPerSegment, buildStudyResultsPerTurn, buildStudyResultsPerAssignment, buildStudyResultsPerStep, wrapSliceProjection, } from "../lib/study-results-projections.js";
|
|
10
12
|
import { VALID_CONTENT_TYPES } from "../lib/types.js";
|
|
11
13
|
import { fetchStudyParticipants } from "../lib/study-participants.js";
|
|
12
14
|
import { parseAssignment, loadAssignmentsFile, validateAssignmentsArray, parseQuestion } from "../lib/study-inputs.js";
|
|
@@ -609,7 +611,7 @@ Next: configure a run with \`ish iteration create --study <id>\`,
|
|
|
609
611
|
});
|
|
610
612
|
study
|
|
611
613
|
.command("get")
|
|
612
|
-
.description("Get study
|
|
614
|
+
.description("Get the full study payload — iterations (with run details), assignments, interview questions, sentiment + status counts. Accepts multiple IDs for batched lookup. NOTE: this is the full payload, not a roll-up — for a compact cross-study comparison view use `study results <id> --summary`.")
|
|
613
615
|
.argument("<ids...>", "Study ID(s) — one or more aliases/UUIDs (space- or comma-separated)")
|
|
614
616
|
.addHelpText("after", `
|
|
615
617
|
Examples:
|
|
@@ -634,6 +636,18 @@ list table layout in human mode.`)
|
|
|
634
636
|
const result = data;
|
|
635
637
|
if (result.id)
|
|
636
638
|
result.alias = tagAlias(ALIAS_PREFIX.study, String(result.id));
|
|
639
|
+
// Pattern I-r3-1: inline iterations carry only label/name/details
|
|
640
|
+
// from the wire; tag each with its `alias` (computed from id via
|
|
641
|
+
// the local alias-store) so agents can drill from `study get` into
|
|
642
|
+
// `iteration get <alias>` / `study results --iteration <alias>`
|
|
643
|
+
// without a separate `iteration list` round-trip.
|
|
644
|
+
if (Array.isArray(result.iterations)) {
|
|
645
|
+
for (const iter of result.iterations) {
|
|
646
|
+
if (typeof iter.id === "string") {
|
|
647
|
+
iter.alias = tagAlias(ALIAS_PREFIX.iteration, iter.id);
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
}
|
|
637
651
|
if (data.product_id) {
|
|
638
652
|
result.url = getWebUrl(globals, `/${data.product_id}/${rid}/overview`);
|
|
639
653
|
}
|
|
@@ -653,6 +667,13 @@ list table layout in human mode.`)
|
|
|
653
667
|
const r = data;
|
|
654
668
|
if (r.id)
|
|
655
669
|
r.alias = tagAlias(ALIAS_PREFIX.study, String(r.id));
|
|
670
|
+
if (Array.isArray(r.iterations)) {
|
|
671
|
+
for (const iter of r.iterations) {
|
|
672
|
+
if (typeof iter.id === "string") {
|
|
673
|
+
iter.alias = tagAlias(ALIAS_PREFIX.iteration, iter.id);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
}
|
|
656
677
|
if (data.product_id) {
|
|
657
678
|
r.url = getWebUrl(globals, `/${data.product_id}/${rid}/overview`);
|
|
658
679
|
}
|
|
@@ -669,22 +690,41 @@ list table layout in human mode.`)
|
|
|
669
690
|
});
|
|
670
691
|
study
|
|
671
692
|
.command("results")
|
|
672
|
-
.description("View aggregated results: participant counts, sentiment, interview answers. Returns a stable envelope with empty fields when no runs have completed.")
|
|
693
|
+
.description("View aggregated results: participant counts, sentiment, interview answers. Returns a stable envelope with empty fields when no runs have completed. Slice with filter flags (--frame [interactive], --segment [video/audio/text/document], --turn [chat], --side [chat participant_pair], --assignment, --step, --sentiment, --actor, --iteration, --participant) or project with --group-by <axis> (iteration | frame [interactive] | segment [media] | turn [chat] | assignment | step).")
|
|
673
694
|
.argument("<id>", "Study ID")
|
|
674
695
|
.option("--workspace <id>", "Workspace ID; accepted for consistency (workspace is inferred from the study)")
|
|
675
|
-
.option("--summary", "Lean summary projection: counts + sentiment + per-participant {alias, status, sentiment, comment}. Drops interview_answers + per-interaction breakdowns.")
|
|
696
|
+
.option("--summary", "Lean summary projection: counts + sentiment + per-participant {alias, status, sentiment, comment}. Drops interview_answers + per-interaction breakdowns. Composes with filters: `--summary --frame login` narrows the summary to the login-screen interactions.")
|
|
676
697
|
// PC-N4: agents reach for `--summarize` (verb) by analogy with the MCP
|
|
677
698
|
// `summarize` action; accept it as a hidden alias of --summary so the
|
|
678
699
|
// canonical flag stays the documented one but the muscle-memory variant
|
|
679
700
|
// works without a round-trip.
|
|
680
701
|
.addOption(new Option("--summarize", "Hidden alias for --summary").hideHelp())
|
|
681
|
-
.option("--transcript <participant_id>", "Chat transcript projection for one participant: flat role/text/turn-index array (chat-modality only). Mirrors the MCP `get_chat_transcript` shape.")
|
|
702
|
+
.option("--transcript <participant_id>", "Chat transcript projection for one participant: flat role/text/turn-index array (chat-modality only). Mirrors the MCP `get_chat_transcript` shape. Cannot combine with filters or --group-by (transcript is a single-participant projection).")
|
|
703
|
+
// --- Slice / projection flags (T5) ---
|
|
704
|
+
.option("--frame <ref>", "Filter to interactions whose Frame name contains <ref> (case-insensitive), or whose Frame UUID / `f-…` alias / frame_version_id matches. Interactive only — warned and ignored on other modalities.")
|
|
705
|
+
.option("--segment <ref>", "Filter media studies (video/audio/text/document) by segment index (integer) or segment label (substring). Image and other modalities: warned and ignored.")
|
|
706
|
+
.option("--turn <n>", "Filter chat interactions to a single `actions[0].data.turn_index`. Non-chat modalities: warned and ignored.")
|
|
707
|
+
.option("--side <a|b>", "Filter participant_pair chat interactions by assignment side. Other modalities: warned and ignored.")
|
|
708
|
+
.option("--assignment <ref>", "Filter to a single assignment by UUID or name (substring, case-insensitive).")
|
|
709
|
+
.option("--step <ref>", "Filter `participant_assignments[].step_results[]` to a single step by step-id or name (substring). Pair with --include-evidence to also drop non-evidence interactions.")
|
|
710
|
+
.option("--sentiment <labels>", "Filter to interactions whose sentiment.label is in the comma-separated list (case-insensitive; repeatable). Drops interactions whose sentiment is null. A participant is kept when at least one of their interactions matches, even if their aggregate session sentiment is null (e.g. failed runs with a pre-error matching interaction).", collectIds, [])
|
|
711
|
+
.option("--actor <actor>", "Filter to interactions whose actor is `ai`, `human`, or `user` (case-insensitive).")
|
|
712
|
+
.option("--iteration <ref>", "Restrict to a single iteration by UUID or label.")
|
|
713
|
+
.option("--participant <ref>", "Restrict to a single participant by UUID or `pt-…` alias.")
|
|
714
|
+
.option("--include-unmatched", "When --frame is set, keep interactions with null frame_version_id under a synthetic `_unmatched` bucket instead of dropping them.")
|
|
715
|
+
.option("--include-evidence", "When --step is set, also drop interactions not listed in any surviving step_results[].evidence_interaction_ids[].")
|
|
716
|
+
.option("--group-by <axis>", "Project results into per-axis slices: iteration | frame | segment | turn | assignment | step. Mutually exclusive with --summary and --transcript.")
|
|
682
717
|
.addHelpText("after", `
|
|
683
718
|
Examples:
|
|
684
719
|
$ ish study results <id>
|
|
685
720
|
$ ish study results <id> --json
|
|
686
721
|
$ ish study results <id> --summary --json
|
|
687
722
|
$ ish study results <id> --transcript pt-d4e --json
|
|
723
|
+
# Slice (filters compose: AND across flags, OR within --sentiment)
|
|
724
|
+
$ ish study results <id> --frame login --group-by iteration
|
|
725
|
+
$ ish study results <id> --segment 3 --sentiment Frustrated
|
|
726
|
+
$ ish study results <id> --assignment "Sign up" --step verify-email --group-by step
|
|
727
|
+
$ ish study results <id> --side a --turn 4
|
|
688
728
|
|
|
689
729
|
Default --json envelope (M10: per-answer sentiment now included):
|
|
690
730
|
{
|
|
@@ -692,6 +732,7 @@ Default --json envelope (M10: per-answer sentiment now included):
|
|
|
692
732
|
"participant_count": 12,
|
|
693
733
|
"completed_count": 8,
|
|
694
734
|
"failed_count": 0,
|
|
735
|
+
"participant_status_counts": { "completed": 8, "running": 3, "draft": 1 },
|
|
695
736
|
"sentiment": { "counts": { "Satisfied": 5, "Frustrated": 2 }, "total": 7 },
|
|
696
737
|
"interview_answers": [
|
|
697
738
|
{ "question": "...", "type": "text",
|
|
@@ -707,6 +748,17 @@ Default --json envelope (M10: per-answer sentiment now included):
|
|
|
707
748
|
]
|
|
708
749
|
}
|
|
709
750
|
|
|
751
|
+
When any filter flag is passed, the envelope gains a \`totals_unfiltered\` field
|
|
752
|
+
({ participant_count, interaction_count }) so callers can sanity-check coverage
|
|
753
|
+
("matched 12 / 80 participants"). A zero-match filter returns the stable
|
|
754
|
+
envelope with participant_count=0 and exit code 0 (not 4).
|
|
755
|
+
|
|
756
|
+
Filtered count semantics: \`participant_count\` is the matched-set total (every
|
|
757
|
+
participant whose interactions matched the filter — including running and
|
|
758
|
+
failed). The unfiltered denominator is \`totals_unfiltered.participant_count\`,
|
|
759
|
+
and the same envelope still carries \`completed_count\` / \`failed_count\` so
|
|
760
|
+
agents can compute "completed AND matched" without a second call.
|
|
761
|
+
|
|
710
762
|
--summary projection (M2-friction-7: drops the interview_answers payload):
|
|
711
763
|
{ study, participant_count, completed_count, failed_count, sentiment, participants: [...] }
|
|
712
764
|
|
|
@@ -723,6 +775,35 @@ Default --json envelope (M10: per-answer sentiment now included):
|
|
|
723
775
|
"participant_summary": { "comment": "...", "sentiment": {...} }
|
|
724
776
|
}
|
|
725
777
|
|
|
778
|
+
--group-by projections share one envelope (uniform across all six axes):
|
|
779
|
+
{ axis, rows, totals_unfiltered, modality_warnings, study_id, modality }
|
|
780
|
+
|
|
781
|
+
axis echoes the requested axis (iteration|frame|segment|turn|assignment|step)
|
|
782
|
+
study_id the \`s-…\` alias
|
|
783
|
+
modality the study's modality
|
|
784
|
+
totals_unfiltered { participant_count, interaction_count } — pre-filter counts
|
|
785
|
+
modality_warnings any filter-flag mismatches (e.g. --turn on a non-chat study)
|
|
786
|
+
|
|
787
|
+
Per-axis row shape (one element of \`rows[]\`):
|
|
788
|
+
|
|
789
|
+
--group-by iteration:
|
|
790
|
+
{ iteration_id, iteration_label, participant_count, interaction_count, sentiment, sample_comments, top_actions }
|
|
791
|
+
|
|
792
|
+
--group-by frame (interactive only):
|
|
793
|
+
{ frame_id, frame_label, interaction_count, sentiment_histogram, sample_comments, participant_aliases }
|
|
794
|
+
|
|
795
|
+
--group-by segment (video/audio/text/document):
|
|
796
|
+
{ segment_index, segment_label, interaction_count, sentiment_histogram, engagement_histogram, sample_comments }
|
|
797
|
+
|
|
798
|
+
--group-by turn (chat only):
|
|
799
|
+
{ turn_index, interaction_count, sentiment_histogram, sample_replies, failures }
|
|
800
|
+
|
|
801
|
+
--group-by assignment:
|
|
802
|
+
{ assignment_id, assignment_name, interaction_count, sentiment_histogram, step_completion }
|
|
803
|
+
|
|
804
|
+
--group-by step:
|
|
805
|
+
{ assignment_id, assignment_name, step_id, step_name, total, passed, inconclusive, failed, rate, participant_verdicts: [{ participant_alias, verdict, reason, evidence_interaction_ids }] }
|
|
806
|
+
|
|
726
807
|
Tips:
|
|
727
808
|
Use \`--get <path>\` for a single value (e.g. \`--get participant_count\`),
|
|
728
809
|
\`--fields a,b,c\` to project the JSON output further.
|
|
@@ -741,6 +822,7 @@ Common --get paths (default envelope):
|
|
|
741
822
|
--get interview_answers # full per-question payload
|
|
742
823
|
--get interview_answers.0.question # text of the first question
|
|
743
824
|
--get interview_answers.0.answers.0.answer # first answer to the first question
|
|
825
|
+
--get totals_unfiltered.participant_count # pre-filter participant count (when slicing)
|
|
744
826
|
|
|
745
827
|
Common --get paths (--transcript <participant_id> envelope):
|
|
746
828
|
--get transcript # full role/text/turn array
|
|
@@ -749,6 +831,25 @@ Common --get paths (--transcript <participant_id> envelope):
|
|
|
749
831
|
--get participant_summary.sentiment # aggregate sentiment map
|
|
750
832
|
--get unique_bot_replies # bot-side message count
|
|
751
833
|
|
|
834
|
+
Common --get paths (--group-by envelope — uniform across axes):
|
|
835
|
+
--get axis # echoes the requested axis
|
|
836
|
+
--get study_id # s-… alias
|
|
837
|
+
--get modality # study's modality
|
|
838
|
+
--get modality_warnings # filter-flag mismatches (one warning per line)
|
|
839
|
+
--get totals_unfiltered.participant_count # pre-filter participant count
|
|
840
|
+
--get totals_unfiltered.interaction_count # pre-filter interaction count
|
|
841
|
+
|
|
842
|
+
--get rows.iteration_label # per-iteration: one label per line
|
|
843
|
+
--get rows.0.participant_count # per-iteration: first row's count
|
|
844
|
+
--get rows.0.frame_label # per-frame: first row's label
|
|
845
|
+
--get rows.0.sentiment_histogram # per-frame/segment/turn: first row's sentiment map
|
|
846
|
+
--get rows.0.segment_index # per-segment: first row's index
|
|
847
|
+
--get rows.0.turn_index # per-turn: first row's index
|
|
848
|
+
--get rows.0.assignment_name # per-assignment/step: first row's assignment
|
|
849
|
+
--get rows.0.step_name # per-step: first row's step
|
|
850
|
+
--get rows.0.rate # per-step: first row's pass-rate
|
|
851
|
+
--get rows.0.participant_verdicts.verdict # per-step: verdict per participant
|
|
852
|
+
|
|
752
853
|
When no runs have completed, the default envelope is returned with zero counts and empty arrays.`)
|
|
753
854
|
.action(async (id, opts, cmd) => {
|
|
754
855
|
await withClient(cmd, async (client, globals) => {
|
|
@@ -756,10 +857,76 @@ When no runs have completed, the default envelope is returned with zero counts a
|
|
|
756
857
|
// into a single boolean before validation so the rest of the
|
|
757
858
|
// handler reads only `summary`.
|
|
758
859
|
const wantsSummary = !!(opts.summary || opts.summarize);
|
|
860
|
+
// T5: detect whether any filter flag was passed. Interaction-level
|
|
861
|
+
// and participant-level flags both count — they all narrow the
|
|
862
|
+
// result set. `--include-unmatched`/`--include-evidence` are
|
|
863
|
+
// modifiers that only make sense alongside --frame/--step but
|
|
864
|
+
// count as "filter intent" for the transcript/conflict check.
|
|
865
|
+
const hasFilter = opts.frame !== undefined ||
|
|
866
|
+
opts.segment !== undefined ||
|
|
867
|
+
opts.turn !== undefined ||
|
|
868
|
+
opts.side !== undefined ||
|
|
869
|
+
opts.assignment !== undefined ||
|
|
870
|
+
opts.step !== undefined ||
|
|
871
|
+
(opts.sentiment !== undefined && opts.sentiment.length > 0) ||
|
|
872
|
+
opts.actor !== undefined ||
|
|
873
|
+
opts.iteration !== undefined ||
|
|
874
|
+
opts.participant !== undefined ||
|
|
875
|
+
opts.includeUnmatched === true ||
|
|
876
|
+
opts.includeEvidence === true;
|
|
877
|
+
const hasGroupBy = opts.groupBy !== undefined;
|
|
878
|
+
// --- Conflict validation (no IO yet) ---
|
|
759
879
|
if (wantsSummary && opts.transcript) {
|
|
760
880
|
throw new ValidationError("Pass only one of: --summary, --transcript.", ["--summary", "--transcript"]);
|
|
761
881
|
}
|
|
882
|
+
if (opts.transcript && (hasFilter || hasGroupBy)) {
|
|
883
|
+
// --transcript is a single-participant chat projection — slicing
|
|
884
|
+
// doesn't make sense.
|
|
885
|
+
throw new ValidationError("--transcript is a single-participant projection; cannot combine with filter flags or --group-by.", ["--transcript"]);
|
|
886
|
+
}
|
|
887
|
+
if (wantsSummary && hasGroupBy) {
|
|
888
|
+
throw new ValidationError("Pass only one of: --summary, --group-by.", ["--summary", "--group-by"]);
|
|
889
|
+
}
|
|
890
|
+
// --side validation: must be exactly "a" or "b" (case-insensitive).
|
|
891
|
+
const sideNormalised = opts.side ? opts.side.toLowerCase() : undefined;
|
|
892
|
+
if (sideNormalised !== undefined && sideNormalised !== "a" && sideNormalised !== "b") {
|
|
893
|
+
throw new ValidationError(`--side must be "a" or "b", got "${opts.side}".`, ["a", "b"]);
|
|
894
|
+
}
|
|
895
|
+
// --actor validation: must be one of ai|human|user (case-insensitive).
|
|
896
|
+
const actorNormalised = opts.actor ? opts.actor.toLowerCase() : undefined;
|
|
897
|
+
if (actorNormalised !== undefined &&
|
|
898
|
+
actorNormalised !== "ai" &&
|
|
899
|
+
actorNormalised !== "human" &&
|
|
900
|
+
actorNormalised !== "user") {
|
|
901
|
+
throw new ValidationError(`--actor must be "ai", "human", or "user", got "${opts.actor}".`, ["ai", "human", "user"]);
|
|
902
|
+
}
|
|
903
|
+
// --turn validation: must parse as a non-negative integer.
|
|
904
|
+
let turnNum;
|
|
905
|
+
if (opts.turn !== undefined) {
|
|
906
|
+
const n = parseInt(opts.turn, 10);
|
|
907
|
+
if (Number.isNaN(n) || n < 0 || String(n) !== opts.turn.trim()) {
|
|
908
|
+
throw new ValidationError(`--turn must be a non-negative integer, got "${opts.turn}".`, []);
|
|
909
|
+
}
|
|
910
|
+
turnNum = n;
|
|
911
|
+
}
|
|
912
|
+
// --group-by axis whitelist.
|
|
913
|
+
const VALID_GROUP_BY = [
|
|
914
|
+
"iteration",
|
|
915
|
+
"frame",
|
|
916
|
+
"segment",
|
|
917
|
+
"turn",
|
|
918
|
+
"assignment",
|
|
919
|
+
"step",
|
|
920
|
+
];
|
|
921
|
+
let groupByKind;
|
|
922
|
+
if (opts.groupBy !== undefined) {
|
|
923
|
+
if (!VALID_GROUP_BY.includes(opts.groupBy)) {
|
|
924
|
+
throw new ValidationError(`--group-by must be one of: ${VALID_GROUP_BY.join(", ")}. Got "${opts.groupBy}".`, VALID_GROUP_BY);
|
|
925
|
+
}
|
|
926
|
+
groupByKind = opts.groupBy;
|
|
927
|
+
}
|
|
762
928
|
const rid = resolveId(id);
|
|
929
|
+
// --- --transcript fast path (no fetch of study payload) ---
|
|
763
930
|
if (opts.transcript) {
|
|
764
931
|
// --transcript <participant_id>: bypass the study aggregator; fetch
|
|
765
932
|
// the named participant directly. Cheaper (one GET, no nested
|
|
@@ -769,20 +936,152 @@ When no runs have completed, the default envelope is returned with zero counts a
|
|
|
769
936
|
output(buildChatTranscript(participant), globals.json, { preProjected: true });
|
|
770
937
|
return;
|
|
771
938
|
}
|
|
772
|
-
|
|
939
|
+
// --- Default-fast path: no filter, no group-by ---
|
|
940
|
+
if (!hasFilter && !hasGroupBy) {
|
|
941
|
+
const [data, participants] = await Promise.all([
|
|
942
|
+
client.get(`/studies/${rid}`),
|
|
943
|
+
fetchStudyParticipants(client, rid),
|
|
944
|
+
]);
|
|
945
|
+
if (wantsSummary) {
|
|
946
|
+
output(buildStudyResultsSummary(data, participants), globals.json, { preProjected: true });
|
|
947
|
+
}
|
|
948
|
+
else {
|
|
949
|
+
formatStudyResults(data, participants, globals.json);
|
|
950
|
+
}
|
|
951
|
+
if (!globals.json && data.product_id) {
|
|
952
|
+
const url = getWebUrl(globals, `/${data.product_id}/${rid}/overview`);
|
|
953
|
+
console.error(`\n ${terminalLink(url, "Open in browser ↗")}\n`);
|
|
954
|
+
}
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
957
|
+
// --- Slice / projection path: fetch in parallel, then filter+project ---
|
|
958
|
+
//
|
|
959
|
+
// Modality gating for --group-by happens AFTER the study fetch
|
|
960
|
+
// (we need study.modality), but BEFORE any further work — see the
|
|
961
|
+
// post-fetch validation block below. Pre-fetch validation above is
|
|
962
|
+
// limited to checks that don't need wire data.
|
|
963
|
+
const fetchFrames = opts.frame !== undefined;
|
|
964
|
+
const [study, participants, framesPayload] = await Promise.all([
|
|
773
965
|
client.get(`/studies/${rid}`),
|
|
774
966
|
fetchStudyParticipants(client, rid),
|
|
967
|
+
fetchFrames
|
|
968
|
+
? client.get(`/studies/${rid}/frames`)
|
|
969
|
+
: Promise.resolve([]),
|
|
775
970
|
]);
|
|
776
|
-
|
|
777
|
-
|
|
971
|
+
const studyRec = study;
|
|
972
|
+
const modality = typeof studyRec.modality === "string" ? studyRec.modality : "unknown";
|
|
973
|
+
// Modality gating for --group-by — router-level, NOT projection-level
|
|
974
|
+
// (devon's T7 note: projection builders are intentionally
|
|
975
|
+
// modality-agnostic and bucket non-matching rows into `_unmatched`;
|
|
976
|
+
// the surface is responsible for refusing nonsensical axes up front).
|
|
977
|
+
// Pattern B: modality-mismatched --group-by names the offending axis's
|
|
978
|
+
// domain AND suggests the axis that DOES apply to the study's current
|
|
979
|
+
// modality, so a cold-start agent can retry productively in one hop.
|
|
980
|
+
const axisHint = (mod) => {
|
|
981
|
+
if (mod === "interactive")
|
|
982
|
+
return "use --group-by frame";
|
|
983
|
+
if (["video", "audio", "text", "document"].includes(mod))
|
|
984
|
+
return "use --group-by segment";
|
|
985
|
+
if (mod === "chat")
|
|
986
|
+
return "use --group-by turn";
|
|
987
|
+
return undefined;
|
|
988
|
+
};
|
|
989
|
+
if (groupByKind === "frame" && modality !== "interactive") {
|
|
990
|
+
throw new ValidationError(`--group-by frame requires modality=interactive; this study is "${modality}".`, ["interactive"], axisHint(modality));
|
|
778
991
|
}
|
|
779
|
-
|
|
780
|
-
|
|
992
|
+
const SEGMENT_MODALITIES = ["video", "audio", "text", "document"];
|
|
993
|
+
if (groupByKind === "segment" && !SEGMENT_MODALITIES.includes(modality)) {
|
|
994
|
+
throw new ValidationError(`--group-by segment requires modality ∈ {${SEGMENT_MODALITIES.join(", ")}}; this study is "${modality}".`, SEGMENT_MODALITIES, axisHint(modality));
|
|
781
995
|
}
|
|
782
|
-
if (
|
|
783
|
-
|
|
784
|
-
console.error(`\n ${terminalLink(url, "Open in browser ↗")}\n`);
|
|
996
|
+
if (groupByKind === "turn" && modality !== "chat") {
|
|
997
|
+
throw new ValidationError(`--group-by turn requires modality=chat; this study is "${modality}".`, ["chat"], axisHint(modality));
|
|
785
998
|
}
|
|
999
|
+
// Coerce the frames payload to a plain array of records (the API
|
|
1000
|
+
// returns a bare array). Tolerate `{items: [...]}` shape in case the
|
|
1001
|
+
// endpoint ever normalises.
|
|
1002
|
+
const rawFrames = Array.isArray(framesPayload)
|
|
1003
|
+
? framesPayload
|
|
1004
|
+
: Array.isArray(framesPayload?.items)
|
|
1005
|
+
? (framesPayload.items)
|
|
1006
|
+
: [];
|
|
1007
|
+
const filters = {
|
|
1008
|
+
frame: opts.frame,
|
|
1009
|
+
segment: opts.segment,
|
|
1010
|
+
turn: turnNum,
|
|
1011
|
+
side: sideNormalised,
|
|
1012
|
+
assignment: opts.assignment,
|
|
1013
|
+
step: opts.step,
|
|
1014
|
+
sentiment: opts.sentiment && opts.sentiment.length > 0 ? opts.sentiment : undefined,
|
|
1015
|
+
actor: actorNormalised,
|
|
1016
|
+
iteration: opts.iteration,
|
|
1017
|
+
participant: opts.participant,
|
|
1018
|
+
includeUnmatched: opts.includeUnmatched === true ? true : undefined,
|
|
1019
|
+
includeEvidence: opts.includeEvidence === true ? true : undefined,
|
|
1020
|
+
};
|
|
1021
|
+
const filtered = applyResultsFilters(studyRec, participants, rawFrames, filters);
|
|
1022
|
+
// Surface modality-mismatch warnings (and any other diagnostics from
|
|
1023
|
+
// applyResultsFilters) on stderr so JSON output stays clean. The
|
|
1024
|
+
// filter pipeline downgrades mismatched flags to no-ops; the warnings
|
|
1025
|
+
// tell the agent which flags were ignored and why.
|
|
1026
|
+
if (filtered.warnings.length > 0 && !globals.quiet) {
|
|
1027
|
+
for (const w of filtered.warnings) {
|
|
1028
|
+
console.error(`warning: ${w}`);
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
// --- Dispatch: --group-by projection > --summary on filtered > filtered envelope ---
|
|
1032
|
+
if (groupByKind !== undefined) {
|
|
1033
|
+
let projection;
|
|
1034
|
+
switch (groupByKind) {
|
|
1035
|
+
case "iteration":
|
|
1036
|
+
projection = buildStudyResultsPerIteration(filtered);
|
|
1037
|
+
break;
|
|
1038
|
+
case "frame":
|
|
1039
|
+
projection = buildStudyResultsPerFrame(filtered);
|
|
1040
|
+
break;
|
|
1041
|
+
case "segment":
|
|
1042
|
+
projection = buildStudyResultsPerSegment(filtered);
|
|
1043
|
+
break;
|
|
1044
|
+
case "turn":
|
|
1045
|
+
projection = buildStudyResultsPerTurn(filtered);
|
|
1046
|
+
break;
|
|
1047
|
+
case "assignment":
|
|
1048
|
+
projection = buildStudyResultsPerAssignment(filtered);
|
|
1049
|
+
break;
|
|
1050
|
+
case "step":
|
|
1051
|
+
projection = buildStudyResultsPerStep(filtered);
|
|
1052
|
+
break;
|
|
1053
|
+
}
|
|
1054
|
+
const envelope = wrapSliceProjection(filtered, groupByKind, projection, rid, modality);
|
|
1055
|
+
formatStudyResultsGroupBy(envelope, groupByKind, globals.json);
|
|
1056
|
+
return;
|
|
1057
|
+
}
|
|
1058
|
+
if (wantsSummary) {
|
|
1059
|
+
// --summary on filtered participants: narrowed summary projection.
|
|
1060
|
+
// Attach totals_unfiltered so callers can still see the pre-filter
|
|
1061
|
+
// denominator (e.g. "12 / 80 participants matched").
|
|
1062
|
+
const summary = buildStudyResultsSummary(filtered.study, filtered.participants);
|
|
1063
|
+
const summaryOut = {
|
|
1064
|
+
...summary,
|
|
1065
|
+
totals_unfiltered: filtered.totals_unfiltered,
|
|
1066
|
+
};
|
|
1067
|
+
output(summaryOut, globals.json, { preProjected: true });
|
|
1068
|
+
return;
|
|
1069
|
+
}
|
|
1070
|
+
// Default (no --group-by, no --summary) but filters set: stable
|
|
1071
|
+
// envelope on the filtered participants + totals_unfiltered + the
|
|
1072
|
+
// modality_warnings array (Pattern U). Without `modality_warnings`
|
|
1073
|
+
// on this envelope, agents who pipe stderr to /dev/null lose the
|
|
1074
|
+
// filter-mismatch signal entirely; the `--group-by` envelope
|
|
1075
|
+
// already carries it (see wrapSliceProjection), so this is just
|
|
1076
|
+
// closing the asymmetry. Empty slice contract: zero matches still
|
|
1077
|
+
// yields participant_count=0 and exit 0, never a 4/not-found.
|
|
1078
|
+
const envelope = buildStudyResultsEnvelope(filtered.study, filtered.participants);
|
|
1079
|
+
const envelopeOut = {
|
|
1080
|
+
...envelope,
|
|
1081
|
+
totals_unfiltered: filtered.totals_unfiltered,
|
|
1082
|
+
modality_warnings: filtered.warnings,
|
|
1083
|
+
};
|
|
1084
|
+
output(envelopeOut, globals.json, { preProjected: true });
|
|
786
1085
|
});
|
|
787
1086
|
});
|
|
788
1087
|
study
|
package/dist/lib/alias-store.js
CHANGED
|
@@ -22,6 +22,7 @@ export const ALIAS_PREFIX = {
|
|
|
22
22
|
askRound: "r",
|
|
23
23
|
chatEndpoint: "ep",
|
|
24
24
|
chatConfig: "cc",
|
|
25
|
+
frame: "f",
|
|
25
26
|
};
|
|
26
27
|
/** Format a number with zero-padding (minimum 2 digits). */
|
|
27
28
|
function padNum(n) {
|
|
@@ -133,6 +134,7 @@ const HYDRATE_HINT = {
|
|
|
133
134
|
a: "ish ask list",
|
|
134
135
|
r: "ish ask get <ask-id>",
|
|
135
136
|
ep: "ish chat endpoint list",
|
|
137
|
+
f: "ish study results <study-id> --frame <name> # frames are discovered via the study's frames endpoint",
|
|
136
138
|
// Legacy two-letter prefixes the deterministic generator may have
|
|
137
139
|
// produced before; defaults below cover anything else.
|
|
138
140
|
};
|
|
@@ -294,15 +294,16 @@ function enforceParticipantCap(ids, flags, opts) {
|
|
|
294
294
|
*/
|
|
295
295
|
export function addPersonFilterFlags(cmd, opts = {}) {
|
|
296
296
|
const allFlag = opts.allFlagName ?? "--all";
|
|
297
|
-
const allDesc = opts.allFlagDescription ?? "Use every person matching the filters"
|
|
297
|
+
const allDesc = (opts.allFlagDescription ?? "Use every person matching the filters")
|
|
298
|
+
+ " (capped at 20 per dispatch — split into multiple slices for larger cohorts)";
|
|
298
299
|
return cmd
|
|
299
300
|
.option("--person <ids>", "Person IDs/aliases (comma-separated or repeatable)", collectIds, [])
|
|
300
|
-
.option("--sample <N>", "Randomly sample N people from the matching pool")
|
|
301
|
+
.option("--sample <N>", "Randomly sample N people from the matching pool (max 20 per dispatch — split into multiple slices for larger cohorts)")
|
|
301
302
|
.option(allFlag, allDesc)
|
|
302
303
|
.option("--search <text>", "Substring match against person name")
|
|
303
304
|
.option("--bio <text>", "Substring match against person bio")
|
|
304
305
|
.option("--occupation <text>", "Substring match against person occupation (repeatable)", collectRepeatable, [])
|
|
305
|
-
.option("--gender <gender>", "Filter by gender (repeatable)", collectRepeatable, [])
|
|
306
|
+
.option("--gender <gender>", "Filter by gender (female, male, nonbinary; repeatable, OR semantics)", collectRepeatable, [])
|
|
306
307
|
.option("--country <code>", "Filter by 2-letter country code (repeatable)", collectRepeatable, [])
|
|
307
308
|
.option("--min-age <n>", "Minimum age (inclusive)")
|
|
308
309
|
.option("--max-age <n>", "Maximum age (inclusive)")
|