@ishlabs/cli 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/chat.js +2 -2
- package/dist/commands/config.js +17 -3
- package/dist/commands/source.js +1 -1
- package/dist/commands/study-analyze.js +15 -2
- package/dist/commands/study-participant.js +19 -0
- package/dist/commands/study-run.d.ts +2 -0
- package/dist/commands/study-run.js +71 -20
- package/dist/commands/study.js +96 -34
- package/dist/lib/command-helpers.js +4 -3
- package/dist/lib/docs.js +114 -43
- package/dist/lib/output.d.ts +14 -9
- package/dist/lib/output.js +91 -19
- package/dist/lib/skill-content.js +10 -1
- package/dist/lib/study-participants.d.ts +3 -0
- package/dist/lib/study-results-filters.js +35 -14
- package/dist/lib/study-results-projections.d.ts +47 -17
- package/dist/lib/study-results-projections.js +39 -36
- package/dist/lib/types.d.ts +4 -0
- package/package.json +1 -1
package/dist/commands/chat.js
CHANGED
|
@@ -115,7 +115,7 @@ Examples:
|
|
|
115
115
|
});
|
|
116
116
|
parent
|
|
117
117
|
.command("create")
|
|
118
|
-
.description("Create a chatbot endpoint from a config
|
|
118
|
+
.description("Create a chatbot endpoint from a hand-written ChatbotEndpointConfig JSON (advanced — use `chat endpoint init` to infer the config from a curl sample, JSON, or template).")
|
|
119
119
|
.requiredOption("--endpoint-config <file>", 'Path to JSON file (or "-" for stdin)')
|
|
120
120
|
.option("--name <name>", "Override the name from the config file")
|
|
121
121
|
.option("--workspace <id>", "Workspace ID")
|
|
@@ -342,7 +342,7 @@ endpoint, apply the override, and PUT the merged result. Field flags win over
|
|
|
342
342
|
function attachChatEndpointInit(parent) {
|
|
343
343
|
parent
|
|
344
344
|
.command("init")
|
|
345
|
-
.description("Author an endpoint from a curl/JSON sample via test-and-map, or from a known-good template")
|
|
345
|
+
.description("Author an endpoint from a curl/JSON sample via test-and-map, or from a known-good template (recommended for most users — use `chat endpoint create` only when you already have a hand-written ChatbotEndpointConfig).")
|
|
346
346
|
.option("--from-curl <file>", 'Path to a curl example file (or "-" for stdin)')
|
|
347
347
|
.option("--from-json <file>", 'Path to a JSON request/response sample (or "-" for stdin)')
|
|
348
348
|
.option("--template <name>", `Start from a known-good template (one of: ${TEMPLATE_NAMES.join(", ")})`)
|
package/dist/commands/config.js
CHANGED
|
@@ -78,11 +78,25 @@ Run \`ish docs overview\` for the full mental model.`);
|
|
|
78
78
|
});
|
|
79
79
|
config
|
|
80
80
|
.command("schema")
|
|
81
|
-
.description("Get simulation config schema with defaults")
|
|
81
|
+
.description("Get simulation config schema with defaults (admin-only — non-admin accounts: ask an admin to share an existing config ID and pass it via `ish study run --config <id>`).")
|
|
82
82
|
.action(async (_opts, cmd) => {
|
|
83
83
|
await withClient(cmd, async (client, globals) => {
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
try {
|
|
85
|
+
const data = await client.get("/dev/simulation-configs/schema");
|
|
86
|
+
output(data, globals.json);
|
|
87
|
+
}
|
|
88
|
+
catch (err) {
|
|
89
|
+
// Pattern Z: re-throw with a hint pointing non-admin agents at the
|
|
90
|
+
// workaround (use a shared config ID via `study run --config`).
|
|
91
|
+
if (err instanceof Error && err.status === 403) {
|
|
92
|
+
const tagged = err;
|
|
93
|
+
const extra = "Non-admin accounts cannot introspect the simulation-config schema. To still use a config, ask an admin to share an existing config ID and pass it via `ish study run --config <id>` (`config --help` for the full workflow).";
|
|
94
|
+
tagged.suggestions = Array.isArray(tagged.suggestions)
|
|
95
|
+
? [...tagged.suggestions, extra]
|
|
96
|
+
: [extra];
|
|
97
|
+
}
|
|
98
|
+
throw err;
|
|
99
|
+
}
|
|
86
100
|
});
|
|
87
101
|
});
|
|
88
102
|
config
|
package/dist/commands/source.js
CHANGED
|
@@ -27,7 +27,7 @@ same attachment across multiple generation runs; otherwise pass a local path dir
|
|
|
27
27
|
to \`person generate --source\` and it auto-uploads.
|
|
28
28
|
|
|
29
29
|
Concept pages: ish docs get-page concepts/source
|
|
30
|
-
ish docs get-page concepts/
|
|
30
|
+
ish docs get-page concepts/person`);
|
|
31
31
|
source
|
|
32
32
|
.command("upload")
|
|
33
33
|
.description("Upload a file as a participant attachment and wait for processing")
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* about latency than load).
|
|
15
15
|
*/
|
|
16
16
|
import { withClient, resolveStudy, parseWaitTimeout } from "../lib/command-helpers.js";
|
|
17
|
-
import { resolveId } from "../lib/alias-store.js";
|
|
17
|
+
import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
18
18
|
import { output, printTable } from "../lib/output.js";
|
|
19
19
|
import { WaitTimeoutError } from "./study-run.js";
|
|
20
20
|
const POLL_INTERVAL_MS = 5_000;
|
|
@@ -160,11 +160,24 @@ Trigger a new run with \`ish study analyze --wait\`.`)
|
|
|
160
160
|
const history = await client.get(`/studies/${studyId}/results`);
|
|
161
161
|
const latest = history[0] ?? null;
|
|
162
162
|
if (globals.json) {
|
|
163
|
+
// Pattern K: never emit empty stdout. When no analyses have run,
|
|
164
|
+
// ship a stable envelope with a hint pointing at the verb that
|
|
165
|
+
// populates it. Mirrors the `study results` empty-envelope contract.
|
|
166
|
+
if (!latest) {
|
|
167
|
+
const studyAlias = tagAlias(ALIAS_PREFIX.study, studyId);
|
|
168
|
+
output({
|
|
169
|
+
latest: null,
|
|
170
|
+
history: [],
|
|
171
|
+
hint: `No analyses run yet. Trigger one with \`ish study analyze ${studyAlias}\`.`,
|
|
172
|
+
}, true, { preProjected: true });
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
163
175
|
output({ latest, history }, true);
|
|
164
176
|
return;
|
|
165
177
|
}
|
|
166
178
|
if (!latest) {
|
|
167
|
-
|
|
179
|
+
const studyAlias = tagAlias(ALIAS_PREFIX.study, studyId);
|
|
180
|
+
console.log(`No analysis runs yet. Trigger one with \`ish study analyze ${studyAlias}\`.`);
|
|
168
181
|
return;
|
|
169
182
|
}
|
|
170
183
|
if (opts.all) {
|
|
@@ -76,6 +76,25 @@ Tips:
|
|
|
76
76
|
const result = data;
|
|
77
77
|
if (result.id)
|
|
78
78
|
result.alias = tagAlias(ALIAS_PREFIX.participant, String(result.id));
|
|
79
|
+
// Pattern L: enrich with parent-graph aliases so agents can traverse
|
|
80
|
+
// from a participant straight to its study without hopping through
|
|
81
|
+
// `iteration get`. The participant response carries `iteration_id` but
|
|
82
|
+
// not `study_id`; one iteration fetch supplies both.
|
|
83
|
+
const iterationId = typeof result.iteration_id === "string" ? result.iteration_id : null;
|
|
84
|
+
if (iterationId) {
|
|
85
|
+
result.iteration_alias = tagAlias(ALIAS_PREFIX.iteration, iterationId);
|
|
86
|
+
try {
|
|
87
|
+
const iter = await client.get(`/iterations/${iterationId}`);
|
|
88
|
+
if (typeof iter.study_id === "string") {
|
|
89
|
+
result.study_id = iter.study_id;
|
|
90
|
+
result.study_alias = tagAlias(ALIAS_PREFIX.study, iter.study_id);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// Best-effort enrichment; if the iteration fetch fails (deleted,
|
|
95
|
+
// permission), keep going with the alias we already injected.
|
|
96
|
+
}
|
|
97
|
+
}
|
|
79
98
|
if (opts.summary) {
|
|
80
99
|
output(buildParticipantSummary(result), globals.json, { preProjected: true });
|
|
81
100
|
return;
|
|
@@ -108,6 +108,26 @@ const POLL_INTERVAL_MS = 5_000;
|
|
|
108
108
|
// transparently reverts to POLL_INTERVAL_MS.
|
|
109
109
|
const SSE_BACKSTOP_INTERVAL_MS = 30_000;
|
|
110
110
|
const TERMINAL_STATUSES = new Set(["completed", "errored", "failed", "cancelled", "canceled"]);
|
|
111
|
+
// If any running participant has been alive longer than this on the
|
|
112
|
+
// server, the wait-timeout message picks up an explicit "likely stuck"
|
|
113
|
+
// hint. Sized just above the worker's in-process stale-heartbeat
|
|
114
|
+
// threshold (600s) so the suggestion matches the backend reaper's
|
|
115
|
+
// verdict (see app/services/jobs/cleanup_stale_participants.py).
|
|
116
|
+
const LIKELY_STUCK_AGE_SECONDS = 900;
|
|
117
|
+
function buildWaitTimeoutMessage(opts) {
|
|
118
|
+
const base = `Timed out after ${opts.timeoutSeconds}s waiting for simulations. ` +
|
|
119
|
+
`${opts.done}/${opts.total} done. ${opts.resumeHint}`;
|
|
120
|
+
const likelyStuck = opts.rows.some((r) => typeof r.age_seconds === "number" &&
|
|
121
|
+
r.age_seconds >= LIKELY_STUCK_AGE_SECONDS &&
|
|
122
|
+
!TERMINAL_STATUSES.has(r.status));
|
|
123
|
+
if (!likelyStuck)
|
|
124
|
+
return base;
|
|
125
|
+
return (base +
|
|
126
|
+
" At least one participant has been running >15 min (see " +
|
|
127
|
+
"`progress.rows[].age_seconds`); the worker likely died. The " +
|
|
128
|
+
"backend reaper will mark it FAILED(stale_worker) within ~15 min — " +
|
|
129
|
+
"don't keep polling.");
|
|
130
|
+
}
|
|
111
131
|
function flattenParticipantStatuses(participants, opts = {}) {
|
|
112
132
|
const rows = [];
|
|
113
133
|
for (const t of participants ?? []) {
|
|
@@ -128,6 +148,8 @@ function flattenParticipantStatuses(participants, opts = {}) {
|
|
|
128
148
|
participant_name: t.person?.name || "Unknown",
|
|
129
149
|
interaction_count: Array.isArray(t.interactions) ? t.interactions.length : 0,
|
|
130
150
|
...(errorMessage && { error_message: String(errorMessage) }),
|
|
151
|
+
...(t.error_kind && { error_kind: t.error_kind }),
|
|
152
|
+
...(typeof t.age_seconds === "number" && { age_seconds: t.age_seconds }),
|
|
131
153
|
});
|
|
132
154
|
}
|
|
133
155
|
return rows;
|
|
@@ -171,8 +193,13 @@ async function pollStudyUntilDone(client, opts) {
|
|
|
171
193
|
return { rows, isMedia };
|
|
172
194
|
}
|
|
173
195
|
if (Date.now() - start > opts.timeoutMs) {
|
|
174
|
-
throw new WaitTimeoutError(
|
|
175
|
-
|
|
196
|
+
throw new WaitTimeoutError(buildWaitTimeoutMessage({
|
|
197
|
+
timeoutSeconds: Math.round(opts.timeoutMs / 1000),
|
|
198
|
+
done,
|
|
199
|
+
total,
|
|
200
|
+
rows,
|
|
201
|
+
resumeHint: `Run \`ish study poll --study ${opts.studyId}\` to check status.`,
|
|
202
|
+
}), {
|
|
176
203
|
study_id: opts.studyId,
|
|
177
204
|
...(opts.iterationId && { iteration_id: opts.iterationId }),
|
|
178
205
|
timeout_seconds: Math.round(opts.timeoutMs / 1000),
|
|
@@ -1128,20 +1155,32 @@ Examples:
|
|
|
1128
1155
|
// M8 + M9 (per-participant wait): structured wait_timeout with the
|
|
1129
1156
|
// current status as `progress.rows[0]` so `study wait <id>`
|
|
1130
1157
|
// always emits machine-readable final state.
|
|
1131
|
-
|
|
1158
|
+
const ageSeconds = typeof data.age_seconds === "number"
|
|
1159
|
+
? data.age_seconds
|
|
1160
|
+
: undefined;
|
|
1161
|
+
const rows = [
|
|
1162
|
+
{
|
|
1163
|
+
id: resolvedParticipant,
|
|
1164
|
+
status,
|
|
1165
|
+
participant_name: String(data.participant_name ?? "Unknown"),
|
|
1166
|
+
interaction_count: 0,
|
|
1167
|
+
...(data.error_kind && { error_kind: String(data.error_kind) }),
|
|
1168
|
+
...(typeof ageSeconds === "number" && { age_seconds: ageSeconds }),
|
|
1169
|
+
},
|
|
1170
|
+
];
|
|
1171
|
+
throw new WaitTimeoutError(buildWaitTimeoutMessage({
|
|
1172
|
+
timeoutSeconds: Math.round(timeoutMs / 1000),
|
|
1173
|
+
done: 0,
|
|
1174
|
+
total: 1,
|
|
1175
|
+
rows,
|
|
1176
|
+
resumeHint: `Last status: ${status}.`,
|
|
1177
|
+
}), {
|
|
1132
1178
|
study_id: resolvedParticipant,
|
|
1133
1179
|
timeout_seconds: Math.round(timeoutMs / 1000),
|
|
1134
1180
|
done: 0,
|
|
1135
1181
|
total: 1,
|
|
1136
1182
|
pending: 1,
|
|
1137
|
-
rows
|
|
1138
|
-
{
|
|
1139
|
-
id: resolvedParticipant,
|
|
1140
|
-
status,
|
|
1141
|
-
participant_name: String(data.participant_name ?? "Unknown"),
|
|
1142
|
-
interaction_count: 0,
|
|
1143
|
-
},
|
|
1144
|
-
],
|
|
1183
|
+
rows,
|
|
1145
1184
|
});
|
|
1146
1185
|
}
|
|
1147
1186
|
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
|
@@ -1352,20 +1391,32 @@ See \`ish docs get-page concepts/extending-a-simulation\` for the full mental mo
|
|
|
1352
1391
|
return;
|
|
1353
1392
|
}
|
|
1354
1393
|
if (Date.now() - start > timeoutMs) {
|
|
1355
|
-
|
|
1394
|
+
const ageSeconds = typeof status.age_seconds === "number"
|
|
1395
|
+
? status.age_seconds
|
|
1396
|
+
: undefined;
|
|
1397
|
+
const rows = [
|
|
1398
|
+
{
|
|
1399
|
+
id: newParticipantId,
|
|
1400
|
+
status: s,
|
|
1401
|
+
participant_name: String(status.participant_name ?? "Unknown"),
|
|
1402
|
+
interaction_count: typeof status.interaction_count === "number" ? status.interaction_count : 0,
|
|
1403
|
+
...(status.error_kind && { error_kind: String(status.error_kind) }),
|
|
1404
|
+
...(typeof ageSeconds === "number" && { age_seconds: ageSeconds }),
|
|
1405
|
+
},
|
|
1406
|
+
];
|
|
1407
|
+
throw new WaitTimeoutError(buildWaitTimeoutMessage({
|
|
1408
|
+
timeoutSeconds: Math.round(timeoutMs / 1000),
|
|
1409
|
+
done: 0,
|
|
1410
|
+
total: 1,
|
|
1411
|
+
rows,
|
|
1412
|
+
resumeHint: `Last status: ${s}.`,
|
|
1413
|
+
}), {
|
|
1356
1414
|
study_id: newParticipantId,
|
|
1357
1415
|
timeout_seconds: Math.round(timeoutMs / 1000),
|
|
1358
1416
|
done: 0,
|
|
1359
1417
|
total: 1,
|
|
1360
1418
|
pending: 1,
|
|
1361
|
-
rows
|
|
1362
|
-
{
|
|
1363
|
-
id: newParticipantId,
|
|
1364
|
-
status: s,
|
|
1365
|
-
participant_name: String(status.participant_name ?? "Unknown"),
|
|
1366
|
-
interaction_count: typeof status.interaction_count === "number" ? status.interaction_count : 0,
|
|
1367
|
-
},
|
|
1368
|
-
],
|
|
1419
|
+
rows,
|
|
1369
1420
|
});
|
|
1370
1421
|
}
|
|
1371
1422
|
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
package/dist/commands/study.js
CHANGED
|
@@ -8,7 +8,7 @@ import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
|
8
8
|
import { loadConfig, saveConfig } from "../config.js";
|
|
9
9
|
import { formatStudyList, formatStudyDetail, formatStudyResults, buildStudyResultsEnvelope, buildStudyResultsSummary, buildChatTranscript, formatStudyResultsGroupBy, output, ValidationError, } from "../lib/output.js";
|
|
10
10
|
import { applyResultsFilters } from "../lib/study-results-filters.js";
|
|
11
|
-
import { buildStudyResultsPerIteration, buildStudyResultsPerFrame, buildStudyResultsPerSegment, buildStudyResultsPerTurn, buildStudyResultsPerAssignment, buildStudyResultsPerStep, } from "../lib/study-results-projections.js";
|
|
11
|
+
import { buildStudyResultsPerIteration, buildStudyResultsPerFrame, buildStudyResultsPerSegment, buildStudyResultsPerTurn, buildStudyResultsPerAssignment, buildStudyResultsPerStep, wrapSliceProjection, } from "../lib/study-results-projections.js";
|
|
12
12
|
import { VALID_CONTENT_TYPES } from "../lib/types.js";
|
|
13
13
|
import { fetchStudyParticipants } from "../lib/study-participants.js";
|
|
14
14
|
import { parseAssignment, loadAssignmentsFile, validateAssignmentsArray, parseQuestion } from "../lib/study-inputs.js";
|
|
@@ -611,7 +611,7 @@ Next: configure a run with \`ish iteration create --study <id>\`,
|
|
|
611
611
|
});
|
|
612
612
|
study
|
|
613
613
|
.command("get")
|
|
614
|
-
.description("Get study
|
|
614
|
+
.description("Get the full study payload — iterations (with run details), assignments, interview questions, sentiment + status counts. Accepts multiple IDs for batched lookup. NOTE: this is the full payload, not a roll-up — for a compact cross-study comparison view use `study results <id> --summary`.")
|
|
615
615
|
.argument("<ids...>", "Study ID(s) — one or more aliases/UUIDs (space- or comma-separated)")
|
|
616
616
|
.addHelpText("after", `
|
|
617
617
|
Examples:
|
|
@@ -636,6 +636,18 @@ list table layout in human mode.`)
|
|
|
636
636
|
const result = data;
|
|
637
637
|
if (result.id)
|
|
638
638
|
result.alias = tagAlias(ALIAS_PREFIX.study, String(result.id));
|
|
639
|
+
// Pattern I-r3-1: inline iterations carry only label/name/details
|
|
640
|
+
// from the wire; tag each with its `alias` (computed from id via
|
|
641
|
+
// the local alias-store) so agents can drill from `study get` into
|
|
642
|
+
// `iteration get <alias>` / `study results --iteration <alias>`
|
|
643
|
+
// without a separate `iteration list` round-trip.
|
|
644
|
+
if (Array.isArray(result.iterations)) {
|
|
645
|
+
for (const iter of result.iterations) {
|
|
646
|
+
if (typeof iter.id === "string") {
|
|
647
|
+
iter.alias = tagAlias(ALIAS_PREFIX.iteration, iter.id);
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
}
|
|
639
651
|
if (data.product_id) {
|
|
640
652
|
result.url = getWebUrl(globals, `/${data.product_id}/${rid}/overview`);
|
|
641
653
|
}
|
|
@@ -655,6 +667,13 @@ list table layout in human mode.`)
|
|
|
655
667
|
const r = data;
|
|
656
668
|
if (r.id)
|
|
657
669
|
r.alias = tagAlias(ALIAS_PREFIX.study, String(r.id));
|
|
670
|
+
if (Array.isArray(r.iterations)) {
|
|
671
|
+
for (const iter of r.iterations) {
|
|
672
|
+
if (typeof iter.id === "string") {
|
|
673
|
+
iter.alias = tagAlias(ALIAS_PREFIX.iteration, iter.id);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
}
|
|
658
677
|
if (data.product_id) {
|
|
659
678
|
r.url = getWebUrl(globals, `/${data.product_id}/${rid}/overview`);
|
|
660
679
|
}
|
|
@@ -671,7 +690,7 @@ list table layout in human mode.`)
|
|
|
671
690
|
});
|
|
672
691
|
study
|
|
673
692
|
.command("results")
|
|
674
|
-
.description("View aggregated results: participant counts, sentiment, interview answers. Returns a stable envelope with empty fields when no runs have completed. Slice with filter flags (--frame, --segment, --turn, --side, --assignment, --step, --sentiment, --actor, --iteration, --participant) or project with --group-by (iteration|frame|segment|turn|assignment|step).")
|
|
693
|
+
.description("View aggregated results: participant counts, sentiment, interview answers. Returns a stable envelope with empty fields when no runs have completed. Slice with filter flags (--frame [interactive], --segment [video/audio/text/document], --turn [chat], --side [chat participant_pair], --assignment, --step, --sentiment, --actor, --iteration, --participant) or project with --group-by <axis> (iteration | frame [interactive] | segment [media] | turn [chat] | assignment | step).")
|
|
675
694
|
.argument("<id>", "Study ID")
|
|
676
695
|
.option("--workspace <id>", "Workspace ID; accepted for consistency (workspace is inferred from the study)")
|
|
677
696
|
.option("--summary", "Lean summary projection: counts + sentiment + per-participant {alias, status, sentiment, comment}. Drops interview_answers + per-interaction breakdowns. Composes with filters: `--summary --frame login` narrows the summary to the login-screen interactions.")
|
|
@@ -688,7 +707,7 @@ list table layout in human mode.`)
|
|
|
688
707
|
.option("--side <a|b>", "Filter participant_pair chat interactions by assignment side. Other modalities: warned and ignored.")
|
|
689
708
|
.option("--assignment <ref>", "Filter to a single assignment by UUID or name (substring, case-insensitive).")
|
|
690
709
|
.option("--step <ref>", "Filter `participant_assignments[].step_results[]` to a single step by step-id or name (substring). Pair with --include-evidence to also drop non-evidence interactions.")
|
|
691
|
-
.option("--sentiment <labels>", "Filter to interactions whose sentiment.label is in the comma-separated list (case-insensitive; repeatable). Drops null
|
|
710
|
+
.option("--sentiment <labels>", "Filter to interactions whose sentiment.label is in the comma-separated list (case-insensitive; repeatable). Drops interactions whose sentiment is null. A participant is kept when at least one of their interactions matches, even if their aggregate session sentiment is null (e.g. failed runs with a pre-error matching interaction).", collectIds, [])
|
|
692
711
|
.option("--actor <actor>", "Filter to interactions whose actor is `ai`, `human`, or `user` (case-insensitive).")
|
|
693
712
|
.option("--iteration <ref>", "Restrict to a single iteration by UUID or label.")
|
|
694
713
|
.option("--participant <ref>", "Restrict to a single participant by UUID or `pt-…` alias.")
|
|
@@ -713,6 +732,7 @@ Default --json envelope (M10: per-answer sentiment now included):
|
|
|
713
732
|
"participant_count": 12,
|
|
714
733
|
"completed_count": 8,
|
|
715
734
|
"failed_count": 0,
|
|
735
|
+
"participant_status_counts": { "completed": 8, "running": 3, "draft": 1 },
|
|
716
736
|
"sentiment": { "counts": { "Satisfied": 5, "Frustrated": 2 }, "total": 7 },
|
|
717
737
|
"interview_answers": [
|
|
718
738
|
{ "question": "...", "type": "text",
|
|
@@ -733,6 +753,12 @@ When any filter flag is passed, the envelope gains a \`totals_unfiltered\` field
|
|
|
733
753
|
("matched 12 / 80 participants"). A zero-match filter returns the stable
|
|
734
754
|
envelope with participant_count=0 and exit code 0 (not 4).
|
|
735
755
|
|
|
756
|
+
Filtered count semantics: \`participant_count\` is the matched-set total (every
|
|
757
|
+
participant whose interactions matched the filter — including running and
|
|
758
|
+
failed). The unfiltered denominator is \`totals_unfiltered.participant_count\`,
|
|
759
|
+
and the same envelope still carries \`completed_count\` / \`failed_count\` so
|
|
760
|
+
agents can compute "completed AND matched" without a second call.
|
|
761
|
+
|
|
736
762
|
--summary projection (M2-friction-7: drops the interview_answers payload):
|
|
737
763
|
{ study, participant_count, completed_count, failed_count, sentiment, participants: [...] }
|
|
738
764
|
|
|
@@ -749,23 +775,34 @@ envelope with participant_count=0 and exit code 0 (not 4).
|
|
|
749
775
|
"participant_summary": { "comment": "...", "sentiment": {...} }
|
|
750
776
|
}
|
|
751
777
|
|
|
752
|
-
--group-by
|
|
753
|
-
{
|
|
778
|
+
--group-by projections share one envelope (uniform across all six axes):
|
|
779
|
+
{ axis, rows, totals_unfiltered, modality_warnings, study_id, modality }
|
|
754
780
|
|
|
755
|
-
|
|
756
|
-
|
|
781
|
+
axis echoes the requested axis (iteration|frame|segment|turn|assignment|step)
|
|
782
|
+
study_id the \`s-…\` alias
|
|
783
|
+
modality the study's modality
|
|
784
|
+
totals_unfiltered { participant_count, interaction_count } — pre-filter counts
|
|
785
|
+
modality_warnings any filter-flag mismatches (e.g. --turn on a non-chat study)
|
|
757
786
|
|
|
758
|
-
|
|
759
|
-
[{ segment_index, segment_label, interaction_count, sentiment_histogram, engagement_histogram, sample_comments }, ...]
|
|
787
|
+
Per-axis row shape (one element of \`rows[]\`):
|
|
760
788
|
|
|
761
|
-
--group-by
|
|
762
|
-
|
|
789
|
+
--group-by iteration:
|
|
790
|
+
{ iteration_id, iteration_label, participant_count, interaction_count, sentiment, sample_comments, top_actions }
|
|
763
791
|
|
|
764
|
-
--group-by
|
|
765
|
-
|
|
792
|
+
--group-by frame (interactive only):
|
|
793
|
+
{ frame_id, frame_label, interaction_count, sentiment_histogram, sample_comments, participant_aliases }
|
|
766
794
|
|
|
767
|
-
--group-by
|
|
768
|
-
|
|
795
|
+
--group-by segment (video/audio/text/document):
|
|
796
|
+
{ segment_index, segment_label, interaction_count, sentiment_histogram, engagement_histogram, sample_comments }
|
|
797
|
+
|
|
798
|
+
--group-by turn (chat only):
|
|
799
|
+
{ turn_index, interaction_count, sentiment_histogram, sample_replies, failures }
|
|
800
|
+
|
|
801
|
+
--group-by assignment:
|
|
802
|
+
{ assignment_id, assignment_name, interaction_count, sentiment_histogram, step_completion }
|
|
803
|
+
|
|
804
|
+
--group-by step:
|
|
805
|
+
{ assignment_id, assignment_name, step_id, step_name, total, passed, inconclusive, failed, rate, participant_verdicts: [{ participant_alias, verdict, reason, evidence_interaction_ids }] }
|
|
769
806
|
|
|
770
807
|
Tips:
|
|
771
808
|
Use \`--get <path>\` for a single value (e.g. \`--get participant_count\`),
|
|
@@ -794,17 +831,24 @@ Common --get paths (--transcript <participant_id> envelope):
|
|
|
794
831
|
--get participant_summary.sentiment # aggregate sentiment map
|
|
795
832
|
--get unique_bot_replies # bot-side message count
|
|
796
833
|
|
|
797
|
-
Common --get paths (--group-by
|
|
798
|
-
--get
|
|
799
|
-
--get
|
|
800
|
-
--get
|
|
801
|
-
--get
|
|
802
|
-
--get
|
|
803
|
-
--get
|
|
804
|
-
|
|
805
|
-
--get
|
|
806
|
-
--get 0.
|
|
807
|
-
--get 0.
|
|
834
|
+
Common --get paths (--group-by envelope — uniform across axes):
|
|
835
|
+
--get axis # echoes the requested axis
|
|
836
|
+
--get study_id # s-… alias
|
|
837
|
+
--get modality # study's modality
|
|
838
|
+
--get modality_warnings # filter-flag mismatches (one warning per line)
|
|
839
|
+
--get totals_unfiltered.participant_count # pre-filter participant count
|
|
840
|
+
--get totals_unfiltered.interaction_count # pre-filter interaction count
|
|
841
|
+
|
|
842
|
+
--get rows.iteration_label # per-iteration: one label per line
|
|
843
|
+
--get rows.0.participant_count # per-iteration: first row's count
|
|
844
|
+
--get rows.0.frame_label # per-frame: first row's label
|
|
845
|
+
--get rows.0.sentiment_histogram # per-frame/segment/turn: first row's sentiment map
|
|
846
|
+
--get rows.0.segment_index # per-segment: first row's index
|
|
847
|
+
--get rows.0.turn_index # per-turn: first row's index
|
|
848
|
+
--get rows.0.assignment_name # per-assignment/step: first row's assignment
|
|
849
|
+
--get rows.0.step_name # per-step: first row's step
|
|
850
|
+
--get rows.0.rate # per-step: first row's pass-rate
|
|
851
|
+
--get rows.0.participant_verdicts.verdict # per-step: verdict per participant
|
|
808
852
|
|
|
809
853
|
When no runs have completed, the default envelope is returned with zero counts and empty arrays.`)
|
|
810
854
|
.action(async (id, opts, cmd) => {
|
|
@@ -930,15 +974,27 @@ When no runs have completed, the default envelope is returned with zero counts a
|
|
|
930
974
|
// (devon's T7 note: projection builders are intentionally
|
|
931
975
|
// modality-agnostic and bucket non-matching rows into `_unmatched`;
|
|
932
976
|
// the surface is responsible for refusing nonsensical axes up front).
|
|
977
|
+
// Pattern B: modality-mismatched --group-by names the offending axis's
|
|
978
|
+
// domain AND suggests the axis that DOES apply to the study's current
|
|
979
|
+
// modality, so a cold-start agent can retry productively in one hop.
|
|
980
|
+
const axisHint = (mod) => {
|
|
981
|
+
if (mod === "interactive")
|
|
982
|
+
return "use --group-by frame";
|
|
983
|
+
if (["video", "audio", "text", "document"].includes(mod))
|
|
984
|
+
return "use --group-by segment";
|
|
985
|
+
if (mod === "chat")
|
|
986
|
+
return "use --group-by turn";
|
|
987
|
+
return undefined;
|
|
988
|
+
};
|
|
933
989
|
if (groupByKind === "frame" && modality !== "interactive") {
|
|
934
|
-
throw new ValidationError(`--group-by frame requires modality=interactive; this study is "${modality}".`, ["interactive"]);
|
|
990
|
+
throw new ValidationError(`--group-by frame requires modality=interactive; this study is "${modality}".`, ["interactive"], axisHint(modality));
|
|
935
991
|
}
|
|
936
992
|
const SEGMENT_MODALITIES = ["video", "audio", "text", "document"];
|
|
937
993
|
if (groupByKind === "segment" && !SEGMENT_MODALITIES.includes(modality)) {
|
|
938
|
-
throw new ValidationError(`--group-by segment requires modality ∈ {${SEGMENT_MODALITIES.join(", ")}}; this study is "${modality}".`, SEGMENT_MODALITIES);
|
|
994
|
+
throw new ValidationError(`--group-by segment requires modality ∈ {${SEGMENT_MODALITIES.join(", ")}}; this study is "${modality}".`, SEGMENT_MODALITIES, axisHint(modality));
|
|
939
995
|
}
|
|
940
996
|
if (groupByKind === "turn" && modality !== "chat") {
|
|
941
|
-
throw new ValidationError(`--group-by turn requires modality=chat; this study is "${modality}".`, ["chat"]);
|
|
997
|
+
throw new ValidationError(`--group-by turn requires modality=chat; this study is "${modality}".`, ["chat"], axisHint(modality));
|
|
942
998
|
}
|
|
943
999
|
// Coerce the frames payload to a plain array of records (the API
|
|
944
1000
|
// returns a bare array). Tolerate `{items: [...]}` shape in case the
|
|
@@ -995,7 +1051,8 @@ When no runs have completed, the default envelope is returned with zero counts a
|
|
|
995
1051
|
projection = buildStudyResultsPerStep(filtered);
|
|
996
1052
|
break;
|
|
997
1053
|
}
|
|
998
|
-
|
|
1054
|
+
const envelope = wrapSliceProjection(filtered, groupByKind, projection, rid, modality);
|
|
1055
|
+
formatStudyResultsGroupBy(envelope, groupByKind, globals.json);
|
|
999
1056
|
return;
|
|
1000
1057
|
}
|
|
1001
1058
|
if (wantsSummary) {
|
|
@@ -1011,13 +1068,18 @@ When no runs have completed, the default envelope is returned with zero counts a
|
|
|
1011
1068
|
return;
|
|
1012
1069
|
}
|
|
1013
1070
|
// Default (no --group-by, no --summary) but filters set: stable
|
|
1014
|
-
// envelope on the filtered participants + totals_unfiltered
|
|
1015
|
-
//
|
|
1016
|
-
//
|
|
1071
|
+
// envelope on the filtered participants + totals_unfiltered + the
|
|
1072
|
+
// modality_warnings array (Pattern U). Without `modality_warnings`
|
|
1073
|
+
// on this envelope, agents who pipe stderr to /dev/null lose the
|
|
1074
|
+
// filter-mismatch signal entirely; the `--group-by` envelope
|
|
1075
|
+
// already carries it (see wrapSliceProjection), so this is just
|
|
1076
|
+
// closing the asymmetry. Empty slice contract: zero matches still
|
|
1077
|
+
// yields participant_count=0 and exit 0, never a 4/not-found.
|
|
1017
1078
|
const envelope = buildStudyResultsEnvelope(filtered.study, filtered.participants);
|
|
1018
1079
|
const envelopeOut = {
|
|
1019
1080
|
...envelope,
|
|
1020
1081
|
totals_unfiltered: filtered.totals_unfiltered,
|
|
1082
|
+
modality_warnings: filtered.warnings,
|
|
1021
1083
|
};
|
|
1022
1084
|
output(envelopeOut, globals.json, { preProjected: true });
|
|
1023
1085
|
});
|
|
@@ -294,15 +294,16 @@ function enforceParticipantCap(ids, flags, opts) {
|
|
|
294
294
|
*/
|
|
295
295
|
export function addPersonFilterFlags(cmd, opts = {}) {
|
|
296
296
|
const allFlag = opts.allFlagName ?? "--all";
|
|
297
|
-
const allDesc = opts.allFlagDescription ?? "Use every person matching the filters"
|
|
297
|
+
const allDesc = (opts.allFlagDescription ?? "Use every person matching the filters")
|
|
298
|
+
+ " (capped at 20 per dispatch — split into multiple slices for larger cohorts)";
|
|
298
299
|
return cmd
|
|
299
300
|
.option("--person <ids>", "Person IDs/aliases (comma-separated or repeatable)", collectIds, [])
|
|
300
|
-
.option("--sample <N>", "Randomly sample N people from the matching pool")
|
|
301
|
+
.option("--sample <N>", "Randomly sample N people from the matching pool (max 20 per dispatch — split into multiple slices for larger cohorts)")
|
|
301
302
|
.option(allFlag, allDesc)
|
|
302
303
|
.option("--search <text>", "Substring match against person name")
|
|
303
304
|
.option("--bio <text>", "Substring match against person bio")
|
|
304
305
|
.option("--occupation <text>", "Substring match against person occupation (repeatable)", collectRepeatable, [])
|
|
305
|
-
.option("--gender <gender>", "Filter by gender (repeatable)", collectRepeatable, [])
|
|
306
|
+
.option("--gender <gender>", "Filter by gender (female, male, nonbinary; repeatable, OR semantics)", collectRepeatable, [])
|
|
306
307
|
.option("--country <code>", "Filter by 2-letter country code (repeatable)", collectRepeatable, [])
|
|
307
308
|
.option("--min-age <n>", "Minimum age (inclusive)")
|
|
308
309
|
.option("--max-age <n>", "Maximum age (inclusive)")
|