@ishlabs/cli 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/ask.js +26 -2
- package/dist/commands/config.js +9 -1
- package/dist/commands/docs.js +6 -7
- package/dist/commands/person.js +123 -9
- package/dist/commands/secret.js +25 -2
- package/dist/commands/source.d.ts +1 -1
- package/dist/commands/source.js +10 -6
- package/dist/commands/study.js +269 -13
- package/dist/commands/workspace.js +41 -6
- package/dist/index.js +227 -44
- package/dist/lib/alias-store.d.ts +1 -0
- package/dist/lib/alias-store.js +25 -4
- package/dist/lib/auth.js +22 -4
- package/dist/lib/baggage.d.ts +15 -6
- package/dist/lib/baggage.js +14 -8
- package/dist/lib/command-helpers.d.ts +1 -0
- package/dist/lib/command-helpers.js +79 -7
- package/dist/lib/docs.js +400 -21
- package/dist/lib/output.d.ts +18 -0
- package/dist/lib/output.js +278 -18
- package/dist/lib/profile-sources.js +18 -0
- package/dist/lib/skill-content.js +78 -2
- package/dist/lib/study-participants.d.ts +13 -0
- package/dist/lib/study-participants.js +13 -0
- package/dist/lib/study-results-filters.d.ts +91 -0
- package/dist/lib/study-results-filters.js +538 -0
- package/dist/lib/study-results-projections.d.ts +122 -0
- package/dist/lib/study-results-projections.js +577 -0
- package/dist/upgrade.js +9 -2
- package/package.json +1 -1
|
@@ -28,7 +28,7 @@ const VISIBILITY_ALIASES = {
|
|
|
28
28
|
private: "workspace",
|
|
29
29
|
public: "platform",
|
|
30
30
|
};
|
|
31
|
-
function normalizeVisibility(raw) {
|
|
31
|
+
export function normalizeVisibility(raw) {
|
|
32
32
|
if (raw === undefined)
|
|
33
33
|
return undefined;
|
|
34
34
|
return VISIBILITY_ALIASES[raw] ?? raw;
|
|
@@ -186,6 +186,12 @@ export async function resolvePersonIds(client, workspace, flags, opts = {}) {
|
|
|
186
186
|
if (sampleN === undefined && !flags.all && !filtersUsed) {
|
|
187
187
|
throw new Error(`Select people: pass --person <id> (repeatable), --sample <N>, ${allFlagName}, or filter flags (--bio, --country, --gender, --min-age, --max-age, --occupation, --search, --visibility).`);
|
|
188
188
|
}
|
|
189
|
+
// NEW-CP-2 / Pattern H: --sample N > backend cap is detectable without
|
|
190
|
+
// any API call — fail fast so a bad value doesn't burn a /people round-trip
|
|
191
|
+
// before the cap message surfaces.
|
|
192
|
+
if (sampleN !== undefined && sampleN > PARTICIPANT_BATCH_CAP) {
|
|
193
|
+
throw new Error(`--sample ${sampleN} exceeds the per-dispatch participant cap of ${PARTICIPANT_BATCH_CAP}. Pass --sample ${PARTICIPANT_BATCH_CAP} or fewer, or split the run into multiple dispatches.`);
|
|
194
|
+
}
|
|
189
195
|
const params = {
|
|
190
196
|
product_id: workspace,
|
|
191
197
|
type: "ai",
|
|
@@ -245,15 +251,41 @@ export async function resolvePersonIds(client, workspace, flags, opts = {}) {
|
|
|
245
251
|
throw new Error(`No ${sim}people found in workspace ${workspace}.${opts.requireSimulatable ? " Create people with simulation configs first." : ""}`);
|
|
246
252
|
}
|
|
247
253
|
if (flags.all)
|
|
248
|
-
return pool.map((p) => p.id);
|
|
254
|
+
return enforceParticipantCap(pool.map((p) => p.id), flags, opts);
|
|
249
255
|
if (sampleN !== undefined) {
|
|
250
256
|
if (sampleN > pool.length) {
|
|
251
257
|
throw new Error(`--sample ${sampleN} requested but only ${pool.length} matching person${pool.length === 1 ? "" : "s"} available.`);
|
|
252
258
|
}
|
|
259
|
+
// sampleN > PARTICIPANT_BATCH_CAP is caught earlier (before the
|
|
260
|
+
// /people fetch) so we don't reach here with an over-cap sample.
|
|
253
261
|
return shuffleInPlace([...pool]).slice(0, sampleN).map((p) => p.id);
|
|
254
262
|
}
|
|
255
|
-
// Filters only, no --sample/--all → return every match.
|
|
256
|
-
return pool.map((p) => p.id);
|
|
263
|
+
// Filters only, no --sample/--all → return every match (subject to cap).
|
|
264
|
+
return enforceParticipantCap(pool.map((p) => p.id), flags, opts);
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Backend caps each dispatch batch at 20 simulations (Pydantic
|
|
268
|
+
* `max_length=20` on the `simulations` array in
|
|
269
|
+
* app/api/models/simulation.py, media.py, and chat.py). Without a
|
|
270
|
+
* client-side guard, `--all` on a workspace with platform-visible
|
|
271
|
+
* people resolves to ~200 (the `/people` pagination limit) and the
|
|
272
|
+
* backend returns a `validation_error` ("List should have at most 20
|
|
273
|
+
* items after validation, not 200") that's confusing without context.
|
|
274
|
+
* Throw a helpful client-side error before the dispatch so the user
|
|
275
|
+
* knows to sample explicitly.
|
|
276
|
+
*
|
|
277
|
+
* Discovered during the 2026-05-26 post-remediation checkpoint sweep
|
|
278
|
+
* (NEW-CP-2). If the backend cap changes, update PARTICIPANT_BATCH_CAP.
|
|
279
|
+
*/
|
|
280
|
+
const PARTICIPANT_BATCH_CAP = 20;
|
|
281
|
+
function enforceParticipantCap(ids, flags, opts) {
|
|
282
|
+
if (ids.length <= PARTICIPANT_BATCH_CAP)
|
|
283
|
+
return ids;
|
|
284
|
+
const allFlagName = opts.allFlagName ?? "--all";
|
|
285
|
+
const filterDesc = describeFilters(flags) || "no filter";
|
|
286
|
+
throw new Error(`Resolved ${ids.length} participants (${filterDesc}) but the backend caps each dispatch at ${PARTICIPANT_BATCH_CAP}. ` +
|
|
287
|
+
`Pass \`--sample ${PARTICIPANT_BATCH_CAP}\` to randomly subsample the pool, narrow your filters, or run the dispatch ` +
|
|
288
|
+
`multiple times against different slices. ${allFlagName} without --sample is only safe when the matching pool is ≤${PARTICIPANT_BATCH_CAP}.`);
|
|
257
289
|
}
|
|
258
290
|
/**
|
|
259
291
|
* Attach the person-selection flag set to a Commander command.
|
|
@@ -375,7 +407,25 @@ export function exitCodeFromError(err) {
|
|
|
375
407
|
return 5;
|
|
376
408
|
}
|
|
377
409
|
if (err instanceof Error) {
|
|
378
|
-
//
|
|
410
|
+
// Pattern D: structured `error_code` on the Error object takes
|
|
411
|
+
// precedence over message-regex sniffing. Sites that self-tag
|
|
412
|
+
// (alias-store, auth, sub-command envelopes) get a deterministic
|
|
413
|
+
// exit code; sites that don't fall back to the legacy regex below.
|
|
414
|
+
// Order matters: this block must run BEFORE the /^invalid /i and
|
|
415
|
+
// /no auth token found/ regexes (which would otherwise misclassify
|
|
416
|
+
// the new `not_found` tag as `2` because the message starts with
|
|
417
|
+
// "Invalid ID").
|
|
418
|
+
const code = err.error_code;
|
|
419
|
+
if (typeof code === "string") {
|
|
420
|
+
if (code === "usage_error")
|
|
421
|
+
return 2;
|
|
422
|
+
if (code === "auth_failed")
|
|
423
|
+
return 3;
|
|
424
|
+
if (code === "not_found")
|
|
425
|
+
return 4;
|
|
426
|
+
}
|
|
427
|
+
// Auth-related client errors (e.g. missing token) — legacy regex
|
|
428
|
+
// path; the new error_code tagging above is the preferred route.
|
|
379
429
|
if (/no auth token found|run "ish login"|saved token is invalid|session expired/i.test(err.message))
|
|
380
430
|
return 3;
|
|
381
431
|
// Client-side validation failures
|
|
@@ -393,8 +443,30 @@ export function exitCodeFromError(err) {
|
|
|
393
443
|
// Structured error_kind on the Error object (set by chat endpoint test/init,
|
|
394
444
|
// simulation routes, etc.). TunnelInactive is the canonical transient one.
|
|
395
445
|
const kind = err.error_kind;
|
|
396
|
-
if (typeof kind === "string"
|
|
397
|
-
|
|
446
|
+
if (typeof kind === "string") {
|
|
447
|
+
// Transient: user can fix the cause and retry (tunnel down, bot
|
|
448
|
+
// auth credentials missing/wrong, upstream rate-limited).
|
|
449
|
+
if (kind === "TunnelInactive" || kind === "BotAuthError")
|
|
450
|
+
return 5;
|
|
451
|
+
// Validation-shaped: user passed something bad to the CLI/endpoint.
|
|
452
|
+
if (kind === "ConfirmationRequired" || kind === "BotShapeError")
|
|
453
|
+
return 2;
|
|
454
|
+
}
|
|
455
|
+
// (error_code mapping moved earlier in the function — see top of the
|
|
456
|
+
// `err instanceof Error` block. Order matters: it must run BEFORE
|
|
457
|
+
// the legacy regex sniffers to honor self-tagged errors.)
|
|
458
|
+
// Pattern E (ISSUE-021): DNS / connection failures are transient.
|
|
459
|
+
// Node's fetch surfaces them as TypeError with .cause = { code: "..." }.
|
|
460
|
+
const cause = err.cause;
|
|
461
|
+
if (cause && typeof cause === "object") {
|
|
462
|
+
const causeCode = cause.code;
|
|
463
|
+
if (typeof causeCode === "string" && (causeCode === "ENOTFOUND"
|
|
464
|
+
|| causeCode === "ECONNREFUSED"
|
|
465
|
+
|| causeCode === "ECONNRESET"
|
|
466
|
+
|| causeCode === "ETIMEDOUT"
|
|
467
|
+
|| causeCode === "EAI_AGAIN"))
|
|
468
|
+
return 5;
|
|
469
|
+
}
|
|
398
470
|
}
|
|
399
471
|
return 1;
|
|
400
472
|
}
|
package/dist/lib/docs.js
CHANGED
|
@@ -156,6 +156,25 @@ first scraping the list.
|
|
|
156
156
|
The full saturated-account walkthrough (with branch logic + a worked
|
|
157
157
|
transcript) lives at \`guides/cold-start\`.
|
|
158
158
|
|
|
159
|
+
## Deleting a workspace
|
|
160
|
+
|
|
161
|
+
\`ish workspace delete <id>\` is the **highest-blast-radius destructive op
|
|
162
|
+
in the CLI** — it removes ALL nested studies, asks, people, secrets,
|
|
163
|
+
configs, sources, and chat endpoints. The confirmation guard is
|
|
164
|
+
mandatory:
|
|
165
|
+
|
|
166
|
+
- **Interactive (TTY)**: prompts on stderr naming the workspace; type
|
|
167
|
+
\`y\` to proceed.
|
|
168
|
+
- **Non-interactive** (\`--json\`, piped, or non-TTY stdin): pass
|
|
169
|
+
\`-y\` / \`--yes\` to confirm. Without it, the CLI exits with usage
|
|
170
|
+
code 2 rather than deleting silently.
|
|
171
|
+
|
|
172
|
+
\`\`\`
|
|
173
|
+
ish workspace delete w-6ec # interactive prompt
|
|
174
|
+
ish workspace delete w-6ec --yes # skip prompt
|
|
175
|
+
ish workspace delete w-6ec --json --yes # JSON/agent consumers must be explicit
|
|
176
|
+
\`\`\`
|
|
177
|
+
|
|
159
178
|
## Related
|
|
160
179
|
|
|
161
180
|
- \`guides/cold-start\` — saturated-account first-step playbook
|
|
@@ -296,6 +315,8 @@ pick was wrong.
|
|
|
296
315
|
- \`concepts/assignment\` — task definition syntax.
|
|
297
316
|
- \`concepts/questionnaire\` — question types and timing.
|
|
298
317
|
- \`concepts/run-verbs\` — when to use \`study run\` vs \`ask run\`.
|
|
318
|
+
- \`guides/slicing-results\` — filter / project \`study results\` by frame,
|
|
319
|
+
segment, turn, sentiment, assignment, step.
|
|
299
320
|
- \`reference/billing-limits\` — \`maxStudiesPerProduct\` cap on study creation.
|
|
300
321
|
- \`reference/credits\` — per-run credit cost & how to preview before dispatch.
|
|
301
322
|
`;
|
|
@@ -832,6 +853,9 @@ ride along when present in the JSON forms.
|
|
|
832
853
|
|
|
833
854
|
- \`concepts/study\` — assignments are immutable to the run; questionnaire is too.
|
|
834
855
|
- \`concepts/questionnaire\` — the other half of the study definition.
|
|
856
|
+
- \`guides/slicing-results\` — slice the post-run envelope by step
|
|
857
|
+
(\`--step verify-email --group-by step\`), surface per-participant verdicts
|
|
858
|
+
inline, or restrict to the evidence interactions with \`--include-evidence\`.
|
|
835
859
|
- \`reference/json-mode\` — how \`step_completion\` renders in lean vs --verbose.
|
|
836
860
|
`;
|
|
837
861
|
const CONCEPT_QUESTIONNAIRE = `# concept: questionnaire
|
|
@@ -1083,12 +1107,32 @@ round-trips when you know them up front:
|
|
|
1083
1107
|
- \`image:./hero-a.png\` — local image (auto-uploaded)
|
|
1084
1108
|
- \`image:./a.png::label=A\` — with explicit label
|
|
1085
1109
|
|
|
1110
|
+
## Deleting an ask
|
|
1111
|
+
|
|
1112
|
+
\`ish ask delete <id>\` requires explicit confirmation (parallels
|
|
1113
|
+
\`workspace delete\`, \`study delete\`, \`person delete\`, \`source
|
|
1114
|
+
delete\`, \`chat endpoint delete\`):
|
|
1115
|
+
|
|
1116
|
+
- **Interactive (TTY)**: prompts on stderr; type \`y\` to proceed.
|
|
1117
|
+
- **Non-interactive** (\`--json\`, piped, or non-TTY stdin): pass
|
|
1118
|
+
\`-y\` / \`--yes\` to confirm. Without it, the CLI exits with usage
|
|
1119
|
+
code 2 rather than deleting silently.
|
|
1120
|
+
|
|
1121
|
+
\`\`\`
|
|
1122
|
+
ish ask delete a-6ec # interactive prompt
|
|
1123
|
+
ish ask delete a-6ec --yes # skip prompt
|
|
1124
|
+
ish ask delete a-6ec --json --yes # JSON consumers must be explicit
|
|
1125
|
+
\`\`\`
|
|
1126
|
+
|
|
1127
|
+
The active ask is auto-cleared from \`~/.ish/config.json\` if the
|
|
1128
|
+
deleted ask was the active one.
|
|
1129
|
+
|
|
1086
1130
|
## Related
|
|
1087
1131
|
|
|
1088
1132
|
- \`concepts/round\` — what a round is and how it executes.
|
|
1089
1133
|
- \`concepts/people\` — how participants are chosen at ask creation.
|
|
1090
1134
|
- \`concepts/run-verbs\` — \`ish ask run\` vs \`ish study run\`.
|
|
1091
|
-
- \`reference/credits\` — ask rounds bill
|
|
1135
|
+
- \`reference/credits\` — ask rounds bill **one credit per successful participant per round**, regardless of how many \`questions\` were included. The backend's asks worker bills \`amount=succeeded\` once per round dispatch; questions and round-summary synthesis don't trigger separate debits. A 3-person panel with 2 follow-up questions costs \`3\` credits when all complete, the same as a no-questions run. Failed participant responses (pre-flight errors, refusals) don't bill.
|
|
1092
1136
|
`;
|
|
1093
1137
|
const CONCEPT_ROUND = `# concept: round
|
|
1094
1138
|
|
|
@@ -1261,6 +1305,21 @@ The legacy \`--tech-savviness\` flag was removed in
|
|
|
1261
1305
|
\`person-schema-v2\`; passing it now produces commander's standard
|
|
1262
1306
|
"unknown option" error.
|
|
1263
1307
|
|
|
1308
|
+
## Deleting a person
|
|
1309
|
+
|
|
1310
|
+
\`ish person delete <id>\` requires explicit confirmation:
|
|
1311
|
+
|
|
1312
|
+
- **Interactive (TTY)**: prompts on stderr; type \`y\` to proceed.
|
|
1313
|
+
- **Non-interactive** (\`--json\`, piped, or non-TTY stdin): pass
|
|
1314
|
+
\`-y\` / \`--yes\` to confirm. Without it, the CLI exits with usage
|
|
1315
|
+
code 2 rather than deleting silently.
|
|
1316
|
+
|
|
1317
|
+
\`\`\`
|
|
1318
|
+
ish person delete p-d4e # interactive prompt
|
|
1319
|
+
ish person delete p-d4e --yes # skip prompt
|
|
1320
|
+
ish person delete p-d4e --json --yes # JSON consumers must be explicit
|
|
1321
|
+
\`\`\`
|
|
1322
|
+
|
|
1264
1323
|
## Related
|
|
1265
1324
|
|
|
1266
1325
|
- \`concepts/source\` — the inputs to \`person generate\`.
|
|
@@ -1303,6 +1362,24 @@ in real customer evidence.
|
|
|
1303
1362
|
ish source get ps-3a4
|
|
1304
1363
|
\`\`\`
|
|
1305
1364
|
|
|
1365
|
+
## Deleting a source
|
|
1366
|
+
|
|
1367
|
+
\`ish source delete <id>\` requires explicit confirmation:
|
|
1368
|
+
|
|
1369
|
+
- **Interactive (TTY)**: prompts on stderr; type \`y\` to proceed.
|
|
1370
|
+
- **Non-interactive** (\`--json\`, piped, or non-TTY stdin): pass
|
|
1371
|
+
\`-y\` / \`--yes\` to confirm. Without it, the CLI exits with usage
|
|
1372
|
+
code 2 rather than deleting silently.
|
|
1373
|
+
|
|
1374
|
+
\`\`\`
|
|
1375
|
+
ish source delete ps-3a4 # interactive prompt
|
|
1376
|
+
ish source delete ps-3a4 --yes # skip prompt
|
|
1377
|
+
ish source delete ps-3a4 --json --yes # JSON consumers must be explicit
|
|
1378
|
+
\`\`\`
|
|
1379
|
+
|
|
1380
|
+
The backend ref-counts the underlying file: the storage object is
|
|
1381
|
+
removed only when no profile mappings remain.
|
|
1382
|
+
|
|
1306
1383
|
## Related
|
|
1307
1384
|
|
|
1308
1385
|
- \`concepts/person\` — sources feed profile generation.
|
|
@@ -1405,6 +1482,30 @@ manager"\` or \`"retail associate"\` return many. Two adaptations:
|
|
|
1405
1482
|
- \`ish ask run\` (without \`--new\`) → cannot change participants; the ask
|
|
1406
1483
|
fixes it at creation. Audience flags only apply with \`--new\`.
|
|
1407
1484
|
|
|
1485
|
+
## Per-dispatch cap (20)
|
|
1486
|
+
|
|
1487
|
+
Each \`study run\` / \`ask run\` / \`chat\` dispatch is capped at **20
|
|
1488
|
+
participants** by the backend (\`max_length=20\` on the \`simulations\`
|
|
1489
|
+
list). The CLI enforces this client-side BEFORE the network round-trip
|
|
1490
|
+
so a too-large \`--sample\` or an unbounded \`--all\` returns a clear
|
|
1491
|
+
error instead of a confusing server-side \`validation_error\`:
|
|
1492
|
+
|
|
1493
|
+
\`\`\`
|
|
1494
|
+
$ ish study run --all # on a workspace with platform pool
|
|
1495
|
+
Error: Resolved 200 participants (no filter) but the backend caps each dispatch at 20.
|
|
1496
|
+
Pass \`--sample 20\` to randomly subsample the pool, narrow your filters, or run
|
|
1497
|
+
the dispatch multiple times against different slices. --all without --sample is
|
|
1498
|
+
only safe when the matching pool is ≤20.
|
|
1499
|
+
|
|
1500
|
+
$ ish study run --sample 25 # bad value caught before /people fetch
|
|
1501
|
+
Error: --sample 25 exceeds the per-dispatch participant cap of 20.
|
|
1502
|
+
Pass --sample 20 or fewer, or split the run into multiple dispatches.
|
|
1503
|
+
\`\`\`
|
|
1504
|
+
|
|
1505
|
+
For larger panels: dispatch multiple times against different demographic
|
|
1506
|
+
slices (\`--country SE\`, then \`--country GB\`, etc.) or use the web UI
|
|
1507
|
+
which batches behind the scenes.
|
|
1508
|
+
|
|
1408
1509
|
## Examples
|
|
1409
1510
|
|
|
1410
1511
|
\`\`\`
|
|
@@ -2254,17 +2355,33 @@ The CLI guarantees these contracts so agents can chain safely:
|
|
|
2254
2355
|
|
|
2255
2356
|
## Exit codes
|
|
2256
2357
|
|
|
2257
|
-
| Code | Meaning |
|
|
2258
|
-
|
|
2259
|
-
| 0 | Success |
|
|
2260
|
-
| 1 | General error |
|
|
2261
|
-
| 2 | Usage / validation |
|
|
2262
|
-
| 3 | Auth (re-run \`ish login\`) |
|
|
2263
|
-
| 4 | Not found |
|
|
2264
|
-
| 5 | Transient — retryable
|
|
2358
|
+
| Code | Meaning | Common \`error_code\` values |
|
|
2359
|
+
|------|----------------------|------------------------------|
|
|
2360
|
+
| 0 | Success | — |
|
|
2361
|
+
| 1 | General error | \`server\`, \`client_error\` (uncategorized) |
|
|
2362
|
+
| 2 | Usage / validation | \`usage_error\` (Commander), \`validation_error\` (server), \`ConfirmationRequired\` |
|
|
2363
|
+
| 3 | Auth (re-run \`ish login\`) | \`auth_failed\`, missing-token errors |
|
|
2364
|
+
| 4 | Not found | \`not_found\` |
|
|
2365
|
+
| 5 | Transient — retryable | \`timeout\`, \`TunnelInactive\`, \`BotAuthError\`, DNS / network (\`ENOTFOUND\`, \`ECONNREFUSED\`) |
|
|
2265
2366
|
|
|
2266
2367
|
Use these to branch in scripts; do not parse the human stderr message.
|
|
2267
2368
|
|
|
2369
|
+
**Commander-level errors** (unknown command, missing required argument,
|
|
2370
|
+
missing required option) all exit **2** with \`error_code: "usage_error"\`.
|
|
2371
|
+
The suggestion field points at the right help target:
|
|
2372
|
+
|
|
2373
|
+
- Unknown command → \`Run \`ish --help\` for usage\` (the typo IS the
|
|
2374
|
+
command name — don't point at it; Commander also appends
|
|
2375
|
+
\`(Did you mean workspace?)\` for near-matches).
|
|
2376
|
+
- Missing argument / option → \`Run \`ish <command> --help\` for usage\`
|
|
2377
|
+
(substituted with the actual command, e.g. \`ish workspace --help\`).
|
|
2378
|
+
|
|
2379
|
+
**DNS / connection failures** (a wrong \`--api-url\`, a backend that's
|
|
2380
|
+
down, a captive portal) exit **5** so scripts retry rather than abort
|
|
2381
|
+
permanently. The underlying \`fetch\` \`TypeError\` is detected via its
|
|
2382
|
+
\`cause.code\` (\`ENOTFOUND\`, \`ECONNREFUSED\`, \`ECONNRESET\`,
|
|
2383
|
+
\`ETIMEDOUT\`, \`EAI_AGAIN\`).
|
|
2384
|
+
|
|
2268
2385
|
## Error envelope
|
|
2269
2386
|
|
|
2270
2387
|
When a command fails with \`--json\` (or piped stdout), the CLI prints
|
|
@@ -2294,6 +2411,12 @@ a structured error object on **stdout** and a human message on
|
|
|
2294
2411
|
(\`validation\`, \`auth\`, \`not_found\`, \`timeout\`, \`server\`,
|
|
2295
2412
|
\`network\`, …). \`retryable: true\` matches exit code 5.
|
|
2296
2413
|
|
|
2414
|
+
The \`status\` field carries the upstream **HTTP status code** when one
|
|
2415
|
+
is available (e.g. \`401\`, \`404\`, \`422\`). It is **omitted entirely**
|
|
2416
|
+
from envelopes that don't originate from an HTTP response (Commander
|
|
2417
|
+
parse errors, local validation failures, alias-resolution errors). Do
|
|
2418
|
+
not branch on \`status: 0\` — that value is never emitted as of 0.20.
|
|
2419
|
+
|
|
2297
2420
|
## Conventions
|
|
2298
2421
|
|
|
2299
2422
|
- Successful commands exit 0 and print one JSON object/array on stdout.
|
|
@@ -2343,6 +2466,184 @@ ish study results --human
|
|
|
2343
2466
|
When you genuinely need multiple fields in one parse pass, \`--json\` is
|
|
2344
2467
|
still the right tool — \`--get\` is for single-value capture, not for
|
|
2345
2468
|
reshaping output.
|
|
2469
|
+
|
|
2470
|
+
## Slicing study results
|
|
2471
|
+
|
|
2472
|
+
\`ish study results <id>\` accepts filter flags (\`--frame\`, \`--segment\`,
|
|
2473
|
+
\`--turn\`, \`--side\`, \`--assignment\`, \`--step\`, \`--sentiment\`,
|
|
2474
|
+
\`--actor\`, \`--iteration\`, \`--participant\`) and projection flags
|
|
2475
|
+
(\`--group-by iteration|frame|segment|turn|assignment|step\`). When any
|
|
2476
|
+
filter is passed, the envelope gains a \`totals_unfiltered\` field
|
|
2477
|
+
(\`{participant_count, interaction_count}\`) so an agent can sanity-check
|
|
2478
|
+
coverage: "matched 12 / 80 participants". A zero-match filter returns
|
|
2479
|
+
the stable envelope with \`participant_count: 0\` and exit code **0**
|
|
2480
|
+
(not 4) — slicing never errors on no-match.
|
|
2481
|
+
|
|
2482
|
+
\`--group-by\` is **router-gated by modality**: \`frame\` requires
|
|
2483
|
+
interactive, \`segment\` requires media (video / audio / text / document),
|
|
2484
|
+
\`turn\` requires chat. Mismatched filter flags (e.g. \`--segment 0\` on
|
|
2485
|
+
an interactive study) emit a stderr warning and are ignored — they
|
|
2486
|
+
don't error. Full worked examples in \`guides/slicing-results\`.
|
|
2487
|
+
`;
|
|
2488
|
+
const GUIDE_SLICING_RESULTS = `# guide: slicing study results
|
|
2489
|
+
|
|
2490
|
+
\`ish study results <id>\` returns a kitchen-sink envelope by default
|
|
2491
|
+
(every participant, every interaction, every interview answer). For
|
|
2492
|
+
narrower questions — *"what differed on the login screen across these
|
|
2493
|
+
five iterations?"*, *"who failed verify-email, and why?"*, *"frustrated
|
|
2494
|
+
reactions to segment 3 of the video"* — \`ish study results\` accepts
|
|
2495
|
+
**filter flags** (which interactions to keep) and **projection flags**
|
|
2496
|
+
(how to roll up what survives). Filters compose with AND across flags
|
|
2497
|
+
and OR within \`--sentiment\`. Filters and projections are pure
|
|
2498
|
+
client-side; no extra round trip beyond the standard study fetch.
|
|
2499
|
+
|
|
2500
|
+
## Filter flags
|
|
2501
|
+
|
|
2502
|
+
| Flag | Matches | Where it applies |
|
|
2503
|
+
|-------------------------------|-----------------------------------------------------------------------------------------------|------------------------------------------------------------------|
|
|
2504
|
+
| \`--frame <ref>\` | Interactions whose Frame name contains \`<ref>\` (case-insensitive). Also accepts a full Frame UUID, an \`f-…\` alias, or a \`frame_version_id\` UUID. | interactive — warn + ignore on chat / media |
|
|
2505
|
+
| \`--segment <ref>\` | Integer matches \`actions[0].data.segment_index\`; non-integer is a substring match against \`segment_label\`. | video, audio, text, document — warn + ignore elsewhere |
|
|
2506
|
+
| \`--turn <n>\` | Interactions whose \`actions[0].data.turn_index == n\`. | chat (external_chatbot + participant_pair) |
|
|
2507
|
+
| \`--side <a\|b>\` | Interactions whose parent assignment has \`side == a\` or \`side == b\`. | chat participant_pair — warn + ignore on other chat / non-chat |
|
|
2508
|
+
| \`--assignment <ref>\` | Assignment UUID, or substring match against the assignment name. | all |
|
|
2509
|
+
| \`--step <ref>\` | Filters \`participant_assignments[].step_results[]\` to verdicts matching the step id or name. | interactive + external_chatbot chat (steps live there) |
|
|
2510
|
+
| \`--sentiment <labels>\` | Comma-separated, case-insensitive label list (repeatable). Drops null-sentiment rows. | all |
|
|
2511
|
+
| \`--actor <ai\|human\|user>\` | Restrict by actor. | all |
|
|
2512
|
+
| \`--iteration <ref>\` | Iteration UUID or label (\`A\`, \`B\`, … case-insensitive). | all |
|
|
2513
|
+
| \`--participant <ref>\` | Participant UUID or \`pt-…\` alias. | all |
|
|
2514
|
+
| \`--include-unmatched\` | With \`--frame\`, keep degraded captures (\`frame_version_id: null\`) under a synthetic \`_unmatched\` bucket instead of dropping them. | interactive |
|
|
2515
|
+
| \`--include-evidence\` | With \`--step\`, also drop interactions not listed in any surviving \`step_results[].evidence_interaction_ids[]\`. | interactive + external_chatbot chat |
|
|
2516
|
+
|
|
2517
|
+
**Modality mismatch is not an error.** Pass \`--segment 0\` on an
|
|
2518
|
+
interactive study and the filter is ignored with a stderr warning.
|
|
2519
|
+
The exception is \`--group-by\` — see below.
|
|
2520
|
+
|
|
2521
|
+
## Projection flags (--group-by)
|
|
2522
|
+
|
|
2523
|
+
| Axis | Output shape | Modality |
|
|
2524
|
+
|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|
|
|
2525
|
+
| \`iteration\` | \`{study, slices: [{iteration_id, iteration_label, participant_count, interaction_count, sentiment, sample_comments, top_actions}, ...], totals_unfiltered, warnings}\` | all |
|
|
2526
|
+
| \`frame\` | \`[{frame_id, frame_label, interaction_count, sentiment_histogram, sample_comments, participant_aliases}, ...]\` | interactive (router errors on non-interactive) |
|
|
2527
|
+
| \`segment\` | \`[{segment_index, segment_label, interaction_count, sentiment_histogram, engagement_histogram, sample_comments}, ...]\` | media (router errors on non-media) |
|
|
2528
|
+
| \`turn\` | \`[{turn_index, interaction_count, sentiment_histogram, sample_replies, failures}, ...]\` | chat (router errors on non-chat) |
|
|
2529
|
+
| \`assignment\` | \`[{assignment_id, assignment_name, interaction_count, sentiment_histogram, step_completion}, ...]\` | all |
|
|
2530
|
+
| \`step\` | \`[{assignment_id, assignment_name, step_id, step_name, total, passed, inconclusive, failed, rate, participant_verdicts: [{participant_alias, verdict, reason, evidence_interaction_ids}, ...]}, ...]\` | interactive + external_chatbot chat |
|
|
2531
|
+
|
|
2532
|
+
\`--group-by\` is **mutually exclusive with \`--summary\` and
|
|
2533
|
+
\`--transcript\`**. \`--group-by frame\` on a chat study, \`--group-by
|
|
2534
|
+
turn\` on a video study, etc. error at the surface (exit 2) with a
|
|
2535
|
+
clear message before any IO.
|
|
2536
|
+
|
|
2537
|
+
## The empty-slice contract
|
|
2538
|
+
|
|
2539
|
+
A filter combination that matches zero interactions returns the
|
|
2540
|
+
**stable envelope shape** with:
|
|
2541
|
+
|
|
2542
|
+
- \`participant_count: 0\`
|
|
2543
|
+
- \`totals_unfiltered: {participant_count: <N>, interaction_count: <M>}\` populated
|
|
2544
|
+
- exit code **0** (not 4)
|
|
2545
|
+
|
|
2546
|
+
\`totals_unfiltered\` is the agent's sanity check: *"my filter matched
|
|
2547
|
+
0 of 80 participants — is the filter too tight, or did the run not
|
|
2548
|
+
produce data?"*. The shape never collapses to \`null\` or a different
|
|
2549
|
+
envelope; \`--get participant_count\` is always safe.
|
|
2550
|
+
|
|
2551
|
+
## Worked examples
|
|
2552
|
+
|
|
2553
|
+
\`\`\`bash
|
|
2554
|
+
# What differed on the login screen across the five iterations?
|
|
2555
|
+
ish study results s-b2c --frame login --group-by iteration
|
|
2556
|
+
|
|
2557
|
+
# Frustrated reactions to segment 3 of the video
|
|
2558
|
+
ish study results s-b2c --segment 3 --sentiment Frustrated
|
|
2559
|
+
|
|
2560
|
+
# Who failed the "verify email" step, and why?
|
|
2561
|
+
ish study results s-b2c --assignment "Sign up" --step verify-email --group-by step
|
|
2562
|
+
|
|
2563
|
+
# Chat participant_pair: only side A turn 4
|
|
2564
|
+
ish study results s-b2c --side a --turn 4
|
|
2565
|
+
|
|
2566
|
+
# Surface degraded captures (frame_version_id: null) under a "_unmatched" bucket:
|
|
2567
|
+
ish study results s-b2c --frame login --include-unmatched --group-by frame
|
|
2568
|
+
|
|
2569
|
+
# Narrow the lean summary to a slice:
|
|
2570
|
+
ish study results s-b2c --summary --frame checkout --json
|
|
2571
|
+
\`\`\`
|
|
2572
|
+
|
|
2573
|
+
## Combining filters
|
|
2574
|
+
|
|
2575
|
+
Filters compose with **AND across flags** and **OR within
|
|
2576
|
+
\`--sentiment\`**. \`--frame login --sentiment Frustrated,Confused\`
|
|
2577
|
+
means "interactions on the login frame whose sentiment is Frustrated
|
|
2578
|
+
OR Confused". \`--summary\` is orthogonal to filters and narrows the
|
|
2579
|
+
summary over the filtered set. \`--transcript\` is single-participant
|
|
2580
|
+
and **errors when any filter or \`--group-by\` is set** (exit 2).
|
|
2581
|
+
|
|
2582
|
+
## Defensive handling of nullable fields
|
|
2583
|
+
|
|
2584
|
+
- \`interaction.sentiment\` is nullable (chat failure stubs,
|
|
2585
|
+
pre-sentiment rows). Dropped **only** when \`--sentiment\` is set; kept
|
|
2586
|
+
by every other filter.
|
|
2587
|
+
- \`interaction.frame_version_id\` is nullable on interactive studies
|
|
2588
|
+
(degraded captures, ~12% on a failing iteration). Dropped by
|
|
2589
|
+
\`--frame\` unless \`--include-unmatched\` is passed; surfaced as a
|
|
2590
|
+
\`_unmatched\` bucket in \`--group-by frame\`.
|
|
2591
|
+
- Chat \`bot_reply.failure\` rows are kept in the default envelope,
|
|
2592
|
+
dropped by \`--sentiment\` (they have \`sentiment: null\`), kept by
|
|
2593
|
+
\`--actor\`, visible in \`--group-by turn\` under a \`failures\`
|
|
2594
|
+
counter.
|
|
2595
|
+
|
|
2596
|
+
## --frame resolution
|
|
2597
|
+
|
|
2598
|
+
\`--frame login\` walks the frame list returned by
|
|
2599
|
+
\`GET /studies/{id}/frames\` and matches **case-insensitive substring**
|
|
2600
|
+
against the frame name. Other accepted shapes:
|
|
2601
|
+
|
|
2602
|
+
- \`--frame 6ec…\` — full Frame UUID (exact match)
|
|
2603
|
+
- \`--frame f-6ec\` — short alias resolved via \`alias-store\`
|
|
2604
|
+
- \`--frame 7ec…\` — a \`frame_version_id\` UUID (matches only that version)
|
|
2605
|
+
|
|
2606
|
+
Ambiguous substring (matches >1 frame) errors with the candidate list:
|
|
2607
|
+
|
|
2608
|
+
\`\`\`
|
|
2609
|
+
ish study results s-b2c --frame log
|
|
2610
|
+
# Error: --frame "log" is ambiguous — matched 2 frames: Login, Logout.
|
|
2611
|
+
# Use a more specific substring, a full Frame UUID, or an \`f-…\` alias.
|
|
2612
|
+
\`\`\`
|
|
2613
|
+
|
|
2614
|
+
No match at all errors and lists the available frame names.
|
|
2615
|
+
|
|
2616
|
+
## Common --get paths on a sliced envelope
|
|
2617
|
+
|
|
2618
|
+
\`\`\`
|
|
2619
|
+
# Sanity-check coverage:
|
|
2620
|
+
--get totals_unfiltered.participant_count
|
|
2621
|
+
--get totals_unfiltered.interaction_count
|
|
2622
|
+
|
|
2623
|
+
# Per-iteration projection:
|
|
2624
|
+
--get slices.iteration_label # one label per line
|
|
2625
|
+
--get slices.0.participant_count
|
|
2626
|
+
--get slices.0.sentiment
|
|
2627
|
+
|
|
2628
|
+
# Per-frame / per-segment / per-turn (bare array):
|
|
2629
|
+
--get 0.frame_label
|
|
2630
|
+
--get 0.segment_index
|
|
2631
|
+
--get 0.sentiment_histogram
|
|
2632
|
+
|
|
2633
|
+
# Per-step:
|
|
2634
|
+
--get 0.rate
|
|
2635
|
+
--get 0.participant_verdicts.verdict # one verdict per participant
|
|
2636
|
+
\`\`\`
|
|
2637
|
+
|
|
2638
|
+
## Related
|
|
2639
|
+
|
|
2640
|
+
- \`concepts/study\` — the parent artifact whose results are being sliced.
|
|
2641
|
+
- \`concepts/assignment\` — defines the steps that \`--step\` and
|
|
2642
|
+
\`--group-by step\` filter against.
|
|
2643
|
+
- \`reference/json-mode\` — display vs capture vs chain output rules
|
|
2644
|
+
(\`--get\`, \`--fields\`, exit codes).
|
|
2645
|
+
- \`reference/aliases\` — \`s-…\` for studies, \`pt-…\` for participants,
|
|
2646
|
+
\`f-…\` for frames. Any UUID-accepting flag also accepts the alias.
|
|
2346
2647
|
`;
|
|
2347
2648
|
const GUIDE_FIRST_STUDY = `# guide: your first study, end to end
|
|
2348
2649
|
|
|
@@ -2411,13 +2712,19 @@ The CLI keeps a small amount of session state in \`~/.ish/config.json\`
|
|
|
2411
2712
|
(or wherever \`ISH_HOME\` points) so commands don't need to repeat IDs:
|
|
2412
2713
|
|
|
2413
2714
|
- \`access_token\` / \`refresh_token\` — the OAuth pair from \`ish login\`.
|
|
2414
|
-
- \`workspace\`
|
|
2415
|
-
- \`study\`
|
|
2416
|
-
- \`ask\`
|
|
2715
|
+
- \`workspace\` — set by \`ish workspace use <id>\`.
|
|
2716
|
+
- \`study\` — set by \`ish study use <id>\` (or implicitly by \`ish study create\`).
|
|
2717
|
+
- \`ask\` — set by \`ish ask use <id>\` (or implicitly by \`ish ask create\`).
|
|
2718
|
+
- \`chat_endpoint\` — set by \`ish chat endpoint use <id>\`.
|
|
2417
2719
|
|
|
2418
2720
|
Most commands fall back to these when their corresponding flag is
|
|
2419
2721
|
omitted (\`--workspace\`, \`--study\`, \`--ask\`).
|
|
2420
2722
|
|
|
2723
|
+
**\`workspace\` is the parent** of \`study\`, \`ask\`, and \`chat_endpoint\` —
|
|
2724
|
+
all three are scoped to a single workspace. Switching workspaces
|
|
2725
|
+
(\`ish workspace use <other>\`) clears all three to avoid cross-workspace
|
|
2726
|
+
footguns, and the CLI prints a one-line stderr note when it does so.
|
|
2727
|
+
|
|
2421
2728
|
## Inspecting active context
|
|
2422
2729
|
|
|
2423
2730
|
\`ish status\` (alias: \`ish whoami\`) is the canonical way to see what's
|
|
@@ -2430,7 +2737,8 @@ ish status
|
|
|
2430
2737
|
# User: you@example.com (token valid, expires in 47m)
|
|
2431
2738
|
# Workspace: Onboarding revamp (w-6ec)
|
|
2432
2739
|
# Study: —
|
|
2433
|
-
# Ask: a-6ec
|
|
2740
|
+
# Ask: tagline AB (a-6ec)
|
|
2741
|
+
# Chat ep: —
|
|
2434
2742
|
# Home: /home/you/.ish
|
|
2435
2743
|
# API: https://api.ishlabs.io
|
|
2436
2744
|
\`\`\`
|
|
@@ -2439,12 +2747,13 @@ JSON shape (\`ish status --json\` or piped):
|
|
|
2439
2747
|
|
|
2440
2748
|
\`\`\`json
|
|
2441
2749
|
{
|
|
2442
|
-
"user":
|
|
2443
|
-
"workspace":
|
|
2444
|
-
"study":
|
|
2445
|
-
"ask":
|
|
2446
|
-
"
|
|
2447
|
-
"
|
|
2750
|
+
"user": { "email": "...", "token_valid": true, "expires_in_seconds": 2820 },
|
|
2751
|
+
"workspace": { "id": "...", "alias": "w-6ec", "name": "Onboarding revamp" },
|
|
2752
|
+
"study": null,
|
|
2753
|
+
"ask": { "id": "...", "alias": "a-6ec", "name": "tagline AB" },
|
|
2754
|
+
"chat_endpoint": null,
|
|
2755
|
+
"api_url": "https://api.ishlabs.io",
|
|
2756
|
+
"home": "/home/you/.ish"
|
|
2448
2757
|
}
|
|
2449
2758
|
\`\`\`
|
|
2450
2759
|
|
|
@@ -2453,19 +2762,83 @@ JSON shape (\`ish status --json\` or piped):
|
|
|
2453
2762
|
\`ish login\`. Safe to run unconditionally at the start of any
|
|
2454
2763
|
script or agent session.
|
|
2455
2764
|
|
|
2765
|
+
### \`ish login\` is idempotent
|
|
2766
|
+
|
|
2767
|
+
When you already have a valid saved token, \`ish login\` short-circuits
|
|
2768
|
+
with a friendly "Already logged in" message and **does not** open a new
|
|
2769
|
+
browser tab or register a fresh OAuth client. Use \`--force\` (or \`-f\`)
|
|
2770
|
+
to bypass the guard — typical reason is switching accounts.
|
|
2771
|
+
|
|
2772
|
+
\`\`\`bash
|
|
2773
|
+
ish login # no-op when already authenticated
|
|
2774
|
+
ish login --force # always re-run the browser flow
|
|
2775
|
+
\`\`\`
|
|
2776
|
+
|
|
2777
|
+
The short-circuit returns a structured envelope under \`--json\`:
|
|
2778
|
+
|
|
2779
|
+
\`\`\`json
|
|
2780
|
+
{
|
|
2781
|
+
"message": "Already logged in",
|
|
2782
|
+
"email": "you@example.com",
|
|
2783
|
+
"token_valid": true,
|
|
2784
|
+
"expires_in_seconds": 2820,
|
|
2785
|
+
"hint": "Pass --force to re-run the browser flow (e.g. to switch accounts)."
|
|
2786
|
+
}
|
|
2787
|
+
\`\`\`
|
|
2788
|
+
|
|
2789
|
+
### Orphan / stale active refs
|
|
2790
|
+
|
|
2791
|
+
If an active ref points at an entity that no longer exists or moved
|
|
2792
|
+
workspace, \`status\` surfaces a \`warning\` field on that ref (instead
|
|
2793
|
+
of silently dropping the \`name\`). Each warned ref also gets a \`hint\`
|
|
2794
|
+
field with the exact command to clear or replace it:
|
|
2795
|
+
|
|
2796
|
+
\`\`\`json
|
|
2797
|
+
{
|
|
2798
|
+
"study": {
|
|
2799
|
+
"id": "...",
|
|
2800
|
+
"alias": "s-74d",
|
|
2801
|
+
"warning": "orphan — entity no longer exists in this workspace",
|
|
2802
|
+
"hint": "Active study is no longer accessible (deleted, moved workspace, or auth issue). Use \`ish study use <id>\` to switch, or \`ish study use --clear\` to drop."
|
|
2803
|
+
}
|
|
2804
|
+
}
|
|
2805
|
+
\`\`\`
|
|
2806
|
+
|
|
2807
|
+
In human output the warning prints as \`⚠ ...\` under the row and a
|
|
2808
|
+
follow-up line shows the hint.
|
|
2809
|
+
|
|
2456
2810
|
## Setting / clearing active context
|
|
2457
2811
|
|
|
2458
2812
|
\`\`\`bash
|
|
2459
|
-
ish workspace use w-6ec # set
|
|
2460
|
-
ish workspace use --clear # clear
|
|
2813
|
+
ish workspace use w-6ec # set (also clears active study/ask/chat_endpoint if workspace changed)
|
|
2814
|
+
ish workspace use --clear # clear workspace + all workspace-scoped children
|
|
2461
2815
|
|
|
2462
2816
|
ish study use s-b2c
|
|
2463
2817
|
ish study use --clear
|
|
2464
2818
|
|
|
2465
2819
|
ish ask use a-6ec
|
|
2466
2820
|
ish ask use --clear
|
|
2821
|
+
|
|
2822
|
+
ish chat endpoint use ep-abc
|
|
2823
|
+
ish chat endpoint use --clear
|
|
2467
2824
|
\`\`\`
|
|
2468
2825
|
|
|
2826
|
+
### Auto-activation on create
|
|
2827
|
+
|
|
2828
|
+
\`ish study create\`, \`ish ask create\`, and \`ish workspace use\` all
|
|
2829
|
+
update the active context as a side-effect (so the natural next command
|
|
2830
|
+
— \`ish iteration create --study <new>\`, \`ish ask add-round\`, etc. —
|
|
2831
|
+
"just works" without re-typing the ID). The CLI **emits a one-line
|
|
2832
|
+
stderr notice** when this happens; consumers piping stdout get the new
|
|
2833
|
+
record while the auto-activate is visible to operators.
|
|
2834
|
+
|
|
2835
|
+
### Cleanup on delete
|
|
2836
|
+
|
|
2837
|
+
\`ish workspace delete\`, \`ish study delete\`, \`ish ask delete\`, and
|
|
2838
|
+
\`ish chat endpoint delete\` automatically clear matching active refs
|
|
2839
|
+
from \`~/.ish/config.json\` so subsequent commands don't render orphans.
|
|
2840
|
+
\`workspace delete\` also clears all workspace-scoped children.
|
|
2841
|
+
|
|
2469
2842
|
## Overriding without persisting
|
|
2470
2843
|
|
|
2471
2844
|
Every read command accepts \`--workspace <id>\`, \`--study <id>\`, or
|
|
@@ -3863,6 +4236,12 @@ const PAGES = [
|
|
|
3863
4236
|
description: "Login → workspace → people → study → iteration → run → results.",
|
|
3864
4237
|
body: GUIDE_FIRST_STUDY,
|
|
3865
4238
|
},
|
|
4239
|
+
{
|
|
4240
|
+
slug: "guides/slicing-results",
|
|
4241
|
+
title: "guide: slicing study results by frame / segment / turn / sentiment",
|
|
4242
|
+
description: "Filter and project `ish study results` — --frame, --segment, --turn, --side, --assignment, --step, --sentiment, --actor, --iteration, --participant; --group-by iteration|frame|segment|turn|assignment|step; totals_unfiltered + empty-slice contract.",
|
|
4243
|
+
body: GUIDE_SLICING_RESULTS,
|
|
4244
|
+
},
|
|
3866
4245
|
{
|
|
3867
4246
|
slug: "guides/chat",
|
|
3868
4247
|
title: "guide: chat-modality studies",
|