@ishlabs/cli 0.9.0 → 0.11.0

package/dist/commands/study-run.js

@@ -11,7 +11,7 @@ import * as readline from "node:readline/promises";
 import { withClient, getWebUrl, terminalLink, resolveWorkspace, resolveStudy, parseWaitTimeout, resolveAudienceProfileIds, addAudienceFilterFlags, hasAudienceFlags, } from "../lib/command-helpers.js";
 import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
 import { output, formatSimulationPoll } from "../lib/output.js";
-import { MEDIA_MODALITIES } from "../lib/types.js";
+import { isMediaModality, isChatModality, iterationHasContent, describeRequiredContentFlag, } from "../lib/modality.js";
 import { runLocalSimulations } from "../lib/local-sim/loop.js";
 import { ensureBrowser } from "../lib/local-sim/install.js";
 function parseMaxInteractions(value) {
@@ -26,45 +26,53 @@ function parseSlowMo(value) {
         throw new Error(`Invalid --slow-mo value: ${value}`);
     return n;
 }
-function isMediaModality(modality) {
-    return !!modality && MEDIA_MODALITIES.includes(modality);
-}
 /**
- * Pattern F (Issue #9): an iteration without content is a footgun.
- * `ish study generate` auto-creates an empty iteration "A" that becomes
- * `study run`'s default target unless `--iteration` is explicit; agents
- * silently dispatch against it. This predicate is the gate that lets
- * study run refuse with a clear suggestion instead of silent failure.
+ * M8 / M9 / Pattern G: typed wait-timeout error so `--wait` paths can
+ * produce a structured envelope (`error_code: "wait_timeout"`, exit 5
+ * transient) distinct from the generic timeout/network/server errors
+ * that the api-client wrapper produces. Carries the in-flight progress
+ * (testers done / total) so `study wait` always emits final state JSON
+ * even when it bails on the timer.
  */
-function iterationHasContent(details, modality) {
-    if (!details || typeof details !== "object")
-        return false;
-    if (modality === "text") {
-        return typeof details.content_text === "string" && details.content_text.length > 0;
-    }
-    if (modality === "video" || modality === "audio" || modality === "document") {
-        return typeof details.content_url === "string" && details.content_url.length > 0;
-    }
-    if (modality === "image") {
-        const urls = details.image_urls;
-        if (Array.isArray(urls))
-            return urls.length > 0;
-        if (typeof urls === "string")
-            return urls.length > 0;
-        return false;
+export class WaitTimeoutError extends Error {
+    progress;
+    error_code = "wait_timeout";
+    retryable = true;
+    constructor(message, progress) {
+        super(message);
+        this.progress = progress;
+        this.name = "WaitTimeoutError";
     }
-    // interactive (default)
-    return typeof details.url === "string" && details.url.length > 0;
 }
-function describeRequiredContentFlag(modality) {
-    if (modality === "text")
-        return "--content-text <text-or-@file>";
-    if (modality === "video" || modality === "audio" || modality === "document") {
-        return "--content-url <url-or-file>";
+/**
+ * M13 / Pattern J: collapse the N near-duplicate `simulations[]` rows the
+ * batch start endpoint returns (one per tester, all sharing study_id) into a
+ * single batch entry with `tester_ids[]` + `tester_aliases[]` + `job_ids[]`.
+ * Agents always need the per-tester ids, but the surrounding scaffolding
+ * (study_id, message) is shared so repeating it N times costs context for no
+ * extra signal. Falls back to the raw rows if the response shape is unfamiliar.
+ */
+function dedupeSimulations(simResults) {
+    if (!Array.isArray(simResults) || simResults.length === 0)
+        return [];
+    const studyIds = new Set(simResults.map((r) => r.study_id));
+    if (studyIds.size !== 1) {
+        // Mixed batches aren't expected today, but if the backend ever returns
+        // them, project per-row to keep the data lossless rather than silently
+        // collapsing across studies.
+        return simResults.map((r) => ({ ...r }));
     }
-    if (modality === "image")
-        return "--image-urls <comma-separated>";
-    return "--url <url>";
+    const studyId = simResults[0].study_id;
+    return [
+        {
+            study_id: studyId,
+            tester_ids: simResults.map((r) => r.tester_id),
+            tester_aliases: simResults.map((r) => tagAlias(ALIAS_PREFIX.tester, String(r.tester_id))),
+            job_ids: simResults.map((r) => r.job_id ?? null),
+            count: simResults.length,
+            message: simResults[0].message,
+        },
+    ];
 }
 const POLL_INTERVAL_MS = 5_000;
 const TERMINAL_STATUSES = new Set(["completed", "errored", "failed", "cancelled", "canceled"]);
@@ -74,13 +82,16 @@ function flattenTesterStatuses(iterations, only) {
         for (const t of iteration.testers ?? []) {
             if (only && !only.has(t.id))
                 continue;
+            // Pattern A (cli half): backend now reports per-tester crash detail at
+            // `error_message`. Keep `error` / `failure_reason` as legacy fallbacks
+            // until every backend deploy is on the new contract.
+            const errorMessage = t.error_message || t.error || t.failure_reason || null;
             rows.push({
                 id: t.id,
                 status: t.status,
                 tester_name: t.tester_profile?.name || "Unknown",
                 interaction_count: Array.isArray(t.interactions) ? t.interactions.length : 0,
-                ...(t.error && { error: t.error }),
-                ...(t.failure_reason && { error: t.failure_reason }),
+                ...(errorMessage && { error_message: errorMessage }),
             });
         }
     }
@@ -109,8 +120,16 @@ async function pollStudyUntilDone(client, opts) {
             return { rows, isMedia };
         }
         if (Date.now() - start > opts.timeoutMs) {
-            throw new Error(`Timed out after ${Math.round(opts.timeoutMs / 1000)}s waiting for simulations. ` +
-                `Run \`ish study poll --study ${opts.studyId}\` to check status.`);
+            throw new WaitTimeoutError(`Timed out after ${Math.round(opts.timeoutMs / 1000)}s waiting for simulations. ` +
+                `${done}/${total} done. Run \`ish study poll --study ${opts.studyId}\` to check status.`, {
+                study_id: opts.studyId,
+                ...(opts.iterationId && { iteration_id: opts.iterationId }),
+                timeout_seconds: Math.round(opts.timeoutMs / 1000),
+                done,
+                total,
+                pending: total - done,
+                rows,
+            });
         }
         await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
     }
@@ -143,9 +162,12 @@ export function attachStudyRunCommands(study) {
     })
         .option("--config <id>", "Simulation config ID (required for media unless every profile has one)")
         .option("--max-interactions <n>", "Max interactions per tester")
+        .option("--max-turns <n>", "Max conversation turns per tester (chat studies only)")
+        .option("--early-termination", "Allow chat agent to end the conversation early when goals are met (chat studies only)")
         .option("--language <lang>", "Language code (e.g. en, sv)")
         .option("--wait", "Wait for all simulations to reach a terminal state before returning")
         .option("--timeout <s>", "Wait timeout in seconds (default 300; only with --wait)")
+        .option("--dispatch-timeout <s>", "Per-POST timeout in seconds for the create-testers + dispatch calls (default 120). Bump if `study run` times out client-side after seeding testers but before dispatch — those testers exist server-side and are surfaced under `seeded_but_not_dispatched_*` in the error envelope so the agent can resume.")
         .option("-y, --yes", "Skip confirmation prompt")
         // Local simulation options
         .option("--local", "Run simulation with local browser (Playwright) instead of remote")
@@ -197,11 +219,49 @@ Examples:
             const log = (msg) => { if (!globals.quiet)
                 console.error(msg); };
             const resolvedWorkspace = resolveWorkspace(opts.workspace);
-            const resolvedStudy = resolveStudy(opts.study);
+            // Pattern A (Sprint 2): when `--iteration` is passed without
+            // `--study`, derive the parent study from the iteration itself
+            // (mirroring `study wait`). The iteration's `study_id` is
+            // authoritative — falling back to the active study config here
+            // would silently target the wrong study when the active context
+            // has drifted, surfacing as the misleading "Iteration X not found
+            // on this study" error N1 hit. If `--study` is also passed, they
+            // must agree.
+            let resolvedStudy;
+            if (opts.iteration && !opts.study) {
+                // The iteration's `study_id` is authoritative. Falling back to
+                // the active study config here would silently target the wrong
+                // study when the active context has drifted (Pattern A from N1)
+                // and surface as the misleading "Iteration X not found on this
+                // study" error. If the lookup fails (404, transport), surface
+                // the lookup error directly rather than masking with a stale
+                // active-study read.
+                const iterId = resolveId(opts.iteration);
+                const iter = await client.get(`/iterations/${iterId}`);
+                if (!iter.study_id) {
+                    throw new Error(`Iteration ${opts.iteration} has no study_id.`);
+                }
+                resolvedStudy = iter.study_id;
+            }
+            else {
+                resolvedStudy = resolveStudy(opts.study);
+            }
+            // B7: per-POST dispatch timeout (default 120s — long enough to survive
+            // a slow-cold backend, short enough that an agent doesn't sit blind
+            // for many minutes). Applied to both the testers/batch POST AND the
+            // simulation start POST so the seed + dispatch budget is the same.
+            const dispatchTimeoutMs = opts.dispatchTimeout
+                ? Math.max(1, parseInt(opts.dispatchTimeout, 10)) * 1000
+                : 120_000;
+            if (opts.dispatchTimeout && (Number.isNaN(parseInt(opts.dispatchTimeout, 10))
+                || parseInt(opts.dispatchTimeout, 10) < 1)) {
+                throw new Error(`--dispatch-timeout must be a positive integer (seconds), got "${opts.dispatchTimeout}".`);
+            }
             // Step 0: Fetch study (with its iterations + their existing testers)
             const study = await client.get(`/studies/${resolvedStudy}`);
             const modality = study.modality || "interactive";
             const isMedia = isMediaModality(modality);
+            const isChat = isChatModality(modality);
             if (!study.assignments || study.assignments.length === 0) {
                 throw new Error("Study has no assignments. Add tasks with --assignments when creating the study, or use `ish study generate`.");
             }
@@ -267,10 +327,11 @@ Examples:
                 throw new Error(`Iteration "${iterationLabel}" has no testers and no audience flags were given. ` +
                     "Pass --profile <ids>, or filter flags (--country, --gender, --min-age, --max-age, --search, --visibility) with --sample <N> or --all.");
             }
-            // Step 3: Resolve simulation config (per-profile fallback for media)
+            // Step 3: Resolve simulation config (per-profile fallback for
+            // media + chat, both of which require a config_id per batch item)
             const resolvedConfigOverride = opts.config ? resolveId(opts.config) : undefined;
             const profileConfigMap = new Map();
-            if (isMedia && !resolvedConfigOverride) {
+            if ((isMedia || isChat) && !resolvedConfigOverride) {
                 for (const pid of profileIds) {
                     const profile = await client.get(`/tester-profiles/${pid}`);
                     if (profile.simulation_config_id) {
@@ -291,7 +352,17 @@ Examples:
                 log(`    Modality:       ${modality}`);
                 if (study.content_type)
                     log(`    Content type:   ${study.content_type}`);
-                if (!isMedia) {
+                if (isChat) {
+                    const epId = typeof iteration.details?.chatbot_endpoint_id === "string"
+                        ? iteration.details.chatbot_endpoint_id : undefined;
+                    if (epId)
+                        log(`    Endpoint:       ${epId}`);
+                    if (opts.maxTurns)
+                        log(`    Max turns:      ${opts.maxTurns}`);
+                    if (opts.earlyTermination)
+                        log(`    Early term:     enabled`);
+                }
+                else if (!isMedia) {
                     log(`    Platform:       ${detailsView.platform || "browser"}`);
                     log(`    Screen format:  ${detailsView.screenFormat || "desktop"}`);
                     if (detailsView.url)
@@ -334,16 +405,16 @@ Examples:
                     tester_type: "ai",
                     status: "draft",
                     ...(opts.language && { language: opts.language }),
-                    ...(!isMedia && { platform: detailsView.platform || "browser" }),
+                    ...(!isMedia && !isChat && { platform: detailsView.platform || "browser" }),
                 }));
                 log(`Creating ${testerInputs.length} tester${testerInputs.length > 1 ? "s" : ""}...`);
-                const batchResult = await client.post(`/iterations/${iterationId}/testers/batch`, { testers: testerInputs });
+                const batchResult = await client.post(`/iterations/${iterationId}/testers/batch`, { testers: testerInputs }, { timeout: dispatchTimeoutMs });
                 createdTesters = batchResult.testers;
                 log(`Created ${createdTesters.length} tester${createdTesters.length > 1 ? "s" : ""}`);
             }
             // Step 6: Dispatch
             if (opts.local) {
-                if (isMedia) {
+                if (isMedia || isChat) {
                     throw new Error("Local mode is only supported for interactive simulations.");
                 }
                 const testerNameMap = new Map();
@@ -386,18 +457,58 @@ Examples:
             }
             log(`Starting ${createdTesters.length} simulation${createdTesters.length > 1 ? "s" : ""}...`);
             let simResults;
-            if (isMedia) {
+            // B7 / Pattern G: tag any failure during the dispatch POST with the
+            // testers that already exist server-side. The client doesn't know
+            // whether the backend processed our POST or never received it, so
+            // re-running `study run` would double-seed (we'd create another
+            // batch of testers on top of these). Instead, surface the seeded
+            // ids in the error envelope under `seeded_but_not_dispatched_ids`
+            // / `_aliases` so the agent can resume by polling them or calling
+            // `study tester delete <id>` then retrying.
+            const dispatchAttempt = async (fn) => {
+                try {
+                    return await fn();
+                }
+                catch (err) {
+                    if (err instanceof Error) {
+                        const tagged = err;
+                        tagged.seeded_but_not_dispatched_ids = createdTesters.map((t) => t.id);
+                        tagged.seeded_but_not_dispatched_aliases = createdTesters.map((t) => tagAlias(ALIAS_PREFIX.tester, String(t.id)));
+                    }
+                    throw err;
+                }
+            };
+            if (isChat) {
+                const chatBatchItems = createdTesters.map((t, i) => ({
+                    study_id: resolvedStudy,
+                    tester_id: t.id,
+                    config_id: resolvedConfigOverride || profileConfigMap.get(profileIds[i]),
+                    ...(opts.language && { language: opts.language }),
+                }));
+                const maxTurns = opts.maxTurns ? parseInt(opts.maxTurns, 10) : undefined;
+                if (opts.maxTurns !== undefined && (Number.isNaN(maxTurns) || maxTurns < 1)) {
+                    throw new Error(`Invalid --max-turns value: ${opts.maxTurns}`);
+                }
+                const simResult = await dispatchAttempt(() => client.post("/simulation/chat/start/batch", {
+                    product_id: resolvedWorkspace,
+                    simulations: chatBatchItems,
+                    ...(maxTurns !== undefined && { max_turns: maxTurns }),
+                    ...(opts.earlyTermination && { early_termination: true }),
+                }, { timeout: dispatchTimeoutMs }));
+                simResults = simResult.results;
+            }
+            else if (isMedia) {
                 const mediaBatchItems = createdTesters.map((t, i) => ({
                     study_id: resolvedStudy,
                     tester_id: t.id,
                     config_id: resolvedConfigOverride || profileConfigMap.get(profileIds[i]),
                     ...(opts.language && { language: opts.language }),
                 }));
-                const simResult = await client.post("/simulation/media/start/batch", {
+                const simResult = await dispatchAttempt(() => client.post("/simulation/media/start/batch", {
                     product_id: resolvedWorkspace,
                     simulations: mediaBatchItems,
                     ...(opts.maxInteractions && { max_interactions: parseMaxInteractions(opts.maxInteractions) }),
-                }, { timeout: 60_000 });
+                }, { timeout: dispatchTimeoutMs }));
                 simResults = simResult.results;
             }
             else {
@@ -408,14 +519,14 @@ Examples:
                     ...(opts.language && { language: opts.language }),
                     ...(detailsView.locale && { locale: detailsView.locale }),
                 }));
-                const simResult = await client.post("/simulation/interactive/start/batch", {
+                const simResult = await dispatchAttempt(() => client.post("/simulation/interactive/start/batch", {
                     product_id: resolvedWorkspace,
                     simulations: simItems,
                     platform: detailsView.platform || "browser",
                     ...(detailsView.url && { url: detailsView.url }),
                     screen_format: detailsView.screenFormat || "desktop",
                     ...(opts.maxInteractions && { max_interactions: parseMaxInteractions(opts.maxInteractions) }),
-                }, { timeout: 60_000 });
+                }, { timeout: dispatchTimeoutMs }));
                 simResults = simResult.results;
             }
             if (!opts.wait) {
@@ -430,7 +541,7 @@ Examples:
                         testers: testersOut,
                         tester_ids: testersOut.map((t) => t.id),
                         tester_aliases: testersOut.map((t) => t.alias),
-                        simulations: simResults,
+                        simulations: dedupeSimulations(simResults),
                     }, true);
                 }
                 else {
@@ -466,7 +577,7 @@ Examples:
                     testers: testersOut,
                     tester_ids: testersOut.map((t) => t.id),
                     tester_aliases: testersOut.map((t) => t.alias),
-                    simulations: simResults,
+                    simulations: dedupeSimulations(simResults),
                     results: rows,
                 }, true);
             }
@@ -490,32 +601,22 @@ Examples:
             if (testerId) {
                 const data = await client.get(`/simulation/status/${resolveId(testerId)}`);
                 output(data, globals.json);
+                return;
             }
-            else if (opts.study) {
-                const rid = resolveId(opts.study);
-                const study = await client.get(`/studies/${rid}`);
-                const isMedia = isMediaModality(study.modality);
-                const allTesters = [];
-                for (const iteration of study.iterations || []) {
-                    for (const tester of iteration.testers || []) {
-                        allTesters.push({
-                            id: tester.id,
-                            status: tester.status,
-                            tester_name: tester.tester_profile?.name || "Unknown",
-                            interaction_count: Array.isArray(tester.interactions) ? tester.interactions.length : 0,
-                            ...(tester.error && { error: tester.error }),
-                            ...(tester.failure_reason && { error: tester.failure_reason }),
-                        });
-                    }
-                }
-                formatSimulationPoll(allTesters, globals.json, isMedia);
-                if (!globals.json && study.product_id) {
-                    const url = getWebUrl(globals, `/${study.product_id}/${rid}/timeline`);
-                    console.error(`\n  ${terminalLink(url, "Open in browser ↗")}\n`);
-                }
-            }
-            else {
-                throw new Error("Provide a tester_id argument or --study flag");
+            // M7: fall back to the active study (set by `ish study use`) when
+            // neither a tester_id nor an explicit --study is passed. This brings
+            // poll into parity with `study results` / `study wait` / `study run`,
+            // all of which already honor the active study. Without the fallback
+            // an agent that ran `study use s-...` then `study poll` would get a
+            // confusing "Provide a tester_id argument or --study flag" error.
+            const rid = resolveStudy(opts.study);
+            const study = await client.get(`/studies/${rid}`);
+            const isMedia = isMediaModality(study.modality);
+            const allTesters = flattenTesterStatuses(study.iterations);
+            formatSimulationPoll(allTesters, globals.json, isMedia);
+            if (!globals.json && study.product_id) {
+                const url = getWebUrl(globals, `/${study.product_id}/${rid}/timeline`);
+                console.error(`\n  ${terminalLink(url, "Open in browser ↗")}\n`);
             }
         });
     });
@@ -535,8 +636,9 @@ Examples:
             if (testerId) {
                 const start = Date.now();
                 let lastStatus = "";
+                const resolvedTester = resolveId(testerId);
                 while (true) {
-                    const data = await client.get(`/simulation/status/${resolveId(testerId)}`, undefined, { timeout: 60_000 });
+                    const data = await client.get(`/simulation/status/${resolvedTester}`, undefined, { timeout: 60_000 });
                     const status = String(data.status ?? "unknown");
                     if (!globals.quiet && status !== lastStatus) {
                         process.stderr.write(`  ${status}\n`);
@@ -547,7 +649,24 @@ Examples:
                         return;
                     }
                     if (Date.now() - start > timeoutMs) {
-                        throw new Error(`Timed out after ${Math.round(timeoutMs / 1000)}s waiting for tester ${testerId}.`);
+                        // M8 + M9 (per-tester wait): structured wait_timeout with the
+                        // current status as `progress.rows[0]` so `study wait <id>`
+                        // always emits machine-readable final state.
+                        throw new WaitTimeoutError(`Timed out after ${Math.round(timeoutMs / 1000)}s waiting for tester ${testerId}. Last status: ${status}.`, {
+                            study_id: resolvedTester,
+                            timeout_seconds: Math.round(timeoutMs / 1000),
+                            done: 0,
+                            total: 1,
+                            pending: 1,
+                            rows: [
+                                {
+                                    id: resolvedTester,
+                                    status,
+                                    tester_name: String(data.tester_name ?? "Unknown"),
+                                    interaction_count: 0,
+                                },
+                            ],
+                        });
                     }
                     await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
                 }

package/dist/commands/study-tester.js

@@ -6,7 +6,7 @@
  */
 import { withClient, readJsonFileOrStdin, resolveWorkspace } from "../lib/command-helpers.js";
 import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
-import { formatTesterDetail, output } from "../lib/output.js";
+import { formatTesterDetail, buildTesterSummary, output } from "../lib/output.js";
 /** Pick the latest iteration on a study (highest order_index, falling back to last). */
 async function latestIterationForStudy(client, studyId) {
     const study = await client.get(`/studies/${studyId}`);
@@ -53,7 +53,18 @@ export function attachStudyTesterCommands(study) {
         .description("Inspect or manage testers (low-level; usually created via `study run`)")
         .argument("[id]", "Tester ID — pass directly to view tester details and results")
         .option("--workspace <id>", "Workspace ID; accepted for consistency (workspace is inferred from the tester)")
-        .addHelpText("after", "\nExamples:\n  $ ish study tester t-d4e                  # show tester details\n  $ ish study tester create --iteration <id> --profile <id>\n  $ ish study tester batch-create --iteration <id> --file testers.json\n  $ ish study tester delete t-d4e")
+        .option("--summary", "Lean projection: alias + status + sentiment + comment + error_message. Drops the action timeline / interactions array.")
+        .addHelpText("after", `
+Examples:
+  $ ish study tester t-d4e                       # show tester details
+  $ ish study tester t-d4e --summary --json      # headline only (sentiment + comment)
+  $ ish study tester create --iteration <id> --profile <id>
+  $ ish study tester batch-create --iteration <id> --file testers.json
+  $ ish study tester delete t-d4e
+
+Tips:
+  Use \`--get <path>\` to grab one value (e.g. \`--get sentiment\`),
+  \`--fields a,b,c\` to project further.`)
         .action(async (id, opts, cmd) => {
         if (!id) {
             cmd.help();
@@ -65,6 +76,10 @@ export function attachStudyTesterCommands(study) {
             const result = data;
             if (result.id)
                 result.alias = tagAlias(ALIAS_PREFIX.tester, String(result.id));
+            if (opts.summary) {
+                output(buildTesterSummary(result), globals.json, { preProjected: true });
+                return;
+            }
             formatTesterDetail(result, globals.json);
         });
     });