@ishlabs/cli 0.9.0 → 0.10.0

package/README.md

@@ -88,7 +88,7 @@ Workspace (= product, top-level container)
 ```bash
 ish login                          # browser auth
 ish logout
-ish connect <port>                 # Cloudflare tunnel exposing localhost
+ish connect <port>                 # Cloudflare tunnel exposing localhost (--detach, ish disconnect, ish connect status)
 ish upgrade                        # self-update (single-binary installs only)
 ish upgrade --release 0.8.1        # pin a specific release
 ```
@@ -98,15 +98,21 @@ ish upgrade --release 0.8.1        # pin a specific release
 ### Workspaces, studies, iterations, profiles, configs (CRUD groups)
 
 ```bash
-ish workspace  list | create | get | update | delete | use
+ish workspace  list | create | get | update | delete | use | info
 ish workspace  site-access status | basic-auth | cookie | login | affirm-public | clear
 ish study      list | create | generate | get | results | update | delete | use
 ish iteration  list | create | get | update | delete
 ish profile    list | create | generate | get | update | delete
 ish source     upload | get | delete
 ish config     list | create | get | schema | update | delete
+ish chat       endpoint list | create | get | update | delete | use | init | test
+ish secret     list | set | delete
 ```
 
+`ish workspace info` reports `studies_used / studies_max / testers_used / testers_max / tier` so an agent can branch on plan caps before a destructive call returns `error_code: usage_limit_reached`.
+
+`ish chat endpoint` configures HTTP-bot endpoints for chat-modality studies (auto-detect from a curl example, smoke-test, edit). `ish secret` is the per-workspace KV store referenced from chatbot endpoint headers via `{{secret:KEY}}` placeholders. Run `ish docs get-page guides/chat` for the end-to-end recipe.
+
 Testers live as a nested group on a study (low-level — usually created via `study run`):
 
 ```bash
@@ -347,16 +353,59 @@ ish profile generate --source tps-3a4 --propose-count
 ish profile generate --source tps-3a4 --count 4
 ```
 
+### Chat-modality studies — `ish chat`
+
+Configure a customer chatbot endpoint and run chat-modality studies against it.
+
+```bash
+# Author from a curl example (or hand-write the config)
+ish chat endpoint init --from-curl ./bot.curl --name my-bot
+ish chat endpoint create --endpoint-config ./bot-config.json --name "my-bot"
+
+# CRUD on saved endpoints (every dialog edit reduces to one of these)
+ish chat endpoint list
+ish chat endpoint get ep-abc --verbose      # round-trippable {id, name, isTunnelBacked, config}
+ish chat endpoint update ep-abc --name "Production support bot"
+ish chat endpoint update ep-abc --url https://api.example.com/v2/chat --mode stateless
+ish chat endpoint get ep-abc --verbose | jq '.config.outgoing.headers["X-API-Key"] = "{{secret:KEY}}"' \
+  | ish chat endpoint update ep-abc --endpoint-config -
+ish chat endpoint delete ep-abc
+ish chat endpoint use ep-abc                 # set as the active chat endpoint
+
+# Smoke test the connection (single turn; tunnel pre-flight when applicable)
+ish chat endpoint test ep-abc -m "Hello"
+ish chat endpoint test ep-abc -m "Tell me more" --conversation-id "$CID"   # stateful threading
+
+# Run a chat-modality study using the saved endpoint (existing study verbs).
+# Audience size lives on study run via --sample / --all / --profile.
+ish study create --modality chat --endpoint ep-abc --name "Sign-up Q1" --assignment "Sign up:Try to sign up"
+ish study run --study stu-xyz --sample 5 --wait
+ish study results stu-xyz --json | jq '.testers'
+```
+
+Local bots (`localhost` / `127.0.0.1` / `0.0.0.0`) auto-flag `is_tunnel_backed=true` on `init`; pair with `ish connect <port>` in another shell. Override with `--tunnel-backed` / `--no-tunnel-backed`.
+
+`init` returns `confidence` (`high` / `medium` / `low`) and a `missingSignals: [...]` array naming any inputs the inference couldn't observe (e.g. `["response_shape", "message_path"]` when no response sample is provided). When confidence is `low`, verify with `chat endpoint test` before running a study.
+
+Failures from `chat endpoint test` carry a structured `error_kind`: `TunnelInactive` (run `ish connect <port>` first), `BotUnreachable` (URL/port wrong or bot down), `BotResponseError` (non-2xx with a status code), `BotEnvelopeError` (200 OK with the bot's own error in the body — see `raw_excerpt`), `BotInvalidResponseError` (response doesn't match the parsing schema), `BotAuthError`, `BotTimeoutError`, `BotRetryExhaustedError`.
+
+Full guide: `ish docs get-page guides/chat`.
+
 ### Expose localhost
 
-For interactive studies that need to reach a service running on your machine:
+For interactive studies (and chat endpoints with `is_tunnel_backed=true`) that need to reach a service running on your machine:
 
 ```bash
-ish connect 3000                       # Cloudflare tunnel to localhost:3000
+ish connect 3000                          # foreground Cloudflare tunnel to :3000
+ish connect 3000 --detach --json          # fork after first heartbeat; prints {pid, tunnel_url, registered}
+ish connect status --json                 # {active, pid, tunnel_url, registered_at} or {active:false}
+ish disconnect --json                     # graceful shutdown of an active tunnel
 ISH_TOKEN=YOUR_TOKEN ish connect 8080
 ```
 
-`connect` is a long-running command — keep it open while testers run. The Cloudflare tunnel URL prints prominently after "Connected"; pass `--json` for one-line machine-readable output (`{"status":"connected","tunnel_url":"...","local_port":3000,"registered":true}`) suitable for scripts.
+Foreground `connect` is long-running — keep it open while testers run. The tunnel URL prints prominently after "Connected"; pass `--json` for one-line machine-readable output (`{"status":"connected","tunnel_url":"...","local_port":3000,"registered":true}`). The `--detach` form forks after the first successful heartbeat and returns immediately, tracking PID + URL in `~/.ish/connect.lock` so `connect status` and `disconnect` find it later.
+
+Destructive verbs in `--json` mode (e.g. `chat endpoint delete`, `study delete`) require an explicit `--yes`; the rejection envelope carries `error_kind: "ConfirmationRequired"` and an `example` field with the same command + `--yes` appended, so an agent can recover without re-reading the help text.
 
 ## Global flags
 

package/dist/commands/ask.d.ts

@@ -2,4 +2,16 @@
  * ish ask — Create and run asks (multi-round surveys with variants).
  */
 import type { Command } from "commander";
+import type { AudienceSubset } from "../lib/types.js";
+/**
+ * Parse the `--subset-round <n> --subset-variant <variant_id>` pair into
+ * an `AudienceSubset` payload (Pattern B). Both flags must be passed
+ * together or neither — half a subset is a misconfiguration the agent
+ * should fix before dispatch, not a silent fallthrough to the full
+ * audience.
+ *
+ * Returns `undefined` when neither flag is set; throws when only one is
+ * set or when `--subset-round` isn't a positive integer.
+ */
+export declare function parseAudienceSubset(subsetRound: string | undefined, subsetVariant: string | undefined): AudienceSubset | undefined;
 export declare function registerAskCommands(program: Command): void;

package/dist/commands/ask.js

@@ -7,6 +7,7 @@ import { loadConfig, saveConfig } from "../config.js";
 import { formatAskList, formatAskDetail, formatRoundDetail, formatAskResults, output, } from "../lib/output.js";
 import { parseVariantInputs, uploadAndBuildVariants, } from "../lib/ask-variants.js";
 import { loadQuestionsManifest } from "../lib/ask-questions.js";
+import { ApiError } from "../lib/api-client.js";
 const POLL_INTERVAL_MS = 5_000;
 // ---------------------------------------------------------------------------
 // Helpers
@@ -108,6 +109,32 @@ async function buildRoundInput(client, productId, opts, quiet) {
         round.questions = questions;
     return round;
 }
+/**
+ * Parse the `--subset-round <n> --subset-variant <variant_id>` pair into
+ * an `AudienceSubset` payload (Pattern B). Both flags must be passed
+ * together or neither — half a subset is a misconfiguration the agent
+ * should fix before dispatch, not a silent fallthrough to the full
+ * audience.
+ *
+ * Returns `undefined` when neither flag is set; throws when only one is
+ * set or when `--subset-round` isn't a positive integer.
+ */
+export function parseAudienceSubset(subsetRound, subsetVariant) {
+    if (subsetRound === undefined && subsetVariant === undefined)
+        return undefined;
+    if (subsetRound === undefined || subsetVariant === undefined) {
+        throw new Error("--subset-round and --subset-variant must be passed together (or both omitted).");
+    }
+    const round = Number.parseInt(subsetRound, 10);
+    if (!Number.isFinite(round) || round < 1 || !/^\d+$/.test(subsetRound)) {
+        throw new Error(`--subset-round must be a positive integer (got "${subsetRound}").`);
+    }
+    const trimmedVariant = subsetVariant.trim();
+    if (trimmedVariant.length === 0) {
+        throw new Error("--subset-variant must be a variant UUID (got empty string).");
+    }
+    return { round, picked_variant_id: trimmedVariant };
+}
 // ---------------------------------------------------------------------------
 // Command registration
 // ---------------------------------------------------------------------------
@@ -145,6 +172,8 @@ Concept pages: ish docs get-page concepts/ask
         allFlagName: "--all-simulatable",
         allFlagDescription: "Use every simulatable AI profile matching the filters (with --new only)",
     })
+        .option("--subset-round <n>", "Drill-in subset (Pattern B) — append-round only. 1-indexed prior round to filter against. Pair with --subset-variant.")
+        .option("--subset-variant <variant_id>", "Drill-in subset (Pattern B) — append-round only. Variant id (UUID) on the prior round whose pickers should inherit. Read from `aggregates.pick_buckets` or `variants[*].id` on the prior round's `ask results --json`.")
         .option("--wait", "Wait until the round completes (or errors)")
         .option("--timeout <s>", "Wait timeout in seconds (default 300)")
         .addHelpText("after", `
@@ -169,6 +198,9 @@ Examples:
                 if (pickedId) {
                     throw new Error("Cannot pass an ask id together with --new. Drop the id, or drop --new to append a round.");
                 }
+                if (opts.subsetRound !== undefined || opts.subsetVariant !== undefined) {
+                    throw new Error("--subset-round / --subset-variant are only valid when appending to an existing ask. Drop --new or drop the subset flags.");
+                }
                 const wid = resolveWorkspace(opts.workspace);
                 const testerIds = await resolveAudienceProfileIds(client, wid, audienceFlags(opts), { requireSimulatable: true, allFlagName: "--all-simulatable" });
                 const round = await buildRoundInput(client, wid, opts, !!globals.quiet);
@@ -180,7 +212,28 @@ Examples:
                     tester_profile_ids: testerIds,
                     first_round: round,
                 };
-                let data = await client.post(`/products/${wid}/asks`, body, { timeout: 120_000 });
+                // M5 / Pattern G: `ask run --new` POSTs to a non-idempotent
+                // create endpoint. If the backend errors after the row is
+                // committed (a 500 mid-pipeline, a network timeout after the
+                // POST landed), an automatic retry would create a duplicate
+                // ask. Override `retryable` to false on any failure here so
+                // agents don't auto-retry. The error envelope also reminds
+                // the agent to inspect `ish ask list --workspace <id>` before
+                // re-running, since the resource may already exist.
+                let data;
+                try {
+                    data = await client.post(`/products/${wid}/asks`, body, { timeout: 120_000 });
+                }
+                catch (err) {
+                    if (err instanceof ApiError) {
+                        err.retryable = false;
+                        const tagged = err;
+                        tagged.suggestions = [
+                            `\`ish ask list --workspace ${wid}\` to check whether the ask was created server-side before retrying — \`ask run --new\` is non-idempotent and will duplicate on retry.`,
+                        ];
+                    }
+                    throw err;
+                }
                 if (data.id) {
                     const config = loadConfig();
                     config.ask = data.id;
@@ -223,6 +276,9 @@ Examples:
             }
             const ask = await client.get(`/asks/${aid}`);
             const round = await buildRoundInput(client, ask.product_id, opts, !!globals.quiet);
+            const subset = parseAudienceSubset(opts.subsetRound, opts.subsetVariant);
+            if (subset)
+                round.audience_subset = subset;
             const created = await client.post(`/asks/${aid}/rounds`, round);
             if (opts.wait) {
                 const timeoutMs = parseWaitTimeout(opts.timeout);
@@ -519,14 +575,32 @@ the model's self-reported confidence in its variant choice. See
         .option("--wants-pick", "Each tester picks a favourite variant (compatible with --wants-ratings; can be set together).")
         .option("--wants-ratings", "Each tester rates every variant 1–5 (compatible with --wants-pick; can be set together). If neither is set, testers leave a free-form comment only.")
         .option("--questions <file.json>", `Questions JSON file: [{"question":"...","type":"text"|"slider"|"likert"|"single-choice"|"multiple-choice"|"number"}]`)
+        .option("--subset-round <n>", "Drill-in subset (Pattern B) — 1-indexed prior round to filter against. Pair with --subset-variant. The new round dispatches only to testers who picked --subset-variant on round N.")
+        .option("--subset-variant <variant_id>", "Drill-in subset (Pattern B) — variant id (UUID) on the prior round whose pickers should inherit. Pair with --subset-round. Read from `aggregates.pick_buckets` or `variants[*].id` on the prior round.")
         .option("--wait", "Wait until the new round completes")
         .option("--timeout <s>", "Wait timeout in seconds (default 300)")
-        .addHelpText("after", "\nExamples:\n  $ ish ask add-round a-6ec --prompt \"And now?\" --variant text:\"Hello\" --variant text:\"Hi\" --wait")
+        .addHelpText("after", `
+Examples:
+  # Append round 2 to the same audience.
+  $ ish ask add-round a-6ec --prompt "And now?" --variant text:"Hello" --variant text:"Hi" --wait
+
+  # Drill round 2 into the round-1-A-pickers (Pattern B).
+  $ ish ask add-round a-6ec \\
+      --prompt "What would make you actually click?" \\
+      --subset-round 1 --subset-variant 5f3a... \\
+      --wait
+
+If --subset-round / --subset-variant fails to resolve (round missing, variant
+not on that round, or zero pickers), the backend returns a 422 with
+error_kind: "audience_subset_invalid".`)
         .action(async (id, opts, cmd) => {
         await withClient(cmd, async (client, globals) => {
             const aid = resolveAsk(pickAskRef(id, opts.ask));
             const ask = await client.get(`/asks/${aid}`);
             const round = await buildRoundInput(client, ask.product_id, opts, !!globals.quiet);
+            const subset = parseAudienceSubset(opts.subsetRound, opts.subsetVariant);
+            if (subset)
+                round.audience_subset = subset;
             const created = await client.post(`/asks/${aid}/rounds`, round);
             if (opts.wait) {
                 const timeoutMs = parseWaitTimeout(opts.timeout);
@@ -607,6 +681,57 @@ text, slider, likert, single-choice, multiple-choice, number.`)
             }, globals.json);
         });
     });
+    // ---- retry --------------------------------------------------------------
+    ask
+        .command("retry")
+        .description("Re-dispatch only the errored responses on a round (idempotent: zero-errored is a no-op).")
+        .argument("[id]", "Ask alias or UUID (defaults to active ask)")
+        .option("--ask <id>", "Ask ID; alternative to positional argument")
+        .requiredOption("--round <n|round-id>", "Round number (1-indexed) or round id/alias")
+        .option("--wait", "Wait until the retried round completes (or errors)")
+        .option("--timeout <s>", "Wait timeout in seconds (default 300)")
+        .addHelpText("after", `
+Examples:
+  # Retry the errored 4 of 5 testers on round 1.
+  $ ish ask retry a-d3e --round 1
+
+  # Retry and wait for the round to settle.
+  $ ish ask retry a-d3e --round 1 --wait
+
+Notes:
+  - COMPLETED responses are left untouched. Only ERRORED rows are reset to PENDING and re-run from scratch.
+  - The round flips back to RUNNING for the duration of the retry; the prior round summary is dropped and rebuilt once the retry settles.
+  - On a round with no errored responses, the verb is a no-op and returns the round unchanged.`)
+        .action(async (id, opts, cmd) => {
+        await withClient(cmd, async (client, globals) => {
+            const aid = resolveAsk(pickAskRef(id, opts.ask));
+            const ask = await client.get(`/asks/${aid}`);
+            const round = getRoundByIndexOrId(ask, opts.round);
+            const updated = await client.post(`/asks/${aid}/rounds/${round.id}/retry`, {});
+            if (opts.wait) {
+                const timeoutMs = parseWaitTimeout(opts.timeout);
+                await pollUntilRoundDone(client, aid, updated.order_index, timeoutMs, !!globals.quiet);
+                const refreshed = await client.get(`/asks/${aid}`);
+                const target = refreshed.rounds.find((r) => r.id === updated.id);
+                if (target) {
+                    formatRoundDetail(target, globals.json);
+                    return;
+                }
+            }
+            if (!globals.json || globals.verbose) {
+                formatRoundDetail(updated, globals.json);
+                return;
+            }
+            output({
+                id: aid,
+                alias: tagAlias(ALIAS_PREFIX.ask, aid),
+                round: {
+                    round_number: updated.order_index + 1,
+                    status: updated.status,
+                },
+            }, globals.json);
+        });
+    });
     // ---- add-testers --------------------------------------------------------
     const askAddTesters = ask
         .command("add-testers")

package/dist/commands/chat.d.ts

@@ -0,0 +1,17 @@
+/**
+ * ish chat — Configure chatbot endpoints and run chat-modality studies.
+ *
+ * The CLI's primary user is autonomous AI agents. Every verb here is
+ * scriptable: deterministic JSON outputs, no interactive prompts, no
+ * REPLs. Endpoint editing matches the editor dialog's semantics
+ * (full-replace via PUT) plus client-side field-shorthand flags for
+ * common one-line edits.
+ *
+ * Chat-modality studies are reached via the existing `ish study create
+ * --modality chat --endpoint <id>` extension; this file does NOT
+ * fork a parallel `chat run` verb tree.
+ */
+import type { Command } from "commander";
+import { envelopeFromRow } from "../lib/chat-endpoint-formatters.js";
+export declare function registerChatCommand(program: Command): void;
+export { envelopeFromRow };