@ishlabs/cli 0.9.0 → 0.11.0

package/dist/lib/modality.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Shared modality helpers.
+ *
+ * Centralises the modality-aware predicates and validators that used to
+ * live alongside individual command files. Keeping them here means a new
+ * modality (or a tweak to an existing one) only needs to land in one place
+ * and the call sites (`study run`, `iteration update`, ...) pick up the
+ * change for free.
+ */
+export type Modality = "interactive" | "video" | "audio" | "text" | "image" | "document" | "chat";
+/** True for the media-bucket modalities that share batch + content semantics. */
+export declare function isMediaModality(modality: string | undefined): boolean;
+/** True for chat modality. Wrapped as a helper so callers don't sprinkle string literals. */
+export declare function isChatModality(modality: string | undefined): boolean;
+/**
+ * Pattern F (Issue #9): an iteration without content is a footgun.
+ * `ish study generate` auto-creates an empty iteration "A" that becomes
+ * `study run`'s default target unless `--iteration` is explicit; agents
+ * silently dispatch against it. This predicate is the gate that lets
+ * study run refuse with a clear suggestion instead of silent failure.
+ */
+export declare function iterationHasContent(details: unknown, modality: string | undefined): boolean;
+/** The flag fragment a user should pass to populate content for a given modality. */
+export declare function describeRequiredContentFlag(modality: string): string;
+export type DetailsValidation = {
+    ok: true;
+} | {
+    ok: false;
+    reason: string;
+    suggestion: string;
+};
+/**
+ * Detect modality/details mismatch on `iteration update --details-json`.
+ *
+ * Catches the silent-corruption footgun where an agent passes
+ * interactive-shape details (`{"url":"..."}`) to a chat iteration (or
+ * vice versa). Returns `{ ok: true }` for unknown modalities so we don't
+ * reject things we don't yet understand. Real shape validation (Zod,
+ * etc.) lives in the backend; this is a quick sanity check at the CLI
+ * boundary.
+ */
+export declare function validateIterationDetails(modality: string | undefined, details: unknown): DetailsValidation;

package/dist/lib/modality.js

@@ -0,0 +1,192 @@
+/**
+ * Shared modality helpers.
+ *
+ * Centralises the modality-aware predicates and validators that used to
+ * live alongside individual command files. Keeping them here means a new
+ * modality (or a tweak to an existing one) only needs to land in one place
+ * and the call sites (`study run`, `iteration update`, ...) pick up the
+ * change for free.
+ */
+import { MEDIA_MODALITIES } from "./types.js";
+/** True for the media-bucket modalities that share batch + content semantics. */
+export function isMediaModality(modality) {
+    return !!modality && MEDIA_MODALITIES.includes(modality);
+}
+/** True for chat modality. Wrapped as a helper so callers don't sprinkle string literals. */
+export function isChatModality(modality) {
+    return modality === "chat";
+}
+/**
+ * Pattern F (Issue #9): an iteration without content is a footgun.
+ * `ish study generate` auto-creates an empty iteration "A" that becomes
+ * `study run`'s default target unless `--iteration` is explicit; agents
+ * silently dispatch against it. This predicate is the gate that lets
+ * study run refuse with a clear suggestion instead of silent failure.
+ */
+export function iterationHasContent(details, modality) {
+    if (!details || typeof details !== "object")
+        return false;
+    const d = details;
+    if (modality === "text") {
+        return typeof d.content_text === "string" && d.content_text.length > 0;
+    }
+    if (modality === "video" || modality === "audio" || modality === "document") {
+        return typeof d.content_url === "string" && d.content_url.length > 0;
+    }
+    if (modality === "image") {
+        const urls = d.image_urls;
+        if (Array.isArray(urls))
+            return urls.length > 0;
+        if (typeof urls === "string")
+            return urls.length > 0;
+        return false;
+    }
+    if (modality === "chat") {
+        // A chat iteration carries either an inline endpoint config or a
+        // pointer to a saved chatbot_endpoints row.
+        const hasEndpoint = !!d.endpoint && typeof d.endpoint === "object";
+        const hasEndpointId = typeof d.chatbot_endpoint_id === "string"
+            && d.chatbot_endpoint_id.length > 0;
+        return hasEndpoint || hasEndpointId;
+    }
+    // interactive (default)
+    return typeof d.url === "string" && d.url.length > 0;
+}
+/** The flag fragment a user should pass to populate content for a given modality. */
+export function describeRequiredContentFlag(modality) {
+    if (modality === "text")
+        return "--content-text <text-or-@file>";
+    if (modality === "video" || modality === "audio" || modality === "document") {
+        return "--content-url <url-or-file>";
+    }
+    if (modality === "image")
+        return "--image-urls <comma-separated>";
+    if (modality === "chat")
+        return "--endpoint <id> | --endpoint-config <file>";
+    return "--url <url>";
+}
+/**
+ * Detect modality/details mismatch on `iteration update --details-json`.
+ *
+ * Catches the silent-corruption footgun where an agent passes
+ * interactive-shape details (`{"url":"..."}`) to a chat iteration (or
+ * vice versa). Returns `{ ok: true }` for unknown modalities so we don't
+ * reject things we don't yet understand. Real shape validation (Zod,
+ * etc.) lives in the backend; this is a quick sanity check at the CLI
+ * boundary.
+ */
+export function validateIterationDetails(modality, details) {
+    if (!details || typeof details !== "object" || Array.isArray(details)) {
+        return { ok: true };
+    }
+    const d = details;
+    const hasUrl = typeof d.url === "string" && d.url.length > 0;
+    const hasEndpointObj = !!d.endpoint && typeof d.endpoint === "object";
+    const hasEndpointId = typeof d.chatbot_endpoint_id === "string"
+        && d.chatbot_endpoint_id.length > 0;
+    const hasContentText = typeof d.content_text === "string" && d.content_text.length > 0;
+    const hasContentUrl = typeof d.content_url === "string" && d.content_url.length > 0;
+    const imageUrls = d.image_urls;
+    const hasImageUrls = Array.isArray(imageUrls)
+        ? imageUrls.length > 0
+        : typeof imageUrls === "string" && imageUrls.length > 0;
+    if (modality === "chat") {
+        if (hasEndpointObj || hasEndpointId)
+            return { ok: true };
+        if (hasUrl) {
+            return {
+                ok: false,
+                reason: "interactive-shape details on a chat iteration",
+                suggestion: "use --details-json with .endpoint or .chatbot_endpoint_id, or use `ish iteration update` with --endpoint <id>",
+            };
+        }
+        if (hasContentText || hasContentUrl || hasImageUrls) {
+            return {
+                ok: false,
+                reason: "media-shape details on a chat iteration",
+                suggestion: "chat iterations require .endpoint (object) or .chatbot_endpoint_id (string); pass --endpoint <id> or --endpoint-config <file>",
+            };
+        }
+        // No recognised content keys at all. Don't reject — agent might be
+        // updating ancillary fields we don't model here.
+        return { ok: true };
+    }
+    if (modality === "interactive") {
+        if (hasUrl)
+            return { ok: true };
+        if (hasEndpointObj || hasEndpointId) {
+            return {
+                ok: false,
+                reason: "chat-shape details on an interactive iteration",
+                suggestion: "use --details-json with .url for interactive iterations, or change the study modality to chat",
+            };
+        }
+        if (hasContentText || hasContentUrl || hasImageUrls) {
+            return {
+                ok: false,
+                reason: "media-shape details on an interactive iteration",
+                suggestion: "interactive iterations require .url; pass --url <url> or --details-json '{\"url\":\"...\"}'",
+            };
+        }
+        return { ok: true };
+    }
+    if (modality === "text") {
+        if (hasContentText)
+            return { ok: true };
+        if (hasUrl) {
+            return {
+                ok: false,
+                reason: "interactive-shape details on a text iteration",
+                suggestion: "text iterations require .content_text; pass --content-text <text-or-@file>",
+            };
+        }
+        if (hasEndpointObj || hasEndpointId) {
+            return {
+                ok: false,
+                reason: "chat-shape details on a text iteration",
+                suggestion: "text iterations require .content_text; pass --content-text <text-or-@file>",
+            };
+        }
+        return { ok: true };
+    }
+    if (modality === "video" || modality === "audio" || modality === "document") {
+        if (hasContentUrl)
+            return { ok: true };
+        if (hasUrl) {
+            return {
+                ok: false,
+                reason: `interactive-shape details on a ${modality} iteration`,
+                suggestion: `${modality} iterations require .content_url; pass --content-url <url-or-file>`,
+            };
+        }
+        if (hasEndpointObj || hasEndpointId) {
+            return {
+                ok: false,
+                reason: `chat-shape details on a ${modality} iteration`,
+                suggestion: `${modality} iterations require .content_url; pass --content-url <url-or-file>`,
+            };
+        }
+        return { ok: true };
+    }
+    if (modality === "image") {
+        if (hasImageUrls)
+            return { ok: true };
+        if (hasUrl) {
+            return {
+                ok: false,
+                reason: "interactive-shape details on an image iteration",
+                suggestion: "image iterations require .image_urls (array or comma-separated string); pass --image-urls <urls>",
+            };
+        }
+        if (hasEndpointObj || hasEndpointId) {
+            return {
+                ok: false,
+                reason: "chat-shape details on an image iteration",
+                suggestion: "image iterations require .image_urls; pass --image-urls <urls>",
+            };
+        }
+        return { ok: true };
+    }
+    // Unknown modality — don't reject, let the backend decide.
+    return { ok: true };
+}

package/dist/lib/output.d.ts

@@ -49,6 +49,30 @@ export declare function formatSiteAccessStatus(summary: import("./site-access.js
 export declare function formatStudyList(studies: Record<string, unknown>[], json: boolean): void;
 export declare function formatStudyDetail(study: Record<string, unknown>, json: boolean, options?: OutputOptions): void;
 export declare function formatStudyResults(study: Record<string, unknown>, json: boolean): void;
+/**
+ * `study results --summary` projection. Drops interview_answers + per-tester
+ * interaction breakdowns; keeps headline counters, sentiment histogram, and a
+ * per-tester {alias, status, sentiment, comment} row. Useful for agents that
+ * need to branch on outcome without paying for the full envelope.
+ */
+export declare function buildStudyResultsSummary(study: Record<string, unknown>): Record<string, unknown>;
+/**
+ * `study results --transcript <tester_id>` projection. Mirrors the schema
+ * MCP's `get_chat_transcript` returns (`src/ish_mcp/projections.py:
+ * build_chat_transcript`) so callers see the same shape regardless of
+ * surface. Tester turns whose action carries no text (e.g. select_option)
+ * surface `text: null`; intent lives on `action_type` + `option_label`.
+ * Bot turns with a `bot_reply.failure` block surface `failure` and
+ * `text: null` and don't count toward `unique_bot_replies`.
+ */
+export declare function buildChatTranscript(tester: Record<string, unknown>): Record<string, unknown>;
+/**
+ * `study tester --summary` projection. Drops the action timeline; keeps the
+ * headline (alias, status, sentiment, comment, error_message). Useful for
+ * the common "did this tester finish, what did they say" check that's
+ * currently buried under the full interactions array.
+ */
+export declare function buildTesterSummary(tester: Record<string, unknown>): Record<string, unknown>;
 export declare function formatIterationList(iterations: Record<string, unknown>[], json: boolean): void;
 export declare function formatTesterDetail(tester: Record<string, unknown>, json: boolean): void;
 export declare function formatTesterProfileList(profiles: unknown, json: boolean, limit?: number): void;
@@ -58,5 +82,22 @@ export declare function formatSimulationPoll(results: Record<string, unknown>[],
 export declare function formatAskList(asks: Record<string, unknown>[], json: boolean): void;
 export declare function formatAskDetail(ask: Record<string, unknown>, json: boolean): void;
 export declare function formatRoundDetail(round: Record<string, unknown>, json: boolean): void;
+/**
+ * Derive a coarse confidence label from sample size + tied-ness + error mix.
+ *
+ * Rules (lowest wins):
+ *   - low:    n < 3 OR tied OR any errored response (we have visible failures)
+ *   - medium: 3 <= n < 10 (small sample but clean)
+ *   - high:   n >= 10 AND no errored responses AND not tied
+ *
+ * Tuned for the typical 5-tester ask: a clean 5/5 lands at "medium" (you
+ * can probably trust the lean), 1/5 with no errors lands at "low" (you
+ * need more data), 5/5 with a tie lands at "low" (no winner to call).
+ */
+export declare function deriveWinnerConfidence(args: {
+    n: number;
+    errored: number;
+    tied: boolean;
+}): "low" | "medium" | "high";
 export declare function formatAskResults(ask: Record<string, unknown>, json: boolean, roundFilter?: number): void;
 export declare function formatConfigList(configs: Record<string, unknown>[], json: boolean): void;