@ishlabs/cli 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -5
- package/dist/commands/ask.d.ts +12 -0
- package/dist/commands/ask.js +127 -2
- package/dist/commands/chat.d.ts +17 -0
- package/dist/commands/chat.js +655 -0
- package/dist/commands/iteration.js +134 -14
- package/dist/commands/secret.d.ts +20 -0
- package/dist/commands/secret.js +246 -0
- package/dist/commands/study-run.d.ts +38 -0
- package/dist/commands/study-run.js +199 -80
- package/dist/commands/study-tester.js +17 -2
- package/dist/commands/study.js +309 -37
- package/dist/commands/workspace.js +81 -0
- package/dist/config.d.ts +3 -0
- package/dist/connect.d.ts +3 -0
- package/dist/connect.js +346 -22
- package/dist/index.js +64 -6
- package/dist/lib/alias-hydrate.d.ts +42 -0
- package/dist/lib/alias-hydrate.js +175 -0
- package/dist/lib/alias-store.d.ts +1 -0
- package/dist/lib/alias-store.js +28 -1
- package/dist/lib/auth.js +4 -2
- package/dist/lib/chat-endpoint-formatters.d.ts +74 -0
- package/dist/lib/chat-endpoint-formatters.js +154 -0
- package/dist/lib/chat-endpoint-templates.d.ts +35 -0
- package/dist/lib/chat-endpoint-templates.js +210 -0
- package/dist/lib/command-helpers.d.ts +18 -0
- package/dist/lib/command-helpers.js +105 -3
- package/dist/lib/docs.js +641 -17
- package/dist/lib/modality.d.ts +42 -0
- package/dist/lib/modality.js +192 -0
- package/dist/lib/output.d.ts +41 -0
- package/dist/lib/output.js +453 -19
- package/dist/lib/paths.d.ts +1 -0
- package/dist/lib/paths.js +3 -0
- package/dist/lib/skill-content.d.ts +18 -0
- package/dist/lib/skill-content.js +223 -12
- package/dist/lib/types.d.ts +15 -0
- package/package.json +2 -2
package/dist/commands/study.js
CHANGED
|
@@ -2,18 +2,33 @@
|
|
|
2
2
|
* ish study — Manage studies.
|
|
3
3
|
*/
|
|
4
4
|
import { readFileSync } from "node:fs";
|
|
5
|
-
import {
|
|
5
|
+
import { Option } from "commander";
|
|
6
|
+
import { withClient, getWebUrl, terminalLink, resolveWorkspace, confirmDestructive, readFileOrStdin } from "../lib/command-helpers.js";
|
|
6
7
|
import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
7
8
|
import { loadConfig, saveConfig } from "../config.js";
|
|
8
|
-
import { formatStudyList, formatStudyDetail, formatStudyResults, output, ValidationError } from "../lib/output.js";
|
|
9
|
+
import { formatStudyList, formatStudyDetail, formatStudyResults, buildStudyResultsSummary, buildChatTranscript, output, ValidationError, } from "../lib/output.js";
|
|
9
10
|
import { VALID_CONTENT_TYPES } from "../lib/types.js";
|
|
10
11
|
import { parseAssignment, loadAssignmentsFile, parseQuestion } from "../lib/study-inputs.js";
|
|
11
12
|
import { loadQuestionsManifest } from "../lib/ask-questions.js";
|
|
13
|
+
import { isLocalPath } from "../lib/upload.js";
|
|
12
14
|
import { attachStudyRunCommands } from "./study-run.js";
|
|
13
15
|
import { attachStudyTesterCommands } from "./study-tester.js";
|
|
14
16
|
function collectRepeatable(value, prev = []) {
|
|
15
17
|
return prev.concat([value]);
|
|
16
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* Pattern G.1: render the `VALID_CONTENT_TYPES` registry as a help-text block.
|
|
21
|
+
* The single source of truth is `src/lib/types.ts`; this helper formats it for
|
|
22
|
+
* `study create --help` so agents don't have to discover the legal
|
|
23
|
+
* --content-type values per modality through a round-trip error.
|
|
24
|
+
*
|
|
25
|
+
* `interactive` and `chat` modalities don't carry a content_type; they're
|
|
26
|
+
* deliberately omitted (matches the registry).
|
|
27
|
+
*/
|
|
28
|
+
function describeContentTypes() {
|
|
29
|
+
const lines = Object.entries(VALID_CONTENT_TYPES).map(([modality, types]) => ` ${modality.padEnd(9)} ${types.join(", ")}`);
|
|
30
|
+
return lines.join("\n");
|
|
31
|
+
}
|
|
17
32
|
function resolveAssignments(opts) {
|
|
18
33
|
const sources = [];
|
|
19
34
|
if (opts.assignment && opts.assignment.length > 0)
|
|
@@ -77,12 +92,12 @@ Concept pages: ish docs get-page concepts/study
|
|
|
77
92
|
});
|
|
78
93
|
study
|
|
79
94
|
.command("create")
|
|
80
|
-
.description("Create a new study (the persistent shape: modality, tasks, questionnaire). Optionally creates iteration A inline when --content-text
|
|
95
|
+
.description("Create a new study (the persistent shape: modality, tasks, questionnaire). Optionally creates iteration A inline when --content-text, --url, --image-urls, --content-url, or --endpoint is passed.")
|
|
81
96
|
.option("--workspace <id>", "Workspace ID")
|
|
82
97
|
.requiredOption("--name <name>", "Study name")
|
|
83
98
|
.option("--description <description>", "Study description")
|
|
84
99
|
.option("--modality <modality>", "Study modality (interactive, video, audio, text, image, document, chat)")
|
|
85
|
-
.option("--content-type <type>", "Content type (
|
|
100
|
+
.option("--content-type <type>", "Content type (per-modality enum — see 'Content types by modality' below). Not used for interactive / chat.")
|
|
86
101
|
.option("--assignment <name:instructions>", "Assignment as 'Name:Instructions' (repeatable)", collectRepeatable, [])
|
|
87
102
|
.option("--assignments-file <path>", "JSON file with assignments array")
|
|
88
103
|
.option("--assignments <json>", "Inline JSON array of assignments (escape hatch)")
|
|
@@ -90,6 +105,13 @@ Concept pages: ish docs get-page concepts/study
|
|
|
90
105
|
.option("--questionnaire <path>", "JSON file defining the questionnaire (supports text, slider, likert, single-choice, multiple-choice, number; timing=before|after)")
|
|
91
106
|
.option("--content-text <text>", "Text content to evaluate, or @filepath to read from file. Creates iteration A inline (text modality only)")
|
|
92
107
|
.option("--url <url>", "URL to test. Creates iteration A inline (interactive modality only)")
|
|
108
|
+
.option("--screen-format <format>", "Screen format for interactive iterations: desktop (default) or mobile_portrait")
|
|
109
|
+
.option("--content-url <url>", "Public URL of the media file. Creates iteration A inline (video, audio, document modalities). For local files, use the 2-step `iteration create` flow.")
|
|
110
|
+
.option("--image-urls <urls>", "Comma-separated public image URLs. Creates iteration A inline (image modality). For local files, use the 2-step `iteration create` flow.")
|
|
111
|
+
.option("--title <title>", "Content title (text + media modalities — image, video, audio, document; optional). Not used for interactive / chat.")
|
|
112
|
+
.option("--endpoint <id>", "Saved chatbot endpoint id or alias. Creates iteration A inline (chat modality only)")
|
|
113
|
+
.option("--endpoint-config <file>", "ChatbotEndpointConfig JSON file (or `-` for stdin); embedded directly. Mutually exclusive with --endpoint (chat modality only)")
|
|
114
|
+
.option("--max-turns <n>", "Maximum conversation turns per tester (chat modality only; default 12)", (v) => Number(v))
|
|
93
115
|
.addHelpText("after", `
|
|
94
116
|
Note: --workspace is optional if set via \`ish workspace use <alias>\`.
|
|
95
117
|
|
|
@@ -98,12 +120,46 @@ quickly add simple text questions, or \`--questionnaire <file.json>\` for richer
|
|
|
98
120
|
types (slider, likert, single-choice, multiple-choice, number) and custom
|
|
99
121
|
timing. The two forms are mutually exclusive — pick one.
|
|
100
122
|
|
|
123
|
+
Inline iteration shortcuts (one-shot study + iteration A):
|
|
124
|
+
--modality interactive --url <url> [--screen-format desktop|mobile_portrait]
|
|
125
|
+
--modality text --content-text <text-or-@file>
|
|
126
|
+
--modality image --image-urls <url1,url2,...>
|
|
127
|
+
--modality video --content-url <url>
|
|
128
|
+
--modality audio --content-url <url>
|
|
129
|
+
--modality document --content-url <url>
|
|
130
|
+
--modality chat --endpoint <id> | --endpoint-config <file>
|
|
131
|
+
|
|
132
|
+
Local file paths in --content-url and --image-urls are NOT accepted on
|
|
133
|
+
\`study create\` (they need an existing study to upload against). For local
|
|
134
|
+
files, use the 2-step flow: \`study create\` (no media flags) then
|
|
135
|
+
\`iteration create --content-url ./file.mp4\`.
|
|
136
|
+
|
|
101
137
|
Examples:
|
|
102
138
|
# Interactive study with one assignment and a single-question questionnaire:
|
|
103
139
|
$ ish study create --name "Onboarding UX" --modality interactive \\
|
|
104
140
|
--assignment "Sign up:Complete the signup flow" \\
|
|
105
141
|
--question "How easy was it?"
|
|
106
142
|
|
|
143
|
+
# One-shot interactive (URL + screen format inline):
|
|
144
|
+
$ ish study create --name "HN scan" --modality interactive \\
|
|
145
|
+
--url https://news.ycombinator.com --screen-format desktop \\
|
|
146
|
+
--assignment "Skim:Skim the top stories"
|
|
147
|
+
|
|
148
|
+
# One-shot image A/B (uses iteration A inline):
|
|
149
|
+
$ ish study create --name "Hero shots" --modality image \\
|
|
150
|
+
--image-urls "https://cdn.example.com/a.png,https://cdn.example.com/b.png" \\
|
|
151
|
+
--assignment "Compare:Which feels more premium?"
|
|
152
|
+
|
|
153
|
+
# One-shot video study:
|
|
154
|
+
$ ish study create --name "Product Ad" --modality video \\
|
|
155
|
+
--content-url https://cdn.example.com/ad.mp4 \\
|
|
156
|
+
--assignment "Watch:Watch this ad and share your reaction"
|
|
157
|
+
|
|
158
|
+
# One-shot document study:
|
|
159
|
+
$ ish study create --name "Whitepaper" --modality document \\
|
|
160
|
+
--content-url https://cdn.example.com/report.pdf \\
|
|
161
|
+
--assignment "Skim:Skim the report and summarise"
|
|
162
|
+
|
|
107
163
|
# Multiple assignments + a richer questionnaire from a file:
|
|
108
164
|
$ ish study create --name "Checkout" --modality interactive \\
|
|
109
165
|
--assignment "Browse:Find a product you like" \\
|
|
@@ -114,11 +170,15 @@ Examples:
|
|
|
114
170
|
$ ish study create --name "Newsletter" --modality text --content-type email \\
|
|
115
171
|
--assignments-file ./assignments.json
|
|
116
172
|
|
|
117
|
-
#
|
|
118
|
-
$ ish study create --name "
|
|
119
|
-
--
|
|
120
|
-
--
|
|
121
|
-
--
|
|
173
|
+
# Chat study targeting a saved chatbot endpoint:
|
|
174
|
+
$ ish study create --name "Onboarding bot" --modality chat \\
|
|
175
|
+
--endpoint ep-abc \\
|
|
176
|
+
--assignment "Sign up:Complete the signup flow" \\
|
|
177
|
+
--max-turns 20
|
|
178
|
+
|
|
179
|
+
# Chat study with an inline endpoint config (stdin or file):
|
|
180
|
+
$ cat ./endpoint.json | ish study create --name "Bot smoke" \\
|
|
181
|
+
--modality chat --endpoint-config -
|
|
122
182
|
|
|
123
183
|
# Sample questionnaire.json (full InterviewQuestion shape):
|
|
124
184
|
# [
|
|
@@ -128,15 +188,15 @@ Examples:
|
|
|
128
188
|
# "options": ["A","B","C"] }
|
|
129
189
|
# ]
|
|
130
190
|
|
|
131
|
-
Content types by modality:
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
191
|
+
Content types by modality (source: VALID_CONTENT_TYPES in src/lib/types.ts; interactive + chat omitted — they don't take --content-type):
|
|
192
|
+
${describeContentTypes()}
|
|
193
|
+
|
|
194
|
+
Tips:
|
|
195
|
+
Use \`--get <path>\` to capture a single value (e.g. \`--get id\`),
|
|
196
|
+
\`--fields a,b,c\` to project the JSON output to listed fields.
|
|
137
197
|
|
|
138
198
|
Next: configure a run with \`ish iteration create --study <id>\`,
|
|
139
|
-
then dispatch with \`ish study run
|
|
199
|
+
then dispatch with \`ish study run\` (audience size is set on run, not create).`)
|
|
140
200
|
.action(async (opts, cmd) => {
|
|
141
201
|
await withClient(cmd, async (client, globals) => {
|
|
142
202
|
const assignments = resolveAssignments(opts);
|
|
@@ -148,13 +208,51 @@ Next: configure a run with \`ish iteration create --study <id>\`,
|
|
|
148
208
|
throw new ValidationError(`Invalid content type "${opts.contentType}" for modality "${opts.modality}".`, validTypes);
|
|
149
209
|
}
|
|
150
210
|
}
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
-
//
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
211
|
+
// --endpoint and --endpoint-config are mutually exclusive. Modality
|
|
212
|
+
// mismatch is enforced inside the chat branch below, mirroring how
|
|
213
|
+
// --content-text / --url validate against modality.
|
|
214
|
+
if (opts.endpoint !== undefined && opts.endpointConfig !== undefined) {
|
|
215
|
+
throw new ValidationError("Pass only one of: --endpoint, --endpoint-config.", ["--endpoint", "--endpoint-config"]);
|
|
216
|
+
}
|
|
217
|
+
// Pattern E + D + J (cli half): build an inline iteration A when one
|
|
218
|
+
// of the modality-specific content flags is provided, so a single
|
|
219
|
+
// `study create` produces a study that's immediately runnable.
|
|
220
|
+
// Without these flags the backend creates zero iterations and the
|
|
221
|
+
// first `iteration create` becomes A. The backend requires a
|
|
222
|
+
// non-empty `name` on the inline iteration; we default to "A" to
|
|
223
|
+
// match the iteration-naming convention.
|
|
224
|
+
//
|
|
225
|
+
// Local file paths in --content-url / --image-urls are NOT supported
|
|
226
|
+
// here because the upload endpoint requires a study_id, which doesn't
|
|
227
|
+
// exist until after `studies` POST. For local files, agents fall
|
|
228
|
+
// back to the existing 2-step `iteration create` path which uploads
|
|
229
|
+
// against the freshly-created study.
|
|
230
|
+
const inlineMediaFlagsSet = [
|
|
231
|
+
opts.contentText !== undefined ? "--content-text" : null,
|
|
232
|
+
opts.url !== undefined ? "--url" : null,
|
|
233
|
+
opts.contentUrl !== undefined ? "--content-url" : null,
|
|
234
|
+
opts.imageUrls !== undefined ? "--image-urls" : null,
|
|
235
|
+
(opts.endpoint !== undefined || opts.endpointConfig !== undefined) ? "--endpoint/--endpoint-config" : null,
|
|
236
|
+
].filter((f) => f !== null);
|
|
237
|
+
if (inlineMediaFlagsSet.length > 1) {
|
|
238
|
+
throw new ValidationError(`Pass only one inline-iteration flag: ${inlineMediaFlagsSet.join(", ")}.`, inlineMediaFlagsSet);
|
|
239
|
+
}
|
|
240
|
+
if (opts.screenFormat !== undefined && opts.url === undefined) {
|
|
241
|
+
throw new Error("--screen-format only applies when --url is set (interactive modality).");
|
|
242
|
+
}
|
|
243
|
+
// Pattern G.2: --title is metadata, not content. The backend
|
|
244
|
+
// accepts it on text + media modalities (see
|
|
245
|
+
// `buildIterationDetails` in iteration.ts). Reject it only on
|
|
246
|
+
// shapes that have no title field — interactive (URL only) and
|
|
247
|
+
// chat (endpoint config carries its own metadata).
|
|
248
|
+
if (opts.title !== undefined
|
|
249
|
+
&& opts.contentText === undefined
|
|
250
|
+
&& opts.contentUrl === undefined
|
|
251
|
+
&& opts.imageUrls === undefined) {
|
|
252
|
+
throw new Error("--title only applies with --content-text (text) or --content-url / --image-urls (media). Interactive + chat iterations don't carry a title.");
|
|
253
|
+
}
|
|
157
254
|
let inlineIteration;
|
|
255
|
+
let chatbotEndpointId = null;
|
|
158
256
|
if (opts.contentText !== undefined) {
|
|
159
257
|
if (opts.modality && opts.modality !== "text") {
|
|
160
258
|
throw new Error(`--content-text is only valid with --modality text (got "${opts.modality}").`);
|
|
@@ -162,7 +260,14 @@ Next: configure a run with \`ish iteration create --study <id>\`,
|
|
|
162
260
|
const text = opts.contentText.startsWith("@")
|
|
163
261
|
? readFileSync(opts.contentText.slice(1), "utf8")
|
|
164
262
|
: opts.contentText;
|
|
165
|
-
inlineIteration = {
|
|
263
|
+
inlineIteration = {
|
|
264
|
+
name: "A",
|
|
265
|
+
details: {
|
|
266
|
+
type: "text",
|
|
267
|
+
content_text: text,
|
|
268
|
+
...(opts.title && { title: opts.title }),
|
|
269
|
+
},
|
|
270
|
+
};
|
|
166
271
|
}
|
|
167
272
|
else if (opts.url !== undefined) {
|
|
168
273
|
if (opts.modality && opts.modality !== "interactive") {
|
|
@@ -170,7 +275,89 @@ Next: configure a run with \`ish iteration create --study <id>\`,
|
|
|
170
275
|
}
|
|
171
276
|
inlineIteration = {
|
|
172
277
|
name: "A",
|
|
173
|
-
details: {
|
|
278
|
+
details: {
|
|
279
|
+
type: "interactive",
|
|
280
|
+
url: opts.url,
|
|
281
|
+
platform: "browser",
|
|
282
|
+
screen_format: opts.screenFormat || "desktop",
|
|
283
|
+
},
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
else if (opts.imageUrls !== undefined) {
|
|
287
|
+
if (opts.modality && opts.modality !== "image") {
|
|
288
|
+
throw new Error(`--image-urls is only valid with --modality image (got "${opts.modality}").`);
|
|
289
|
+
}
|
|
290
|
+
const urls = opts.imageUrls.split(",").map((s) => s.trim()).filter(Boolean);
|
|
291
|
+
if (urls.length === 0) {
|
|
292
|
+
throw new Error("--image-urls is empty. Provide one or more comma-separated URLs.");
|
|
293
|
+
}
|
|
294
|
+
const localPaths = urls.filter((u) => isLocalPath(u));
|
|
295
|
+
if (localPaths.length > 0) {
|
|
296
|
+
throw new Error(`--image-urls on \`study create\` only accepts http(s) URLs (local files need an existing study to upload against). Got local path(s): ${localPaths.join(", ")}. Use the 2-step flow: \`ish study create\` (no --image-urls) then \`ish iteration create --image-urls "${opts.imageUrls}"\`.`);
|
|
297
|
+
}
|
|
298
|
+
inlineIteration = {
|
|
299
|
+
name: "A",
|
|
300
|
+
details: {
|
|
301
|
+
type: "image",
|
|
302
|
+
image_urls: urls,
|
|
303
|
+
...(opts.title && { title: opts.title }),
|
|
304
|
+
},
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
else if (opts.contentUrl !== undefined) {
|
|
308
|
+
const mediaModalities = ["video", "audio", "document"];
|
|
309
|
+
if (opts.modality && !mediaModalities.includes(opts.modality)) {
|
|
310
|
+
throw new Error(`--content-url is only valid with --modality video|audio|document (got "${opts.modality}").`);
|
|
311
|
+
}
|
|
312
|
+
if (!opts.modality) {
|
|
313
|
+
throw new Error("--content-url requires --modality video|audio|document so the iteration shape is unambiguous.");
|
|
314
|
+
}
|
|
315
|
+
if (isLocalPath(opts.contentUrl)) {
|
|
316
|
+
throw new Error(`--content-url on \`study create\` only accepts an http(s) URL (local files need an existing study to upload against). Got local path: ${opts.contentUrl}. Use the 2-step flow: \`ish study create\` (no --content-url) then \`ish iteration create --content-url "${opts.contentUrl}"\`.`);
|
|
317
|
+
}
|
|
318
|
+
inlineIteration = {
|
|
319
|
+
name: "A",
|
|
320
|
+
details: {
|
|
321
|
+
type: opts.modality,
|
|
322
|
+
content_url: opts.contentUrl,
|
|
323
|
+
...(opts.title && { title: opts.title }),
|
|
324
|
+
},
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
else if (opts.endpoint !== undefined || opts.endpointConfig !== undefined) {
|
|
328
|
+
if (opts.modality && opts.modality !== "chat") {
|
|
329
|
+
throw new ValidationError(`--endpoint / --endpoint-config require --modality chat (got "${opts.modality}").`, ["chat"]);
|
|
330
|
+
}
|
|
331
|
+
let endpointConfig;
|
|
332
|
+
if (opts.endpoint !== undefined) {
|
|
333
|
+
const epId = resolveId(opts.endpoint);
|
|
334
|
+
const ep = await client.get(`/chatbot-endpoints/${epId}`);
|
|
335
|
+
const cfg = ep?.config;
|
|
336
|
+
if (!cfg || typeof cfg !== "object") {
|
|
337
|
+
throw new Error(`Chatbot endpoint ${epId} returned no config.`);
|
|
338
|
+
}
|
|
339
|
+
endpointConfig = cfg;
|
|
340
|
+
chatbotEndpointId = epId;
|
|
341
|
+
}
|
|
342
|
+
else {
|
|
343
|
+
const raw = await readFileOrStdin(opts.endpointConfig);
|
|
344
|
+
try {
|
|
345
|
+
endpointConfig = JSON.parse(raw);
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
throw new Error("Invalid --endpoint-config JSON.");
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
const maxTurns = opts.maxTurns ?? 12;
|
|
352
|
+
inlineIteration = {
|
|
353
|
+
name: "A",
|
|
354
|
+
details: {
|
|
355
|
+
type: "chat",
|
|
356
|
+
endpoint: endpointConfig,
|
|
357
|
+
chatbot_endpoint_id: chatbotEndpointId,
|
|
358
|
+
max_turns: maxTurns,
|
|
359
|
+
early_termination: true,
|
|
360
|
+
},
|
|
174
361
|
};
|
|
175
362
|
}
|
|
176
363
|
const resolvedWs = resolveWorkspace(opts.workspace);
|
|
@@ -193,6 +380,15 @@ Next: configure a run with \`ish iteration create --study <id>\`,
|
|
|
193
380
|
const result = data;
|
|
194
381
|
if (result.id)
|
|
195
382
|
result.alias = tagAlias(ALIAS_PREFIX.study, String(result.id));
|
|
383
|
+
const firstIteration = Array.isArray(data.iterations) && data.iterations.length > 0
|
|
384
|
+
? data.iterations[0]
|
|
385
|
+
: undefined;
|
|
386
|
+
if (firstIteration?.id) {
|
|
387
|
+
result.iteration_id = firstIteration.id;
|
|
388
|
+
}
|
|
389
|
+
if (opts.modality === "chat" && inlineIteration) {
|
|
390
|
+
result.chatbot_endpoint_id = chatbotEndpointId;
|
|
391
|
+
}
|
|
196
392
|
formatStudyDetail(result, globals.json, { writePath: true });
|
|
197
393
|
if (!globals.json && data.id) {
|
|
198
394
|
const url = getWebUrl(globals, `/${resolvedWs}/${data.id}/overview`);
|
|
@@ -276,34 +472,110 @@ list table layout in human mode.`)
|
|
|
276
472
|
.description("View aggregated results: tester counts, sentiment, interview answers. Returns a stable envelope with empty fields when no runs have completed.")
|
|
277
473
|
.argument("<id>", "Study ID")
|
|
278
474
|
.option("--workspace <id>", "Workspace ID; accepted for consistency (workspace is inferred from the study)")
|
|
475
|
+
.option("--summary", "Lean summary projection: counts + sentiment + per-tester {alias, status, sentiment, comment}. Drops interview_answers + per-interaction breakdowns.")
|
|
476
|
+
// PC-N4: agents reach for `--summarize` (verb) by analogy with the MCP
|
|
477
|
+
// `summarize` action; accept it as a hidden alias of --summary so the
|
|
478
|
+
// canonical flag stays the documented one but the muscle-memory variant
|
|
479
|
+
// works without a round-trip.
|
|
480
|
+
.addOption(new Option("--summarize", "Hidden alias for --summary").hideHelp())
|
|
481
|
+
.option("--transcript <tester_id>", "Chat transcript projection for one tester: flat role/text/turn-index array (chat-modality only). Mirrors the MCP `get_chat_transcript` shape.")
|
|
279
482
|
.addHelpText("after", `
|
|
280
483
|
Examples:
|
|
281
484
|
$ ish study results <id>
|
|
282
485
|
$ ish study results <id> --json
|
|
486
|
+
$ ish study results <id> --summary --json
|
|
487
|
+
$ ish study results <id> --transcript t-d4e --json
|
|
283
488
|
|
|
284
|
-
|
|
489
|
+
Default --json envelope (M10: per-answer sentiment now included):
|
|
285
490
|
{
|
|
286
|
-
"
|
|
287
|
-
"alias": "s-...",
|
|
288
|
-
"name": "...",
|
|
491
|
+
"study": { "alias": "s-...", "name": "...", "modality": "..." },
|
|
289
492
|
"tester_count": 12,
|
|
290
493
|
"completed_count": 8,
|
|
291
|
-
"
|
|
292
|
-
"
|
|
293
|
-
"sentiment": { "Satisfied": 5, "Frustrated": 2, "Neutral": 1 },
|
|
494
|
+
"failed_count": 0,
|
|
495
|
+
"sentiment": { "counts": { "Satisfied": 5, "Frustrated": 2 }, "total": 7 },
|
|
294
496
|
"interview_answers": [
|
|
295
|
-
{ "
|
|
296
|
-
"answers": [
|
|
497
|
+
{ "question": "...", "type": "text",
|
|
498
|
+
"answers": [
|
|
499
|
+
{ "tester_alias": "t-...", "tester_name": "...", "iteration": "A",
|
|
500
|
+
"answer": "...", "sentiment": "Satisfied" }
|
|
501
|
+
] }
|
|
502
|
+
],
|
|
503
|
+
"testers": [
|
|
504
|
+
{ "alias": "t-...", "name": "...", "iteration": "A", "status": "completed",
|
|
505
|
+
"interaction_count": 12, "sentiment": "Satisfied", "comment": "...",
|
|
506
|
+
"error_message": "..." }
|
|
507
|
+
]
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
--summary projection (M2-friction-7: drops the interview_answers payload):
|
|
511
|
+
{ study, tester_count, completed_count, failed_count, sentiment, testers: [...] }
|
|
512
|
+
|
|
513
|
+
--transcript <tester_id> projection (M2-friction-12, chat modality):
|
|
514
|
+
{
|
|
515
|
+
"tester_id": "...", "tester_alias": "t-...",
|
|
516
|
+
"instance_name": "...", "modality": "chat",
|
|
517
|
+
"transcript": [
|
|
518
|
+
{ "role": "bot", "text": "Hi…", "turn_index": 0, "failure": null },
|
|
519
|
+
{ "role": "tester", "text": "Pricing?", "turn_index": 0,
|
|
520
|
+
"action_type": "send_text", "option_label": null, "sentiment": null }
|
|
297
521
|
],
|
|
298
|
-
"
|
|
522
|
+
"unique_bot_replies": 2,
|
|
523
|
+
"tester_summary": { "comment": "...", "sentiment": {...} }
|
|
299
524
|
}
|
|
300
525
|
|
|
301
|
-
|
|
302
|
-
|
|
526
|
+
Tips:
|
|
527
|
+
Use \`--get <path>\` for a single value (e.g. \`--get tester_count\`),
|
|
528
|
+
\`--fields a,b,c\` to project the JSON output further.
|
|
529
|
+
|
|
530
|
+
Common --get paths (default envelope):
|
|
531
|
+
--get tester_count # how many testers ran
|
|
532
|
+
--get completed_count # how many finished
|
|
533
|
+
--get failed_count # how many errored
|
|
534
|
+
--get sentiment # {counts, total} histogram
|
|
535
|
+
--get sentiment.counts # bare label→count map
|
|
536
|
+
--get sentiment.total # total sentiment-tagged answers
|
|
537
|
+
--get study.modality # interactive | text | image | …
|
|
538
|
+
--get testers.alias # one alias per line
|
|
539
|
+
--get testers.0.comment # first tester's narrative comment
|
|
540
|
+
--get testers.0.sentiment # first tester's aggregate sentiment
|
|
541
|
+
--get interview_answers # full per-question payload
|
|
542
|
+
--get interview_answers.0.question # text of the first question
|
|
543
|
+
--get interview_answers.0.answers.0.answer # first answer to the first question
|
|
544
|
+
|
|
545
|
+
Common --get paths (--transcript <tester_id> envelope):
|
|
546
|
+
--get transcript # full role/text/turn array
|
|
547
|
+
--get transcript.text # one text per turn
|
|
548
|
+
--get tester_summary.comment # narrative comment
|
|
549
|
+
--get tester_summary.sentiment # aggregate sentiment map
|
|
550
|
+
--get unique_bot_replies # bot-side message count
|
|
551
|
+
|
|
552
|
+
When no runs have completed, the default envelope is returned with zero counts and empty arrays.`)
|
|
553
|
+
.action(async (id, opts, cmd) => {
|
|
303
554
|
await withClient(cmd, async (client, globals) => {
|
|
555
|
+
// PC-N4: --summarize is a hidden alias for --summary. Merge them
|
|
556
|
+
// into a single boolean before validation so the rest of the
|
|
557
|
+
// handler reads only `summary`.
|
|
558
|
+
const wantsSummary = !!(opts.summary || opts.summarize);
|
|
559
|
+
if (wantsSummary && opts.transcript) {
|
|
560
|
+
throw new ValidationError("Pass only one of: --summary, --transcript.", ["--summary", "--transcript"]);
|
|
561
|
+
}
|
|
304
562
|
const rid = resolveId(id);
|
|
563
|
+
if (opts.transcript) {
|
|
564
|
+
// --transcript <tester_id>: bypass the study aggregator; fetch
|
|
565
|
+
// the named tester directly. Cheaper (one GET, no nested
|
|
566
|
+
// iterations payload) and shapes 1:1 with the MCP transcript.
|
|
567
|
+
const testerId = resolveId(opts.transcript);
|
|
568
|
+
const tester = await client.get(`/testers/${testerId}`);
|
|
569
|
+
output(buildChatTranscript(tester), globals.json, { preProjected: true });
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
305
572
|
const data = await client.get(`/studies/${rid}`);
|
|
306
|
-
|
|
573
|
+
if (wantsSummary) {
|
|
574
|
+
output(buildStudyResultsSummary(data), globals.json, { preProjected: true });
|
|
575
|
+
}
|
|
576
|
+
else {
|
|
577
|
+
formatStudyResults(data, globals.json);
|
|
578
|
+
}
|
|
307
579
|
if (!globals.json && data.product_id) {
|
|
308
580
|
const url = getWebUrl(globals, `/${data.product_id}/${rid}/overview`);
|
|
309
581
|
console.error(`\n ${terminalLink(url, "Open in browser ↗")}\n`);
|
|
@@ -112,6 +112,38 @@ Concept pages: ish docs get-page concepts/workspace
|
|
|
112
112
|
});
|
|
113
113
|
});
|
|
114
114
|
registerSiteAccessCommands(workspace);
|
|
115
|
+
workspace
|
|
116
|
+
.command("info")
|
|
117
|
+
.description("Show workspace details + plan-limit usage counters")
|
|
118
|
+
.option("--workspace <id>", "Workspace ID; defaults to active workspace")
|
|
119
|
+
.addHelpText("after", `
|
|
120
|
+
Usage counters:
|
|
121
|
+
studies_used / studies_max — current study count vs the user's plan cap
|
|
122
|
+
testers_used / testers_max — workspace-private tester profile count vs cap
|
|
123
|
+
|
|
124
|
+
Caps fall back to null when the user's plan grants unlimited (math.inf). The
|
|
125
|
+
account tier is read from /account/me; limit tables come from /billing/limits.
|
|
126
|
+
|
|
127
|
+
Examples:
|
|
128
|
+
$ ish workspace info
|
|
129
|
+
$ ish workspace info --json | jq '{studies_used, studies_max}'`)
|
|
130
|
+
.action(async (opts, cmd) => {
|
|
131
|
+
await withClient(cmd, async (client, globals) => {
|
|
132
|
+
const wid = resolveWorkspace(opts.workspace);
|
|
133
|
+
const usage = await collectWorkspaceUsage(client, wid);
|
|
134
|
+
if (globals.json) {
|
|
135
|
+
output(usage, true);
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
const renderCap = (n) => (n === null ? "∞" : String(n));
|
|
139
|
+
console.log(`Workspace: ${usage.name ?? wid} (${tagAlias(ALIAS_PREFIX.workspace, wid)})`);
|
|
140
|
+
console.log(`ID: ${wid}`);
|
|
141
|
+
if (usage.tier)
|
|
142
|
+
console.log(`Plan: ${usage.tier}`);
|
|
143
|
+
console.log(`Studies: ${usage.studies_used} / ${renderCap(usage.studies_max)}`);
|
|
144
|
+
console.log(`Custom testers: ${usage.testers_used} / ${renderCap(usage.testers_max)}`);
|
|
145
|
+
});
|
|
146
|
+
});
|
|
115
147
|
workspace
|
|
116
148
|
.command("use")
|
|
117
149
|
.description("Set the active workspace (saved to ~/.ish/config.json)")
|
|
@@ -141,6 +173,55 @@ Concept pages: ish docs get-page concepts/workspace
|
|
|
141
173
|
});
|
|
142
174
|
});
|
|
143
175
|
}
|
|
176
|
+
async function collectWorkspaceUsage(client, workspaceId) {
|
|
177
|
+
// Workspace shape — name + base_url are nice-to-have for human render.
|
|
178
|
+
const productPromise = client.get(`/products/${workspaceId}`).catch(() => null);
|
|
179
|
+
// studies_used: list endpoint returns an array; length is the count.
|
|
180
|
+
const studiesPromise = client
|
|
181
|
+
.get(`/products/${workspaceId}/studies`)
|
|
182
|
+
.catch(() => []);
|
|
183
|
+
// testers_used: paginated list returns { total, items, ... }. Backend
|
|
184
|
+
// gates `maxCustomTesterProfiles` on visibility=private.
|
|
185
|
+
const testersPromise = client
|
|
186
|
+
.get("/tester-profiles", {
|
|
187
|
+
product_id: workspaceId,
|
|
188
|
+
visibility: "private",
|
|
189
|
+
type: "ai",
|
|
190
|
+
limit: "1",
|
|
191
|
+
offset: "0",
|
|
192
|
+
})
|
|
193
|
+
.catch(() => ({ total: 0 }));
|
|
194
|
+
// Plan caps: tier from /account/me, table from /billing/limits.
|
|
195
|
+
const acctPromise = client
|
|
196
|
+
.get("/account/me")
|
|
197
|
+
.catch(() => ({}));
|
|
198
|
+
const limitsPromise = client
|
|
199
|
+
.get("/billing/limits")
|
|
200
|
+
.catch(() => ({ tiers: {} }));
|
|
201
|
+
const [product, studies, testers, account, limits] = await Promise.all([
|
|
202
|
+
productPromise,
|
|
203
|
+
studiesPromise,
|
|
204
|
+
testersPromise,
|
|
205
|
+
acctPromise,
|
|
206
|
+
limitsPromise,
|
|
207
|
+
]);
|
|
208
|
+
const tier = typeof account.credits?.tier === "string" ? account.credits.tier : null;
|
|
209
|
+
const tierTable = tier ? limits.tiers?.[tier] ?? null : null;
|
|
210
|
+
const studiesMax = tierTable && "maxStudiesPerProduct" in tierTable ? tierTable.maxStudiesPerProduct : null;
|
|
211
|
+
const testersMax = tierTable && "maxCustomTesterProfiles" in tierTable
|
|
212
|
+
? tierTable.maxCustomTesterProfiles
|
|
213
|
+
: null;
|
|
214
|
+
return {
|
|
215
|
+
id: workspaceId,
|
|
216
|
+
name: product?.name ?? null,
|
|
217
|
+
base_url: product?.base_url ?? null,
|
|
218
|
+
tier,
|
|
219
|
+
studies_used: Array.isArray(studies) ? studies.length : 0,
|
|
220
|
+
studies_max: studiesMax,
|
|
221
|
+
testers_used: typeof testers.total === "number" ? testers.total : 0,
|
|
222
|
+
testers_max: testersMax,
|
|
223
|
+
};
|
|
224
|
+
}
|
|
144
225
|
// ---------------------------------------------------------------------------
|
|
145
226
|
// site-access — workspace-level credentials for gated test sites.
|
|
146
227
|
// Stored as product secrets with reserved keys (see src/lib/site-access.ts).
|
package/dist/config.d.ts
CHANGED
|
@@ -13,6 +13,9 @@ export interface IshConfig {
|
|
|
13
13
|
workspace?: string;
|
|
14
14
|
study?: string;
|
|
15
15
|
ask?: string;
|
|
16
|
+
/** Active chatbot endpoint id; used as the default for `ish chat endpoint *` verbs
|
|
17
|
+
* when a positional id is omitted. Set by `ish chat endpoint use <id>`. */
|
|
18
|
+
chat_endpoint?: string;
|
|
16
19
|
[key: string]: string | undefined;
|
|
17
20
|
}
|
|
18
21
|
export declare function loadConfig(): IshConfig;
|
package/dist/connect.d.ts
CHANGED
|
@@ -5,3 +5,6 @@ export declare function runTunnel(port: number, tokenArg?: string, apiUrlArg?: s
|
|
|
5
5
|
json?: boolean;
|
|
6
6
|
quiet?: boolean;
|
|
7
7
|
}): Promise<void>;
|
|
8
|
+
export declare function runDetached(port: number, apiUrlArg: string | undefined, tokenArg: string | undefined, tokenFileArg: string | undefined): Promise<void>;
|
|
9
|
+
export declare function connectStatus(json: boolean): void;
|
|
10
|
+
export declare function disconnect(json: boolean): Promise<void>;
|