@ishlabs/cli 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/iteration.js +219 -22
- package/dist/commands/profile.js +75 -9
- package/dist/commands/source.js +6 -4
- package/dist/commands/study-run.js +359 -30
- package/dist/commands/study.js +170 -9
- package/dist/commands/workspace.js +35 -2
- package/dist/lib/accessibility-profile.d.ts +12 -0
- package/dist/lib/accessibility-profile.js +136 -0
- package/dist/lib/ask-questions.js +9 -0
- package/dist/lib/billing.d.ts +55 -0
- package/dist/lib/billing.js +77 -0
- package/dist/lib/docs.js +1106 -36
- package/dist/lib/enums.d.ts +54 -0
- package/dist/lib/enums.js +100 -0
- package/dist/lib/local-sim/actions.d.ts +2 -1
- package/dist/lib/local-sim/actions.js +88 -13
- package/dist/lib/local-sim/loop.js +49 -19
- package/dist/lib/local-sim/tabs.d.ts +27 -0
- package/dist/lib/local-sim/tabs.js +157 -0
- package/dist/lib/local-sim/types.d.ts +15 -0
- package/dist/lib/modality.d.ts +70 -1
- package/dist/lib/modality.js +323 -17
- package/dist/lib/output.js +61 -4
- package/dist/lib/skill-content.js +382 -19
- package/dist/lib/types.d.ts +6 -1
- package/package.json +1 -1
|
@@ -5,11 +5,46 @@
|
|
|
5
5
|
* content/file for media). Create one before `ish study run` dispatches
|
|
6
6
|
* simulations against it.
|
|
7
7
|
*/
|
|
8
|
-
import {
|
|
8
|
+
import { readFileSync } from "node:fs";
|
|
9
|
+
import { withClient, resolveStudy, resolveWorkspace, readFileOrStdin, collectIds } from "../lib/command-helpers.js";
|
|
9
10
|
import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
10
11
|
import { output, formatIterationList, ValidationError } from "../lib/output.js";
|
|
11
12
|
import { resolveContentUrl, resolveContentUrls, resolveTextContent } from "../lib/upload.js";
|
|
12
|
-
import { isMediaModality, validateIterationDetails } from "../lib/modality.js";
|
|
13
|
+
import { isMediaModality, validateIterationDetails, normalizeChatMode, validateRoleCriteria } from "../lib/modality.js";
|
|
14
|
+
import { normalizeEnumValue, SCREEN_FORMATS } from "../lib/enums.js";
|
|
15
|
+
/**
|
|
16
|
+
* Read text inline or from a file when prefixed with `@/path/to/file`.
|
|
17
|
+
* Mirrors the `--content-text @./email.html` pattern used elsewhere.
|
|
18
|
+
*/
|
|
19
|
+
function readTextOrAtFile(value) {
|
|
20
|
+
if (value.startsWith("@")) {
|
|
21
|
+
return readFileSync(value.slice(1), "utf8");
|
|
22
|
+
}
|
|
23
|
+
return value;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Parse a JSON-blob flag that also supports `@filepath` (read from disk).
|
|
27
|
+
* Used for tester_pair `--role-criteria-a/-b` and any future blob inputs.
|
|
28
|
+
* Throws a descriptive Error on bad JSON or wrong top-level type.
|
|
29
|
+
*/
|
|
30
|
+
function parseJsonOrAtFile(raw, flagName) {
|
|
31
|
+
if (raw === undefined)
|
|
32
|
+
return undefined;
|
|
33
|
+
const text = readTextOrAtFile(raw).trim();
|
|
34
|
+
if (text.length === 0)
|
|
35
|
+
return undefined;
|
|
36
|
+
let parsed;
|
|
37
|
+
try {
|
|
38
|
+
parsed = JSON.parse(text);
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
throw new Error(`Invalid ${flagName}: expected valid JSON object.`);
|
|
42
|
+
}
|
|
43
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
44
|
+
throw new Error(`Invalid ${flagName}: expected a JSON object.`);
|
|
45
|
+
}
|
|
46
|
+
return parsed;
|
|
47
|
+
}
|
|
13
48
|
/**
|
|
14
49
|
* Pattern C / M12: project each tester row on an iteration response so its
|
|
15
50
|
* `alias` and `name` survive `leanJson` (which strips raw UUID values). The
|
|
@@ -127,9 +162,9 @@ function buildIterationDetails(modality, opts) {
|
|
|
127
162
|
...mediaExtras(opts),
|
|
128
163
|
};
|
|
129
164
|
case "chat": {
|
|
130
|
-
const
|
|
131
|
-
if (
|
|
132
|
-
throw new
|
|
165
|
+
const mode = opts.chatMode === undefined ? "external_chatbot" : normalizeChatMode(opts.chatMode);
|
|
166
|
+
if (mode === null) {
|
|
167
|
+
throw new ValidationError(`Invalid --chat-mode "${opts.chatMode}". Expected "external_chatbot" or "tester_pair" (hyphenated variants accepted: "external-chatbot", "tester-pair").`, ["external_chatbot", "tester_pair"]);
|
|
133
168
|
}
|
|
134
169
|
let maxTurns;
|
|
135
170
|
if (opts.maxTurns !== undefined) {
|
|
@@ -139,13 +174,111 @@ function buildIterationDetails(modality, opts) {
|
|
|
139
174
|
}
|
|
140
175
|
maxTurns = parsed;
|
|
141
176
|
}
|
|
142
|
-
|
|
177
|
+
const topLevel = {
|
|
143
178
|
type: "chat",
|
|
144
|
-
...(endpoint && { endpoint }),
|
|
145
|
-
...(opts.chatEndpointId && { chatbot_endpoint_id: opts.chatEndpointId }),
|
|
146
179
|
...(maxTurns !== undefined && { max_turns: maxTurns }),
|
|
147
180
|
...(opts.earlyTermination !== undefined && { early_termination: opts.earlyTermination }),
|
|
148
181
|
};
|
|
182
|
+
if (mode === "tester_pair") {
|
|
183
|
+
// Reject external-chatbot flags so users don't silently lose them.
|
|
184
|
+
const conflictingFlags = [];
|
|
185
|
+
if (opts.chatEndpointId)
|
|
186
|
+
conflictingFlags.push("--chat-endpoint-id");
|
|
187
|
+
if (opts.chatEndpointJson)
|
|
188
|
+
conflictingFlags.push("--chat-endpoint-json");
|
|
189
|
+
if (conflictingFlags.length > 0) {
|
|
190
|
+
throw new ValidationError(`--chat-mode tester_pair is incompatible with ${conflictingFlags.join(", ")}. Use --audience-a/-b or --role-criteria-a/-b plus --scenario-a/-b instead.`, conflictingFlags);
|
|
191
|
+
}
|
|
192
|
+
const audA = (opts.audienceA ?? []).map(resolveId);
|
|
193
|
+
const audB = (opts.audienceB ?? []).map(resolveId);
|
|
194
|
+
const critARaw = parseJsonOrAtFile(opts.roleCriteriaA, "--role-criteria-a");
|
|
195
|
+
const critBRaw = parseJsonOrAtFile(opts.roleCriteriaB, "--role-criteria-b");
|
|
196
|
+
let critA;
|
|
197
|
+
let critB;
|
|
198
|
+
try {
|
|
199
|
+
critA = validateRoleCriteria(critARaw, "--role-criteria-a");
|
|
200
|
+
critB = validateRoleCriteria(critBRaw, "--role-criteria-b");
|
|
201
|
+
}
|
|
202
|
+
catch (err) {
|
|
203
|
+
throw new ValidationError(err instanceof Error ? err.message : "Invalid role criteria.", ["--role-criteria-a", "--role-criteria-b"]);
|
|
204
|
+
}
|
|
205
|
+
const sideAHasInput = audA.length > 0 || !!critA;
|
|
206
|
+
const sideBHasInput = audB.length > 0 || !!critB;
|
|
207
|
+
if (!sideAHasInput || !sideBHasInput) {
|
|
208
|
+
throw new ValidationError("tester_pair chat iterations require, for each side, either an explicit audience (--audience-a / --audience-b) or a role-criteria filter (--role-criteria-a / --role-criteria-b).", ["--audience-a", "--audience-b", "--role-criteria-a", "--role-criteria-b"]);
|
|
209
|
+
}
|
|
210
|
+
// 1×N broadcast: the canonical "rehearse one side against N
|
|
211
|
+
// variations" shape. When one side has exactly one explicit
|
|
212
|
+
// profile and the other has more, broadcast the singleton so
|
|
213
|
+
// every conversation has the same fixed side and a distinct
|
|
214
|
+
// varying side. Example: --audience-a tp-rep --audience-b
|
|
215
|
+
// tp-cto1,tp-cto2,tp-cto3 → N=3 conversations, same rep vs
|
|
216
|
+
// 3 different CTO personas. Same backend wire shape, just
|
|
217
|
+
// CLI-side ergonomics.
|
|
218
|
+
let audA_final = audA;
|
|
219
|
+
let audB_final = audB;
|
|
220
|
+
let broadcastMsg;
|
|
221
|
+
if (audA.length === 1 && audB.length > 1 && !critA && !critB) {
|
|
222
|
+
audA_final = Array(audB.length).fill(audA[0]);
|
|
223
|
+
broadcastMsg = `Broadcasting --audience-a (1 profile) to length ${audB.length} to match --audience-b — same side-A profile across all ${audB.length} conversations.`;
|
|
224
|
+
}
|
|
225
|
+
else if (audB.length === 1 && audA.length > 1 && !critA && !critB) {
|
|
226
|
+
audB_final = Array(audA.length).fill(audB[0]);
|
|
227
|
+
broadcastMsg = `Broadcasting --audience-b (1 profile) to length ${audA.length} to match --audience-a — same side-B profile across all ${audA.length} conversations.`;
|
|
228
|
+
}
|
|
229
|
+
if (broadcastMsg) {
|
|
230
|
+
// stderr so it doesn't pollute --json stdout.
|
|
231
|
+
console.error(broadcastMsg);
|
|
232
|
+
}
|
|
233
|
+
// Pairing rule (after CLI's 1×N broadcast cloning above): equal
|
|
234
|
+
// counts zip 1:1 by index; mismatched counts > 1 reject. Criteria
|
|
235
|
+
// resolution happens server-side and may yield differing counts.
|
|
236
|
+
const bothExplicit = audA_final.length > 0 && audB_final.length > 0 && !critA && !critB;
|
|
237
|
+
if (bothExplicit && audA_final.length !== audB_final.length) {
|
|
238
|
+
throw new ValidationError(`--audience-a (${audA_final.length}) and --audience-b (${audB_final.length}) cannot be paired. ` +
|
|
239
|
+
`Pick the same number on each side (1:1 by index), or pass exactly one profile on one side to broadcast ` +
|
|
240
|
+
`(e.g. --audience-a tp-rep --audience-b tp-cto1,tp-cto2,tp-cto3), ` +
|
|
241
|
+
`or use --role-criteria-a/-b on either side to let the backend resolve the pool.`, ["--audience-a", "--audience-b"]);
|
|
242
|
+
}
|
|
243
|
+
if (!opts.scenarioA || !opts.scenarioB) {
|
|
244
|
+
throw new ValidationError("tester_pair chat iterations require --scenario-a and --scenario-b (text or @filepath).", ["--scenario-a", "--scenario-b"]);
|
|
245
|
+
}
|
|
246
|
+
const scenarioA = readTextOrAtFile(opts.scenarioA);
|
|
247
|
+
const scenarioB = readTextOrAtFile(opts.scenarioB);
|
|
248
|
+
if (scenarioA.trim().length === 0 || scenarioB.trim().length === 0) {
|
|
249
|
+
throw new Error("--scenario-a and --scenario-b must be non-empty.");
|
|
250
|
+
}
|
|
251
|
+
const initiatorRaw = (opts.initiatorSide ?? "a").toLowerCase();
|
|
252
|
+
if (initiatorRaw !== "a" && initiatorRaw !== "b") {
|
|
253
|
+
throw new ValidationError(`Invalid --initiator-side "${opts.initiatorSide}". Expected "a" or "b".`, ["a", "b"]);
|
|
254
|
+
}
|
|
255
|
+
return {
|
|
256
|
+
...topLevel,
|
|
257
|
+
mode_details: {
|
|
258
|
+
mode: "tester_pair",
|
|
259
|
+
audience_a: audA_final,
|
|
260
|
+
audience_b: audB_final,
|
|
261
|
+
scenario_a: scenarioA,
|
|
262
|
+
scenario_b: scenarioB,
|
|
263
|
+
initiator_side: initiatorRaw,
|
|
264
|
+
...(critA && { role_criteria_a: critA }),
|
|
265
|
+
...(critB && { role_criteria_b: critB }),
|
|
266
|
+
},
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
// external_chatbot (default)
|
|
270
|
+
const endpoint = parseJsonFlag(opts.chatEndpointJson, "--chat-endpoint-json");
|
|
271
|
+
if (!endpoint && !opts.chatEndpointId) {
|
|
272
|
+
throw new Error("Chat iterations require either --chat-endpoint-id (saved endpoint) or --chat-endpoint-json (inline config). For two-AI rehearsal use --chat-mode tester_pair with --audience-a/-b and --scenario-a/-b.");
|
|
273
|
+
}
|
|
274
|
+
return {
|
|
275
|
+
...topLevel,
|
|
276
|
+
mode_details: {
|
|
277
|
+
mode: "external_chatbot",
|
|
278
|
+
...(endpoint && { endpoint }),
|
|
279
|
+
...(opts.chatEndpointId && { chatbot_endpoint_id: opts.chatEndpointId }),
|
|
280
|
+
},
|
|
281
|
+
};
|
|
149
282
|
}
|
|
150
283
|
default:
|
|
151
284
|
if (!opts.url) {
|
|
@@ -154,11 +287,19 @@ function buildIterationDetails(modality, opts) {
|
|
|
154
287
|
if (opts.platform === "figma" && (!opts.fileKey || !opts.startNodeId)) {
|
|
155
288
|
throw new Error("Figma interactive iterations require both --file-key and --start-node-id.");
|
|
156
289
|
}
|
|
290
|
+
let screenFormat = "desktop";
|
|
291
|
+
if (opts.screenFormat !== undefined) {
|
|
292
|
+
const normalized = normalizeEnumValue(opts.screenFormat, SCREEN_FORMATS);
|
|
293
|
+
if (normalized === null) {
|
|
294
|
+
throw new ValidationError(`Invalid --screen-format "${opts.screenFormat}". Expected: ${SCREEN_FORMATS.join(" | ")} (hyphen/underscore variants accepted).`, [...SCREEN_FORMATS]);
|
|
295
|
+
}
|
|
296
|
+
screenFormat = normalized;
|
|
297
|
+
}
|
|
157
298
|
return {
|
|
158
299
|
type: "interactive",
|
|
159
300
|
platform: opts.platform || "browser",
|
|
160
301
|
url: opts.url,
|
|
161
|
-
screen_format:
|
|
302
|
+
screen_format: screenFormat,
|
|
162
303
|
...(opts.locale && { locale: opts.locale }),
|
|
163
304
|
...(opts.fileKey && { file_key: opts.fileKey }),
|
|
164
305
|
...(opts.startNodeId && { start_node_id: opts.startNodeId }),
|
|
@@ -225,7 +366,7 @@ Concept pages: ish docs get-page concepts/iteration
|
|
|
225
366
|
// Interactive
|
|
226
367
|
.option("--platform <platform>", "Platform (browser, android, figma, code) — interactive only")
|
|
227
368
|
.option("--url <url>", "URL to test — interactive only")
|
|
228
|
-
.option("--screen-format <format>", "Screen format (mobile_portrait, desktop) — interactive only")
|
|
369
|
+
.option("--screen-format <format>", "Screen format (mobile_portrait, desktop) — interactive only; hyphen/underscore variants accepted")
|
|
229
370
|
.option("--locale <locale>", "Locale code (e.g. en-US) — interactive only")
|
|
230
371
|
.option("--file-key <key>", "Figma file key — required when --platform=figma")
|
|
231
372
|
.option("--start-node-id <id>", "Figma start node id — required when --platform=figma")
|
|
@@ -252,17 +393,29 @@ Concept pages: ish docs get-page concepts/iteration
|
|
|
252
393
|
.option("--segmentation-json <json>", "Segmentation JSON — time_based {intervals_seconds, labels?}, section_based {sections[{name,label,...}]}, or page_based {} — media modalities")
|
|
253
394
|
.option("--content-config-json <json>", "Content config JSON — {early_termination, selected_segment_indices?} — media modalities")
|
|
254
395
|
// Chat modality
|
|
255
|
-
.option("--chat-
|
|
256
|
-
.option("--chat-endpoint-
|
|
396
|
+
.option("--chat-mode <mode>", "Chat mode: external_chatbot (default; probe a customer chatbot) or tester_pair (two AI audiences talk to each other)")
|
|
397
|
+
.option("--chat-endpoint-id <id>", "Saved chatbot endpoint id — chat modality, external_chatbot mode (legacy; prefer --endpoint)")
|
|
398
|
+
.option("--chat-endpoint-json <json>", "Inline chatbot endpoint config JSON — chat modality, external_chatbot mode (legacy; prefer --endpoint-config)")
|
|
257
399
|
.option("--max-turns <n>", "Max tester turns (1-50) — chat modality (default 12)")
|
|
258
400
|
.option("--early-termination", "End the chat session early when the tester signals stop — chat modality")
|
|
259
401
|
// Agent-friendly chat shortcuts: --endpoint <id> resolves a saved
|
|
260
402
|
// chatbot endpoint via alias / UUID and fetches its config inline;
|
|
261
403
|
// --endpoint-config <file> takes a raw config file (or `-` for
|
|
262
404
|
// stdin). Mutually exclusive with each other and with the legacy
|
|
263
|
-
// --chat-endpoint-* flags. Only meaningful for chat-modality studies
|
|
264
|
-
|
|
265
|
-
.option("--endpoint
|
|
405
|
+
// --chat-endpoint-* flags. Only meaningful for chat-modality studies,
|
|
406
|
+
// external_chatbot mode.
|
|
407
|
+
.option("--endpoint <id>", "Saved chatbot endpoint id (alias or UUID) — chat modality, external_chatbot mode")
|
|
408
|
+
.option("--endpoint-config <file>", "Raw ChatbotEndpointConfig JSON file or `-` for stdin — chat modality, external_chatbot mode")
|
|
409
|
+
// tester_pair (two-AI rehearsal) flags. audience_a and audience_b must
|
|
410
|
+
// be the same length — pairs are 1:1 by index. Each side has its own
|
|
411
|
+
// scenario + goal; the partner does NOT see the other side's prompt.
|
|
412
|
+
.option("--audience-a <ids>", "Tester profile IDs/aliases for audience A (comma-separated or repeatable). Pass a single profile and N on --audience-b to broadcast (1×N rehearsal: fix side A, vary side B) — chat tester_pair mode", collectIds, [])
|
|
413
|
+
.option("--audience-b <ids>", "Tester profile IDs/aliases for audience B (comma-separated or repeatable). When both sides are explicit they must be equal length, BUT if either side is a singleton it's auto-broadcast to match the other (1×N rehearsal) — chat tester_pair mode", collectIds, [])
|
|
414
|
+
.option("--scenario-a <text-or-@file>", "Side-A scenario + goal text, or @filepath — chat tester_pair mode")
|
|
415
|
+
.option("--scenario-b <text-or-@file>", "Side-B scenario + goal text, or @filepath — chat tester_pair mode")
|
|
416
|
+
.option("--initiator-side <a|b>", "Which side speaks first (default: a) — chat tester_pair mode")
|
|
417
|
+
.option("--role-criteria-a <json-or-@file>", 'RoleCriteria filter for side A (inline JSON or @filepath). Keys: occupation[], min_age, max_age, gender[], country[], education_level_in[], household_in[], locale_type_in[], income_level_in[], employment_status_in[], requires_captions, uses_screen_reader, prefers_reduced_motion, prefers_high_contrast, has_any_accessibility_need. The five *_in arrays accept snake_case spec values; the five accessibility filters are booleans. Persona-first: filters the eligible profile pool without altering personas. Use INSTEAD of --audience-a or alongside it (criteria then validates the explicit list). chat tester_pair mode.')
|
|
418
|
+
.option("--role-criteria-b <json-or-@file>", "RoleCriteria filter for side B — same shape as --role-criteria-a. chat tester_pair mode.")
|
|
266
419
|
// Escape hatch
|
|
267
420
|
.option("--details-json <json>", "Raw iteration details JSON (overrides individual flags)")
|
|
268
421
|
.addHelpText("after", `
|
|
@@ -364,6 +517,17 @@ Next: \`ish study run\` to dispatch simulations against this iteration.`)
|
|
|
364
517
|
if (!isChat && (opts.endpoint !== undefined || opts.endpointConfig !== undefined)) {
|
|
365
518
|
throw new Error(`This study uses "${modality}" modality — --endpoint / --endpoint-config are for chat studies.`);
|
|
366
519
|
}
|
|
520
|
+
const pairFlagsSet = (opts.audienceA && opts.audienceA.length > 0)
|
|
521
|
+
|| (opts.audienceB && opts.audienceB.length > 0)
|
|
522
|
+
|| opts.scenarioA !== undefined
|
|
523
|
+
|| opts.scenarioB !== undefined
|
|
524
|
+
|| opts.initiatorSide !== undefined
|
|
525
|
+
|| opts.roleCriteriaA !== undefined
|
|
526
|
+
|| opts.roleCriteriaB !== undefined
|
|
527
|
+
|| normalizeChatMode(opts.chatMode) === "tester_pair";
|
|
528
|
+
if (!isChat && pairFlagsSet) {
|
|
529
|
+
throw new Error(`This study uses "${modality}" modality — --chat-mode / --audience-a/-b / --scenario-a/-b / --role-criteria-a/-b are for chat studies.`);
|
|
530
|
+
}
|
|
367
531
|
// Validate per-modality required flags BEFORE any upload so we don't
|
|
368
532
|
// orphan blobs in storage when the wrong flag is passed (e.g.
|
|
369
533
|
// --content-url to an image-modality study).
|
|
@@ -389,18 +553,47 @@ Next: \`ish study run\` to dispatch simulations against this iteration.`)
|
|
|
389
553
|
throw new Error("Document iterations require --content-url. Provide the URL or local file path.");
|
|
390
554
|
}
|
|
391
555
|
break;
|
|
392
|
-
case "chat":
|
|
556
|
+
case "chat": {
|
|
557
|
+
const mode = opts.chatMode === undefined ? "external_chatbot" : normalizeChatMode(opts.chatMode);
|
|
558
|
+
if (mode === null) {
|
|
559
|
+
throw new ValidationError(`Invalid --chat-mode "${opts.chatMode}". Expected "external_chatbot" or "tester_pair" (hyphenated variants accepted).`, ["external_chatbot", "tester_pair"]);
|
|
560
|
+
}
|
|
561
|
+
if (mode === "tester_pair") {
|
|
562
|
+
const conflicting = [];
|
|
563
|
+
if (opts.endpoint)
|
|
564
|
+
conflicting.push("--endpoint");
|
|
565
|
+
if (opts.endpointConfig)
|
|
566
|
+
conflicting.push("--endpoint-config");
|
|
567
|
+
if (opts.chatEndpointId)
|
|
568
|
+
conflicting.push("--chat-endpoint-id");
|
|
569
|
+
if (opts.chatEndpointJson)
|
|
570
|
+
conflicting.push("--chat-endpoint-json");
|
|
571
|
+
if (conflicting.length > 0) {
|
|
572
|
+
throw new ValidationError(`--chat-mode tester_pair is incompatible with ${conflicting.join(", ")}. Use --audience-a/-b or --role-criteria-a/-b plus --scenario-a/-b instead.`, conflicting);
|
|
573
|
+
}
|
|
574
|
+
const sideAHasInput = (opts.audienceA && opts.audienceA.length > 0) || !!opts.roleCriteriaA;
|
|
575
|
+
const sideBHasInput = (opts.audienceB && opts.audienceB.length > 0) || !!opts.roleCriteriaB;
|
|
576
|
+
if (!sideAHasInput || !sideBHasInput) {
|
|
577
|
+
throw new Error("tester_pair chat iterations require, for each side, either an explicit audience (--audience-a / --audience-b) or a role-criteria filter (--role-criteria-a / --role-criteria-b).");
|
|
578
|
+
}
|
|
579
|
+
if (!opts.scenarioA || !opts.scenarioB) {
|
|
580
|
+
throw new Error("tester_pair chat iterations require --scenario-a <text-or-@file> and --scenario-b <text-or-@file>.");
|
|
581
|
+
}
|
|
582
|
+
break;
|
|
583
|
+
}
|
|
584
|
+
// external_chatbot mode (default)
|
|
393
585
|
if (!opts.chatEndpointId
|
|
394
586
|
&& !opts.chatEndpointJson
|
|
395
587
|
&& !opts.endpoint
|
|
396
588
|
&& !opts.endpointConfig) {
|
|
397
|
-
throw new Error("Chat iterations require one of: --endpoint <id>, --endpoint-config <file>, --chat-endpoint-id, or --chat-endpoint-json.");
|
|
589
|
+
throw new Error("Chat iterations (external_chatbot mode) require one of: --endpoint <id>, --endpoint-config <file>, --chat-endpoint-id, or --chat-endpoint-json. For two-AI rehearsal use --chat-mode tester_pair.");
|
|
398
590
|
}
|
|
399
591
|
if ((opts.endpoint || opts.endpointConfig)
|
|
400
592
|
&& (opts.chatEndpointId || opts.chatEndpointJson)) {
|
|
401
593
|
throw new ValidationError("Pass only one of: --endpoint / --endpoint-config (preferred) or the legacy --chat-endpoint-id / --chat-endpoint-json.", ["--endpoint", "--endpoint-config"]);
|
|
402
594
|
}
|
|
403
595
|
break;
|
|
596
|
+
}
|
|
404
597
|
default:
|
|
405
598
|
if (!opts.url) {
|
|
406
599
|
throw new Error("Interactive iterations require --url. Provide the URL to test.");
|
|
@@ -427,9 +620,10 @@ Next: \`ish study run\` to dispatch simulations against this iteration.`)
|
|
|
427
620
|
if (isChat && (opts.endpoint || opts.endpointConfig)) {
|
|
428
621
|
// Agent-friendly path: fetch a saved endpoint by alias / UUID
|
|
429
622
|
// (or read a raw config from a file / stdin) and embed the full
|
|
430
|
-
// ChatbotEndpointConfig inline at
|
|
431
|
-
// Backend's chat
|
|
432
|
-
//
|
|
623
|
+
// ChatbotEndpointConfig inline at
|
|
624
|
+
// `iteration.details.mode_details.endpoint`. Backend's chat
|
|
625
|
+
// iteration shape now wraps mode-specific fields under
|
|
626
|
+
// `mode_details` with a `mode` discriminator.
|
|
433
627
|
let endpointConfig;
|
|
434
628
|
let chatbotEndpointId = null;
|
|
435
629
|
if (opts.endpoint) {
|
|
@@ -463,8 +657,11 @@ Next: \`ish study run\` to dispatch simulations against this iteration.`)
|
|
|
463
657
|
const earlyTermination = opts.earlyTermination !== undefined ? opts.earlyTermination : true;
|
|
464
658
|
details = {
|
|
465
659
|
type: "chat",
|
|
466
|
-
|
|
467
|
-
|
|
660
|
+
mode_details: {
|
|
661
|
+
mode: "external_chatbot",
|
|
662
|
+
endpoint: endpointConfig,
|
|
663
|
+
...(chatbotEndpointId && { chatbot_endpoint_id: chatbotEndpointId }),
|
|
664
|
+
},
|
|
468
665
|
max_turns: maxTurns,
|
|
469
666
|
early_termination: earlyTermination,
|
|
470
667
|
};
|
package/dist/commands/profile.js
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* ish profile — Manage profiles, audience generation, and source uploads.
|
|
3
3
|
*/
|
|
4
|
+
import fs from "node:fs";
|
|
4
5
|
import { withClient, readJsonFileOrStdin, resolveWorkspace } from "../lib/command-helpers.js";
|
|
5
6
|
import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
6
7
|
import { formatTesterProfileList, formatGeneratedProfileList, output, } from "../lib/output.js";
|
|
7
8
|
import { resolveTextContent } from "../lib/upload.js";
|
|
8
9
|
import { isUuid, resolveSourceRef } from "../lib/profile-sources.js";
|
|
10
|
+
import { assertEnumValue, EDUCATION_LEVELS, HOUSEHOLDS, LOCALE_TYPES, INCOME_LEVELS, EMPLOYMENT_STATUSES, } from "../lib/enums.js";
|
|
11
|
+
import { validateAccessibilityProfile } from "../lib/accessibility-profile.js";
|
|
9
12
|
function collect(value, prev) {
|
|
10
13
|
return prev.concat(value);
|
|
11
14
|
}
|
|
@@ -301,16 +304,34 @@ list table layout in human mode. Use --fields to project per-item.`)
|
|
|
301
304
|
.option("--country <code>", "Country code, e.g. US")
|
|
302
305
|
.option("--gender <g>", "Gender, e.g. female")
|
|
303
306
|
.option("--date-of-birth <YYYY-MM-DD>", "Date of birth")
|
|
304
|
-
.option("--
|
|
307
|
+
.option("--education-level <value>", `Education level. One of: ${EDUCATION_LEVELS.join(", ")}`)
|
|
308
|
+
.option("--household <value>", `Household composition (MECE). One of: ${HOUSEHOLDS.join(", ")}. A couple raising children is couple_with_kids, not couple_no_kids; "single" means lives alone with no partner, roommates, parents, or children in the household.`)
|
|
309
|
+
.option("--locale-type <value>", `Self-described neighborhood type. One of: ${LOCALE_TYPES.join(", ")}`)
|
|
310
|
+
.option("--income-level <value>", `Self-identified relative socioeconomic position. One of: ${INCOME_LEVELS.join(", ")}`)
|
|
311
|
+
.option("--employment-status <value>", `Primary daytime activity / labor-force status. One of: ${EMPLOYMENT_STATUSES.join(", ")}`)
|
|
312
|
+
.option("--accessibility-profile <json-or-path>", "AccessibilityProfile v1.0 as an inline JSON string OR a path to a JSON file. Empty object {} is the canonical default. Validated client-side against the spec before submit.")
|
|
305
313
|
.addHelpText("after", `
|
|
306
314
|
Examples:
|
|
307
315
|
$ ish profile update <id> --bio "Edited bio"
|
|
308
|
-
$ ish profile update <id> --name "Alice" --country US --
|
|
316
|
+
$ ish profile update <id> --name "Alice" --country US --education-level bachelor
|
|
317
|
+
$ ish profile update <id> --household couple_with_kids --locale-type suburban
|
|
318
|
+
$ ish profile update <id> --income-level middle --employment-status employed_full_time
|
|
319
|
+
$ ish profile update <id> --accessibility-profile '{"version":"1.0","visual":{"uses_screen_reader":true,"text_size":"large"},"cognitive":{"reduce_motion":true},"assistive_tech":["VoiceOver"]}'
|
|
320
|
+
$ ish profile update <id> --accessibility-profile ./a11y.json
|
|
309
321
|
$ ish profile update <id> --file updates.json
|
|
310
322
|
|
|
311
323
|
Inline flags compose into the patch body. --file is an escape hatch when you
|
|
312
324
|
need fields not covered by the inline flags. When both are provided, inline
|
|
313
|
-
flags override values from --file
|
|
325
|
+
flags override values from --file.
|
|
326
|
+
|
|
327
|
+
Household MECE rule: a couple raising children is \`couple_with_kids\`, not
|
|
328
|
+
\`couple_no_kids\`. \`single\` means lives alone with no partner, roommates,
|
|
329
|
+
parents, or children sharing the household.
|
|
330
|
+
|
|
331
|
+
\`--accessibility-profile\` accepts either an inline JSON string OR a path to
|
|
332
|
+
a JSON file. An empty object \`{}\` means "no accessibility configuration
|
|
333
|
+
declared". When non-empty, \`version\` is required and must be \`"1.0"\`.
|
|
334
|
+
Schema: https://ishlabs.io/spec/accessibility-profile-schema.v1.json`)
|
|
314
335
|
.action(async (id, opts, cmd) => {
|
|
315
336
|
await withClient(cmd, async (client, globals) => {
|
|
316
337
|
let body = {};
|
|
@@ -331,12 +352,23 @@ flags override values from --file.`)
|
|
|
331
352
|
body.gender = opts.gender;
|
|
332
353
|
if (opts.dateOfBirth !== undefined)
|
|
333
354
|
body.date_of_birth = opts.dateOfBirth;
|
|
334
|
-
if (opts.
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
355
|
+
if (opts.educationLevel !== undefined) {
|
|
356
|
+
body.education_level = assertEnumValue(opts.educationLevel, EDUCATION_LEVELS, "--education-level");
|
|
357
|
+
}
|
|
358
|
+
if (opts.household !== undefined) {
|
|
359
|
+
body.household = assertEnumValue(opts.household, HOUSEHOLDS, "--household");
|
|
360
|
+
}
|
|
361
|
+
if (opts.localeType !== undefined) {
|
|
362
|
+
body.locale_type = assertEnumValue(opts.localeType, LOCALE_TYPES, "--locale-type");
|
|
363
|
+
}
|
|
364
|
+
if (opts.incomeLevel !== undefined) {
|
|
365
|
+
body.income_level = assertEnumValue(opts.incomeLevel, INCOME_LEVELS, "--income-level");
|
|
366
|
+
}
|
|
367
|
+
if (opts.employmentStatus !== undefined) {
|
|
368
|
+
body.employment_status = assertEnumValue(opts.employmentStatus, EMPLOYMENT_STATUSES, "--employment-status");
|
|
369
|
+
}
|
|
370
|
+
if (opts.accessibilityProfile !== undefined) {
|
|
371
|
+
body.accessibility_profile = parseAccessibilityProfileFlag(opts.accessibilityProfile);
|
|
340
372
|
}
|
|
341
373
|
if (Object.keys(body).length === 0) {
|
|
342
374
|
throw new Error("Nothing to update. Provide --file or at least one inline flag (e.g. --bio).");
|
|
@@ -377,3 +409,37 @@ function tryResolveSourceAlias(value) {
|
|
|
377
409
|
return undefined;
|
|
378
410
|
}
|
|
379
411
|
}
|
|
412
|
+
/**
|
|
413
|
+
* Resolve `--accessibility-profile` from either an inline JSON string or a
|
|
414
|
+
* filesystem path, then validate against the v1.0 schema. Returns the
|
|
415
|
+
* canonical object (an empty `{}` means "no accessibility configuration").
|
|
416
|
+
*/
|
|
417
|
+
function parseAccessibilityProfileFlag(raw) {
|
|
418
|
+
const trimmed = raw.trim();
|
|
419
|
+
let parsed;
|
|
420
|
+
const looksLikeJson = trimmed.startsWith("{") || trimmed.startsWith("[");
|
|
421
|
+
if (looksLikeJson) {
|
|
422
|
+
try {
|
|
423
|
+
parsed = JSON.parse(trimmed);
|
|
424
|
+
}
|
|
425
|
+
catch (e) {
|
|
426
|
+
throw new Error(`Invalid --accessibility-profile: JSON parse failed (${e.message}).`);
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
else {
|
|
430
|
+
let content;
|
|
431
|
+
try {
|
|
432
|
+
content = fs.readFileSync(trimmed, "utf-8");
|
|
433
|
+
}
|
|
434
|
+
catch {
|
|
435
|
+
throw new Error(`Invalid --accessibility-profile: "${trimmed}" is neither inline JSON (must start with "{") nor a readable file.`);
|
|
436
|
+
}
|
|
437
|
+
try {
|
|
438
|
+
parsed = JSON.parse(content);
|
|
439
|
+
}
|
|
440
|
+
catch (e) {
|
|
441
|
+
throw new Error(`Invalid --accessibility-profile: file "${trimmed}" is not valid JSON (${e.message}).`);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
return validateAccessibilityProfile(parsed);
|
|
445
|
+
}
|
package/dist/commands/source.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { withClient, resolveWorkspace } from "../lib/command-helpers.js";
|
|
10
10
|
import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
11
|
+
import { normalizeEnumValue } from "../lib/enums.js";
|
|
11
12
|
import { formatAudienceSource, output } from "../lib/output.js";
|
|
12
13
|
import { inferSourceKind, uploadAndProcessSource, } from "../lib/profile-sources.js";
|
|
13
14
|
const VALID_KINDS = ["text_file", "audio", "image"];
|
|
@@ -27,7 +28,7 @@ Concept pages: ish docs get-page concepts/source
|
|
|
27
28
|
.description("Upload a file as an audience source and wait for processing")
|
|
28
29
|
.argument("<file>", "Local file path (transcript, audio, image, PDF, etc.)")
|
|
29
30
|
.option("--workspace <id>", "Workspace (product) ID; falls back to active workspace")
|
|
30
|
-
.option("--kind <kind>", "Source kind: text_file | audio | image (auto-detected if omitted)")
|
|
31
|
+
.option("--kind <kind>", "Source kind: text_file | audio | image (auto-detected if omitted; hyphen/underscore variants accepted)")
|
|
31
32
|
.option("--description <text>", "Context note attached to the source (max 500 chars)")
|
|
32
33
|
.option("--diarize", "Apply speaker diarization to audio sources (silently ignored for text/image)")
|
|
33
34
|
.option("--no-wait", "Don't poll until terminal status — return after confirm")
|
|
@@ -42,10 +43,11 @@ Examples:
|
|
|
42
43
|
const productId = resolveWorkspace(opts.workspace);
|
|
43
44
|
let kind;
|
|
44
45
|
if (opts.kind) {
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
const normalized = normalizeEnumValue(opts.kind, VALID_KINDS);
|
|
47
|
+
if (normalized === null) {
|
|
48
|
+
throw new Error(`Invalid --kind "${opts.kind}". Valid: ${VALID_KINDS.join(", ")} (hyphen/underscore variants accepted).`);
|
|
47
49
|
}
|
|
48
|
-
kind =
|
|
50
|
+
kind = normalized;
|
|
49
51
|
}
|
|
50
52
|
else {
|
|
51
53
|
kind = inferSourceKind(file);
|