@ishlabs/cli 0.17.7 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -54
- package/dist/commands/ask.d.ts +4 -4
- package/dist/commands/ask.js +66 -66
- package/dist/commands/chat.js +10 -10
- package/dist/commands/config.js +1 -1
- package/dist/commands/docs.js +1 -1
- package/dist/commands/iteration.js +57 -57
- package/dist/commands/mcp.d.ts +23 -0
- package/dist/commands/mcp.js +676 -0
- package/dist/commands/person.d.ts +5 -0
- package/dist/commands/{profile.js → person.js} +197 -162
- package/dist/commands/source.d.ts +6 -2
- package/dist/commands/source.js +35 -30
- package/dist/commands/study-analyze.d.ts +1 -1
- package/dist/commands/study-analyze.js +3 -3
- package/dist/commands/study-participant.d.ts +8 -0
- package/dist/commands/{study-tester.js → study-participant.js} +50 -50
- package/dist/commands/study-run.d.ts +6 -6
- package/dist/commands/study-run.js +341 -290
- package/dist/commands/study.js +106 -72
- package/dist/commands/workspace.js +13 -13
- package/dist/connect.js +5 -5
- package/dist/index.js +6 -4
- package/dist/lib/accessibility-profile.d.ts +1 -1
- package/dist/lib/accessibility-profile.js +1 -1
- package/dist/lib/alias-hydrate.js +4 -4
- package/dist/lib/alias-store.d.ts +5 -5
- package/dist/lib/alias-store.js +8 -8
- package/dist/lib/api-client.d.ts +1 -1
- package/dist/lib/api-client.js +1 -1
- package/dist/lib/billing.d.ts +11 -11
- package/dist/lib/billing.js +16 -16
- package/dist/lib/chat-endpoint-templates.js +1 -1
- package/dist/lib/command-helpers.d.ts +18 -18
- package/dist/lib/command-helpers.js +49 -37
- package/dist/lib/docs.js +570 -387
- package/dist/lib/enums.d.ts +2 -2
- package/dist/lib/enums.js +2 -2
- package/dist/lib/local-sim/browser.d.ts +1 -1
- package/dist/lib/local-sim/browser.js +1 -1
- package/dist/lib/local-sim/debug-report.d.ts +2 -2
- package/dist/lib/local-sim/debug-report.js +3 -3
- package/dist/lib/local-sim/loop.d.ts +5 -5
- package/dist/lib/local-sim/loop.js +38 -38
- package/dist/lib/local-sim/types.d.ts +12 -12
- package/dist/lib/mcp-clients.d.ts +51 -0
- package/dist/lib/mcp-clients.js +175 -0
- package/dist/lib/modality.d.ts +10 -10
- package/dist/lib/modality.js +46 -46
- package/dist/lib/output.d.ts +16 -15
- package/dist/lib/output.js +291 -226
- package/dist/lib/profile-sources.d.ts +64 -16
- package/dist/lib/profile-sources.js +91 -30
- package/dist/lib/skill-content.js +216 -168
- package/dist/lib/study-events.d.ts +3 -3
- package/dist/lib/study-events.js +1 -1
- package/dist/lib/study-inputs.d.ts +11 -1
- package/dist/lib/study-inputs.js +68 -17
- package/dist/lib/study-participants.d.ts +32 -0
- package/dist/lib/study-participants.js +12 -0
- package/dist/lib/types.d.ts +104 -34
- package/package.json +1 -1
- package/dist/commands/profile.d.ts +0 -5
- package/dist/commands/study-tester.d.ts +0 -8
package/dist/lib/modality.js
CHANGED
|
@@ -44,28 +44,28 @@ export function iterationHasContent(details, modality) {
|
|
|
44
44
|
}
|
|
45
45
|
if (modality === "chat") {
|
|
46
46
|
const mode = readChatMode(details);
|
|
47
|
-
if (mode === "
|
|
48
|
-
const pair =
|
|
47
|
+
if (mode === "participant_pair") {
|
|
48
|
+
const pair = readParticipantPairConfig(details);
|
|
49
49
|
if (!pair)
|
|
50
50
|
return false;
|
|
51
|
-
// Each side needs *either* an explicit
|
|
51
|
+
// Each side needs *either* an explicit group or a role-criteria
|
|
52
52
|
// filter. Scenarios are always required (they're how the persona
|
|
53
53
|
// gets a role to inhabit).
|
|
54
|
-
const
|
|
55
|
-
const
|
|
56
|
-
if (!
|
|
54
|
+
const sideAHasGroup = pair.group_a.length > 0 || !!pair.role_criteria_a;
|
|
55
|
+
const sideBHasGroup = pair.group_b.length > 0 || !!pair.role_criteria_b;
|
|
56
|
+
if (!sideAHasGroup || !sideBHasGroup)
|
|
57
57
|
return false;
|
|
58
58
|
if (pair.scenario_a.length === 0 || pair.scenario_b.length === 0)
|
|
59
59
|
return false;
|
|
60
60
|
// Equal-length pairing is only enforced when *both* sides ship an
|
|
61
|
-
// explicit
|
|
62
|
-
// and persisted back to
|
|
61
|
+
// explicit group. Criteria-driven sides are resolved server-side
|
|
62
|
+
// and persisted back to group_*; until then the lengths may
|
|
63
63
|
// legitimately differ (or be zero).
|
|
64
|
-
const
|
|
65
|
-
&& pair.
|
|
64
|
+
const bothGroupsExplicit = pair.group_a.length > 0
|
|
65
|
+
&& pair.group_b.length > 0
|
|
66
66
|
&& !pair.role_criteria_a
|
|
67
67
|
&& !pair.role_criteria_b;
|
|
68
|
-
if (
|
|
68
|
+
if (bothGroupsExplicit && pair.group_a.length !== pair.group_b.length) {
|
|
69
69
|
return false;
|
|
70
70
|
}
|
|
71
71
|
return true;
|
|
@@ -89,8 +89,8 @@ export function describeRequiredContentFlag(modality, chatMode) {
|
|
|
89
89
|
if (modality === "image")
|
|
90
90
|
return "--image-urls <comma-separated>";
|
|
91
91
|
if (modality === "chat") {
|
|
92
|
-
if (chatMode === "
|
|
93
|
-
return "--chat-mode
|
|
92
|
+
if (chatMode === "participant_pair") {
|
|
93
|
+
return "--chat-mode participant_pair (--group-a <ids> | --role-criteria-a <json-or-@file>) (--group-b <ids> | --role-criteria-b <json-or-@file>) --scenario-a <text-or-@file> --scenario-b <text-or-@file>";
|
|
94
94
|
}
|
|
95
95
|
return "--endpoint <id> | --endpoint-config <file>";
|
|
96
96
|
}
|
|
@@ -103,10 +103,10 @@ export function describeRequiredContentFlag(modality, chatMode) {
|
|
|
103
103
|
* the read helpers below fall back to that legacy shape and treat it as
|
|
104
104
|
* `external_chatbot`.
|
|
105
105
|
*/
|
|
106
|
-
const CHAT_MODES = ["external_chatbot", "
|
|
106
|
+
const CHAT_MODES = ["external_chatbot", "participant_pair"];
|
|
107
107
|
/**
|
|
108
|
-
* Normalise user-supplied `--chat-mode` values. Accepts both `
|
|
109
|
-
* (canonical) and `
|
|
108
|
+
* Normalise user-supplied `--chat-mode` values. Accepts both `participant_pair`
|
|
109
|
+
* (canonical) and `participant-pair` (hyphenated — matches CLI flag convention
|
|
110
110
|
* elsewhere in `ish`); same for `external-chatbot`. Returns `null` for
|
|
111
111
|
* anything unrecognised so callers can throw a clean ValidationError.
|
|
112
112
|
*/
|
|
@@ -120,8 +120,8 @@ export function readChatMode(details) {
|
|
|
120
120
|
const md = details.mode_details;
|
|
121
121
|
if (md && typeof md === "object") {
|
|
122
122
|
const mode = md.mode;
|
|
123
|
-
if (mode === "
|
|
124
|
-
return "
|
|
123
|
+
if (mode === "participant_pair")
|
|
124
|
+
return "participant_pair";
|
|
125
125
|
if (mode === "external_chatbot")
|
|
126
126
|
return "external_chatbot";
|
|
127
127
|
}
|
|
@@ -155,22 +155,22 @@ export function readExternalChatbotEndpoint(details) {
|
|
|
155
155
|
};
|
|
156
156
|
}
|
|
157
157
|
/**
|
|
158
|
-
* Extract the
|
|
159
|
-
* `mode_details.mode !== "
|
|
158
|
+
* Extract the participant-pair payload from chat details. Returns `undefined` if
|
|
159
|
+
* `mode_details.mode !== "participant_pair"`. Does not validate equal lengths or
|
|
160
160
|
* non-empty scenarios — that's the caller's job (see `iterationHasContent`
|
|
161
161
|
* and `validateIterationDetails`).
|
|
162
162
|
*/
|
|
163
|
-
export function
|
|
163
|
+
export function readParticipantPairConfig(details) {
|
|
164
164
|
if (!details || typeof details !== "object")
|
|
165
165
|
return undefined;
|
|
166
166
|
const md = details.mode_details;
|
|
167
167
|
if (!md || typeof md !== "object")
|
|
168
168
|
return undefined;
|
|
169
169
|
const m = md;
|
|
170
|
-
if (m.mode !== "
|
|
170
|
+
if (m.mode !== "participant_pair")
|
|
171
171
|
return undefined;
|
|
172
|
-
const audA = Array.isArray(m.
|
|
173
|
-
const audB = Array.isArray(m.
|
|
172
|
+
const audA = Array.isArray(m.group_a) ? m.group_a.filter((x) => typeof x === "string") : [];
|
|
173
|
+
const audB = Array.isArray(m.group_b) ? m.group_b.filter((x) => typeof x === "string") : [];
|
|
174
174
|
const scenA = typeof m.scenario_a === "string" ? m.scenario_a : "";
|
|
175
175
|
const scenB = typeof m.scenario_b === "string" ? m.scenario_b : "";
|
|
176
176
|
const init = m.initiator_side === "b" ? "b" : "a";
|
|
@@ -181,8 +181,8 @@ export function readTesterPairConfig(details) {
|
|
|
181
181
|
? m.role_criteria_b
|
|
182
182
|
: undefined;
|
|
183
183
|
return {
|
|
184
|
-
|
|
185
|
-
|
|
184
|
+
group_a: audA,
|
|
185
|
+
group_b: audB,
|
|
186
186
|
scenario_a: scenA,
|
|
187
187
|
scenario_b: scenB,
|
|
188
188
|
initiator_side: init,
|
|
@@ -191,8 +191,8 @@ export function readTesterPairConfig(details) {
|
|
|
191
191
|
};
|
|
192
192
|
}
|
|
193
193
|
/**
|
|
194
|
-
* RoleCriteria — persona-first filter that resolves a
|
|
195
|
-
* for one side of a
|
|
194
|
+
* RoleCriteria — persona-first filter that resolves a person pool
|
|
195
|
+
* for one side of a participant_pair iteration. Mirrors the backend Pydantic
|
|
196
196
|
* shape in `app/api/models/iterations.py:RoleCriteria` (swift-charm plan).
|
|
197
197
|
* All fields optional; an empty object is treated as "no filter".
|
|
198
198
|
*/
|
|
@@ -328,7 +328,7 @@ export function validateIterationDetails(modality, details) {
|
|
|
328
328
|
const hasEndpointObj = !!ep.endpoint;
|
|
329
329
|
const hasEndpointId = !!ep.chatbot_endpoint_id;
|
|
330
330
|
const hasModeDetails = !!d.mode_details && typeof d.mode_details === "object";
|
|
331
|
-
const isPair = readChatMode(details) === "
|
|
331
|
+
const isPair = readChatMode(details) === "participant_pair";
|
|
332
332
|
const hasContentText = typeof d.content_text === "string" && d.content_text.length > 0;
|
|
333
333
|
const hasContentUrl = typeof d.content_url === "string" && d.content_url.length > 0;
|
|
334
334
|
const imageUrls = d.image_urls;
|
|
@@ -337,12 +337,12 @@ export function validateIterationDetails(modality, details) {
|
|
|
337
337
|
: typeof imageUrls === "string" && imageUrls.length > 0;
|
|
338
338
|
if (modality === "chat") {
|
|
339
339
|
if (isPair) {
|
|
340
|
-
const pair =
|
|
340
|
+
const pair = readParticipantPairConfig(details);
|
|
341
341
|
if (!pair) {
|
|
342
342
|
return {
|
|
343
343
|
ok: false,
|
|
344
|
-
reason: "missing or malformed mode_details for
|
|
345
|
-
suggestion: "
|
|
344
|
+
reason: "missing or malformed mode_details for participant_pair chat iteration",
|
|
345
|
+
suggestion: "participant_pair iterations require mode_details: { mode: 'participant_pair', group_a|role_criteria_a, group_b|role_criteria_b, scenario_a, scenario_b }",
|
|
346
346
|
};
|
|
347
347
|
}
|
|
348
348
|
// Validate criteria shape if present; surfaces client-side errors
|
|
@@ -358,33 +358,33 @@ export function validateIterationDetails(modality, details) {
|
|
|
358
358
|
suggestion: `allowed keys: ${ROLE_CRITERIA_KEYS.join(", ")}`,
|
|
359
359
|
};
|
|
360
360
|
}
|
|
361
|
-
const
|
|
362
|
-
const
|
|
363
|
-
if (!
|
|
361
|
+
const sideAHasGroup = pair.group_a.length > 0 || !!pair.role_criteria_a;
|
|
362
|
+
const sideBHasGroup = pair.group_b.length > 0 || !!pair.role_criteria_b;
|
|
363
|
+
if (!sideAHasGroup || !sideBHasGroup) {
|
|
364
364
|
return {
|
|
365
365
|
ok: false,
|
|
366
|
-
reason: "
|
|
367
|
-
suggestion: "each side needs either an explicit
|
|
366
|
+
reason: "participant_pair iteration is missing group input on at least one side",
|
|
367
|
+
suggestion: "each side needs either an explicit group (group_a/group_b) or a role-criteria filter (role_criteria_a/role_criteria_b)",
|
|
368
368
|
};
|
|
369
369
|
}
|
|
370
|
-
const
|
|
371
|
-
&& pair.
|
|
370
|
+
const bothGroupsExplicit = pair.group_a.length > 0
|
|
371
|
+
&& pair.group_b.length > 0
|
|
372
372
|
&& !pair.role_criteria_a
|
|
373
373
|
&& !pair.role_criteria_b;
|
|
374
|
-
const lenA = pair.
|
|
375
|
-
const lenB = pair.
|
|
374
|
+
const lenA = pair.group_a.length;
|
|
375
|
+
const lenB = pair.group_b.length;
|
|
376
376
|
const validPairing = lenA === lenB || lenA === 1 || lenB === 1;
|
|
377
|
-
if (
|
|
377
|
+
if (bothGroupsExplicit && !validPairing) {
|
|
378
378
|
return {
|
|
379
379
|
ok: false,
|
|
380
|
-
reason: `
|
|
380
|
+
reason: `participant_pair groups cannot be paired (group_a=${lenA}, group_b=${lenB})`,
|
|
381
381
|
suggestion: "pick the same number on each side (zip 1:1 by index), or exactly 1 on one side to broadcast across the other; or use role_criteria_a/_b to let the backend resolve the pool",
|
|
382
382
|
};
|
|
383
383
|
}
|
|
384
384
|
if (pair.scenario_a.length === 0 || pair.scenario_b.length === 0) {
|
|
385
385
|
return {
|
|
386
386
|
ok: false,
|
|
387
|
-
reason: "
|
|
387
|
+
reason: "participant_pair iteration is missing scenario_a or scenario_b",
|
|
388
388
|
suggestion: "both scenarios are required and asymmetric; pass --scenario-a and --scenario-b on `iteration create`",
|
|
389
389
|
};
|
|
390
390
|
}
|
|
@@ -396,7 +396,7 @@ export function validateIterationDetails(modality, details) {
|
|
|
396
396
|
return {
|
|
397
397
|
ok: false,
|
|
398
398
|
reason: "unrecognised mode_details for chat iteration",
|
|
399
|
-
suggestion: "mode_details must be either { mode:'external_chatbot', endpoint|chatbot_endpoint_id } or { mode:'
|
|
399
|
+
suggestion: "mode_details must be either { mode:'external_chatbot', endpoint|chatbot_endpoint_id } or { mode:'participant_pair', group_a, group_b, scenario_a, scenario_b }",
|
|
400
400
|
};
|
|
401
401
|
}
|
|
402
402
|
if (hasUrl) {
|
|
@@ -410,7 +410,7 @@ export function validateIterationDetails(modality, details) {
|
|
|
410
410
|
return {
|
|
411
411
|
ok: false,
|
|
412
412
|
reason: "media-shape details on a chat iteration",
|
|
413
|
-
suggestion: "chat iterations require mode_details with endpoint/chatbot_endpoint_id (external_chatbot mode) or
|
|
413
|
+
suggestion: "chat iterations require mode_details with endpoint/chatbot_endpoint_id (external_chatbot mode) or group_a/_b + scenario_a/_b (participant_pair mode)",
|
|
414
414
|
};
|
|
415
415
|
}
|
|
416
416
|
// No recognised content keys at all. Don't reject — agent might be
|
package/dist/lib/output.d.ts
CHANGED
|
@@ -47,36 +47,37 @@ export declare function formatWorkspaceList(workspaces: Record<string, unknown>[
|
|
|
47
47
|
export declare function formatWorkspaceDetail(workspace: Record<string, unknown>, json: boolean, options?: OutputOptions): void;
|
|
48
48
|
export declare function formatSiteAccessStatus(summary: import("./site-access.js").SiteAccessSummary, json: boolean): void;
|
|
49
49
|
export declare function formatStudyList(studies: Record<string, unknown>[], json: boolean): void;
|
|
50
|
-
export declare function formatStudyDetail(study: Record<string, unknown>, json: boolean, options?: OutputOptions): void;
|
|
51
|
-
export declare function formatStudyResults(study: Record<string, unknown>, json: boolean): void;
|
|
50
|
+
export declare function formatStudyDetail(study: Record<string, unknown>, json: boolean, options?: OutputOptions, participants?: ReadonlyArray<Record<string, unknown>>): void;
|
|
51
|
+
export declare function formatStudyResults(study: Record<string, unknown>, participants: ReadonlyArray<Record<string, unknown>>, json: boolean): void;
|
|
52
52
|
/**
|
|
53
|
-
* `study results --summary` projection. Drops interview_answers + per-
|
|
53
|
+
* `study results --summary` projection. Drops interview_answers + per-participant
|
|
54
54
|
* interaction breakdowns; keeps headline counters, sentiment histogram, and a
|
|
55
|
-
* per-
|
|
55
|
+
* per-participant {alias, status, sentiment, comment} row. Useful for agents that
|
|
56
56
|
* need to branch on outcome without paying for the full envelope.
|
|
57
57
|
*/
|
|
58
|
-
export declare function buildStudyResultsSummary(study: Record<string, unknown
|
|
58
|
+
export declare function buildStudyResultsSummary(study: Record<string, unknown>, participants: ReadonlyArray<Record<string, unknown>>): Record<string, unknown>;
|
|
59
59
|
/**
|
|
60
|
-
* `study results --transcript <
|
|
60
|
+
* `study results --transcript <participant_id>` projection. Mirrors the schema
|
|
61
61
|
* MCP's `get_chat_transcript` returns (`src/ish_mcp/projections.py:
|
|
62
62
|
* build_chat_transcript`) so callers see the same shape regardless of
|
|
63
|
-
* surface.
|
|
63
|
+
* surface. Participant turns whose action carries no text (e.g. select_option)
|
|
64
64
|
* surface `text: null`; intent lives on `action_type` + `option_label`.
|
|
65
65
|
* Bot turns with a `bot_reply.failure` block surface `failure` and
|
|
66
66
|
* `text: null` and don't count toward `unique_bot_replies`.
|
|
67
67
|
*/
|
|
68
|
-
export declare function buildChatTranscript(
|
|
68
|
+
export declare function buildChatTranscript(participant: Record<string, unknown>): Record<string, unknown>;
|
|
69
69
|
/**
|
|
70
|
-
* `study
|
|
70
|
+
* `study participant --summary` projection. Drops the action timeline; keeps the
|
|
71
71
|
* headline (alias, status, sentiment, comment, error_message). Useful for
|
|
72
|
-
* the common "did this
|
|
72
|
+
* the common "did this participant finish, what did they say" check that's
|
|
73
73
|
* currently buried under the full interactions array.
|
|
74
74
|
*/
|
|
75
|
-
export declare function
|
|
75
|
+
export declare function buildParticipantSummary(participant: Record<string, unknown>): Record<string, unknown>;
|
|
76
76
|
export declare function formatIterationList(iterations: Record<string, unknown>[], json: boolean): void;
|
|
77
|
-
export declare function
|
|
78
|
-
export declare function
|
|
79
|
-
export declare function
|
|
77
|
+
export declare function formatParticipantDetail(participant: Record<string, unknown>, json: boolean): void;
|
|
78
|
+
export declare function formatPersonList(profiles: unknown, json: boolean, limit?: number): void;
|
|
79
|
+
export declare function formatAttachment(attachment: Record<string, unknown>, json: boolean): void;
|
|
80
|
+
export declare const formatPersonSource: typeof formatAttachment;
|
|
80
81
|
export declare function formatGeneratedProfileList(profiles: unknown, json: boolean): void;
|
|
81
82
|
export declare function formatSimulationPoll(results: Record<string, unknown>[], json: boolean, isMedia?: boolean): void;
|
|
82
83
|
export declare function formatAskList(asks: Record<string, unknown>[], json: boolean): void;
|
|
@@ -90,7 +91,7 @@ export declare function formatRoundDetail(round: Record<string, unknown>, json:
|
|
|
90
91
|
* - medium: 3 <= n < 10 (small sample but clean)
|
|
91
92
|
* - high: n >= 10 AND no errored responses AND not tied
|
|
92
93
|
*
|
|
93
|
-
* Tuned for the typical 5-
|
|
94
|
+
* Tuned for the typical 5-participant ask: a clean 5/5 lands at "medium" (you
|
|
94
95
|
* can probably trust the lean), 1/5 with no errors lands at "low" (you
|
|
95
96
|
* need more data), 5/5 with a tie lands at "low" (no winner to call).
|
|
96
97
|
*/
|