@ishlabs/cli 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/chat.js +2 -2
- package/dist/commands/config.js +17 -3
- package/dist/commands/source.js +1 -1
- package/dist/commands/study-analyze.js +15 -2
- package/dist/commands/study-participant.js +19 -0
- package/dist/commands/study-run.d.ts +2 -0
- package/dist/commands/study-run.js +71 -20
- package/dist/commands/study.js +96 -34
- package/dist/lib/command-helpers.js +4 -3
- package/dist/lib/docs.js +114 -43
- package/dist/lib/output.d.ts +14 -9
- package/dist/lib/output.js +91 -19
- package/dist/lib/skill-content.js +10 -1
- package/dist/lib/study-participants.d.ts +3 -0
- package/dist/lib/study-results-filters.js +35 -14
- package/dist/lib/study-results-projections.d.ts +47 -17
- package/dist/lib/study-results-projections.js +39 -36
- package/dist/lib/types.d.ts +4 -0
- package/package.json +1 -1
|
@@ -79,10 +79,16 @@ function normaliseIterations(rawIterations) {
|
|
|
79
79
|
}
|
|
80
80
|
return out;
|
|
81
81
|
}
|
|
82
|
-
/** Throw a ValidationError-tagged Error (the wrapper maps it to exit 2).
|
|
83
|
-
|
|
82
|
+
/** Throw a ValidationError-tagged Error (the wrapper maps it to exit 2).
|
|
83
|
+
* Optional `availableValues` rides along as a structured field in the
|
|
84
|
+
* emitted JSON envelope so agents can parse the recovery set without
|
|
85
|
+
* string-extracting it from the message (Pattern V from round 3). */
|
|
86
|
+
function validationError(message, availableValues) {
|
|
84
87
|
const err = new Error(message);
|
|
85
88
|
err.name = "ValidationError";
|
|
89
|
+
if (availableValues && availableValues.length > 0) {
|
|
90
|
+
err.available_values = availableValues;
|
|
91
|
+
}
|
|
86
92
|
return err;
|
|
87
93
|
}
|
|
88
94
|
/** Resolve `--frame <ref>` into a set of frame_version_ids.
|
|
@@ -143,19 +149,21 @@ function resolveFrameRef(ref, frames) {
|
|
|
143
149
|
const needle = ref.toLowerCase();
|
|
144
150
|
const candidates = frames.filter((f) => f.name && f.name.toLowerCase().includes(needle));
|
|
145
151
|
if (candidates.length === 0) {
|
|
146
|
-
|
|
152
|
+
// Pattern V: dedupe + cap the frame list so duplicates (workspace-side
|
|
153
|
+
// data hazards) don't double-list in the error prose. Carry the
|
|
154
|
+
// deduped list as a structured `available_values` field for agents.
|
|
155
|
+
const dedupedNames = Array.from(new Set(frames
|
|
147
156
|
.map((f) => f.name)
|
|
148
|
-
.filter((n) => typeof n === "string")
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
? ` Available frames: ${available.join(", ")}.`
|
|
157
|
+
.filter((n) => typeof n === "string"))).slice(0, 10);
|
|
158
|
+
const hint = dedupedNames.length > 0
|
|
159
|
+
? ` Available frames: ${dedupedNames.join(", ")}.`
|
|
152
160
|
: " This study has no named frames yet.";
|
|
153
|
-
throw validationError(`--frame "${ref}" matched no frames on this study.${hint}
|
|
161
|
+
throw validationError(`--frame "${ref}" matched no frames on this study.${hint}`, dedupedNames);
|
|
154
162
|
}
|
|
155
163
|
if (candidates.length > 1) {
|
|
156
|
-
const names = candidates.map((c) => c.name).filter((n) => !!n);
|
|
164
|
+
const names = Array.from(new Set(candidates.map((c) => c.name).filter((n) => !!n)));
|
|
157
165
|
throw validationError(`--frame "${ref}" is ambiguous — matched ${candidates.length} frames: ${names.join(", ")}. ` +
|
|
158
|
-
`Use a more specific substring, a full Frame UUID, or an \`f-…\` alias
|
|
166
|
+
`Use a more specific substring, a full Frame UUID, or an \`f-…\` alias.`, names);
|
|
159
167
|
}
|
|
160
168
|
indexFrame(candidates[0]);
|
|
161
169
|
return { matchedFrameVersionIds: matched, frameVersionLookup: lookup };
|
|
@@ -192,10 +200,23 @@ function resolveAssignmentRef(ref, assignments) {
|
|
|
192
200
|
return out;
|
|
193
201
|
}
|
|
194
202
|
/** Resolve `--iteration <ref>` to a single iteration_id, or null if no
|
|
195
|
-
* match (caller errors). UUID
|
|
203
|
+
* match (caller errors). Accepts UUID, iteration alias (`i-…`), or label. */
|
|
196
204
|
function resolveIterationRef(ref, iterations) {
|
|
197
|
-
|
|
198
|
-
|
|
205
|
+
// Pattern M: iteration aliases (`i-…`) are the canonical short ID
|
|
206
|
+
// everywhere else in the CLI; accept them here too. Try alias resolution
|
|
207
|
+
// first, then fall through to UUID-direct, then label match.
|
|
208
|
+
let candidate = ref;
|
|
209
|
+
if (ref.startsWith("i-")) {
|
|
210
|
+
try {
|
|
211
|
+
candidate = resolveId(ref);
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
// Unknown alias — let the downstream "matched no iterations" branch
|
|
215
|
+
// emit the labels hint; the resolveId error doesn't add value here.
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
if (UUID_RE.test(candidate)) {
|
|
219
|
+
const m = iterations.find((i) => i.id === candidate);
|
|
199
220
|
if (!m) {
|
|
200
221
|
throw validationError(`No iteration matches "${ref}" on this study.`);
|
|
201
222
|
}
|
|
@@ -383,7 +404,7 @@ export function applyResultsFilters(study, participants, rawFrames, filters) {
|
|
|
383
404
|
warnings.push(`--turn is only meaningful on chat studies; ignored on modality "${modality}".`);
|
|
384
405
|
}
|
|
385
406
|
if (filters.side !== undefined && !isParticipantPair(study)) {
|
|
386
|
-
warnings.push(`--side is only meaningful on participant_pair chat studies; ignored.`);
|
|
407
|
+
warnings.push(`--side is only meaningful on participant_pair chat studies; ignored on modality "${modality}".`);
|
|
387
408
|
}
|
|
388
409
|
// Normalise sentiment labels to lowercase up-front (case-insensitive).
|
|
389
410
|
const sentimentFilter = filters.sentiment
|
|
@@ -3,14 +3,10 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Each `buildStudyResultsPer<Kind>` consumes a `FilteredResults` (the output
|
|
5
5
|
* of `applyResultsFilters` in `study-results-filters.ts`) and returns a
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* `
|
|
9
|
-
*
|
|
10
|
-
* Per-iteration is the only projection that wraps a `{study, slices, ...}`
|
|
11
|
-
* envelope; the others return plain arrays of slice objects. The surface
|
|
12
|
-
* attaches `totals_unfiltered` and `warnings` from the same `FilteredResults`
|
|
13
|
-
* for the array projections.
|
|
6
|
+
* bare array of slice objects. The surface (`commands/study.ts`) wraps the
|
|
7
|
+
* array uniformly in a `SliceResponse` envelope alongside `totals_unfiltered`,
|
|
8
|
+
* `modality_warnings`, `study_id`, and `modality` before handing it off to
|
|
9
|
+
* `formatStudyResultsGroupBy` for JSON or human rendering.
|
|
14
10
|
*
|
|
15
11
|
* Conventions mirror `buildStudyResultsEnvelope` (`output.ts:1081`) and
|
|
16
12
|
* `buildStudyResultsSummary` (`output.ts:1292`):
|
|
@@ -23,18 +19,52 @@
|
|
|
23
19
|
* Has no IO and no console side-effects.
|
|
24
20
|
*/
|
|
25
21
|
import type { FilteredResults } from "./study-results-filters.js";
|
|
22
|
+
import type { StudyResultsGroupByKind } from "./output.js";
|
|
26
23
|
export type { FilteredResults } from "./study-results-filters.js";
|
|
27
24
|
/**
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
25
|
+
* Uniform envelope emitted for every `ish study results --group-by <axis>`
|
|
26
|
+
* call, mirroring the MCP backend's `SliceResponse[T]`. Six top-level keys,
|
|
27
|
+
* stable across all six axes — `rows` carries the bare slice array returned
|
|
28
|
+
* by the matching `buildStudyResultsPer<Kind>` builder.
|
|
29
|
+
*/
|
|
30
|
+
export interface SliceResponse<T> {
|
|
31
|
+
axis: StudyResultsGroupByKind;
|
|
32
|
+
rows: T[];
|
|
33
|
+
totals_unfiltered: {
|
|
34
|
+
participant_count: number;
|
|
35
|
+
interaction_count: number;
|
|
36
|
+
};
|
|
37
|
+
modality_warnings: string[];
|
|
38
|
+
study_id: string;
|
|
39
|
+
modality: string;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Wrap a bare projection array in the uniform `SliceResponse` envelope.
|
|
43
|
+
* The surface calls this once after dispatching to one of the six
|
|
44
|
+
* `buildStudyResultsPer<Kind>` builders, then hands the envelope to
|
|
45
|
+
* `formatStudyResultsGroupBy`.
|
|
46
|
+
*/
|
|
47
|
+
export declare function wrapSliceProjection<T>(filtered: FilteredResults, axis: StudyResultsGroupByKind, rows: T[], studyId: string, modality: string): SliceResponse<T>;
|
|
48
|
+
interface IterationSlice {
|
|
49
|
+
iteration_id: string;
|
|
50
|
+
iteration_label: string | null;
|
|
51
|
+
participant_count: number;
|
|
52
|
+
interaction_count: number;
|
|
53
|
+
sentiment: Record<string, number>;
|
|
54
|
+
sample_comments: string[];
|
|
55
|
+
top_actions: Array<{
|
|
56
|
+
action_type: string;
|
|
57
|
+
count: number;
|
|
58
|
+
}>;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* `--group-by iteration` — one slice per declared iteration, in the same
|
|
62
|
+
* order as `ish study get`. Iterations with zero surviving participants
|
|
63
|
+
* still appear with `participant_count: 0` so the consumer sees the full
|
|
64
|
+
* matrix at stable size. Returns a bare array; the surface wraps it in
|
|
65
|
+
* the uniform `SliceResponse` envelope.
|
|
36
66
|
*/
|
|
37
|
-
export declare function buildStudyResultsPerIteration(filtered: FilteredResults):
|
|
67
|
+
export declare function buildStudyResultsPerIteration(filtered: FilteredResults): IterationSlice[];
|
|
38
68
|
interface FrameSlice {
|
|
39
69
|
frame_id: string;
|
|
40
70
|
frame_label: string | null;
|
|
@@ -3,14 +3,10 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Each `buildStudyResultsPer<Kind>` consumes a `FilteredResults` (the output
|
|
5
5
|
* of `applyResultsFilters` in `study-results-filters.ts`) and returns a
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* `
|
|
9
|
-
*
|
|
10
|
-
* Per-iteration is the only projection that wraps a `{study, slices, ...}`
|
|
11
|
-
* envelope; the others return plain arrays of slice objects. The surface
|
|
12
|
-
* attaches `totals_unfiltered` and `warnings` from the same `FilteredResults`
|
|
13
|
-
* for the array projections.
|
|
6
|
+
* bare array of slice objects. The surface (`commands/study.ts`) wraps the
|
|
7
|
+
* array uniformly in a `SliceResponse` envelope alongside `totals_unfiltered`,
|
|
8
|
+
* `modality_warnings`, `study_id`, and `modality` before handing it off to
|
|
9
|
+
* `formatStudyResultsGroupBy` for JSON or human rendering.
|
|
14
10
|
*
|
|
15
11
|
* Conventions mirror `buildStudyResultsEnvelope` (`output.ts:1081`) and
|
|
16
12
|
* `buildStudyResultsSummary` (`output.ts:1292`):
|
|
@@ -23,6 +19,22 @@
|
|
|
23
19
|
* Has no IO and no console side-effects.
|
|
24
20
|
*/
|
|
25
21
|
import { deterministicAlias, ALIAS_PREFIX } from "./alias-store.js";
|
|
22
|
+
/**
|
|
23
|
+
* Wrap a bare projection array in the uniform `SliceResponse` envelope.
|
|
24
|
+
* The surface calls this once after dispatching to one of the six
|
|
25
|
+
* `buildStudyResultsPer<Kind>` builders, then hands the envelope to
|
|
26
|
+
* `formatStudyResultsGroupBy`.
|
|
27
|
+
*/
|
|
28
|
+
export function wrapSliceProjection(filtered, axis, rows, studyId, modality) {
|
|
29
|
+
return {
|
|
30
|
+
axis,
|
|
31
|
+
rows,
|
|
32
|
+
totals_unfiltered: filtered.totals_unfiltered,
|
|
33
|
+
modality_warnings: filtered.warnings,
|
|
34
|
+
study_id: deterministicAlias(ALIAS_PREFIX.study, studyId),
|
|
35
|
+
modality,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
26
38
|
const SAMPLE_COMMENT_CAP = 5;
|
|
27
39
|
const SAMPLE_COMMENT_MAX_LEN = 200;
|
|
28
40
|
const PARTICIPANT_ALIAS_CAP = 10;
|
|
@@ -44,21 +56,24 @@ function asString(v) {
|
|
|
44
56
|
function truncate(str, maxLen) {
|
|
45
57
|
if (str.length <= maxLen)
|
|
46
58
|
return str;
|
|
47
|
-
|
|
59
|
+
// Pattern H: trim at the last word boundary before the cap so quoted
|
|
60
|
+
// `sample_comments` / `sample_replies` don't end mid-word ("…koncentrera").
|
|
61
|
+
// Prefer the last sentence terminator if one exists in the kept range;
|
|
62
|
+
// otherwise fall back to the last whitespace. If neither is found
|
|
63
|
+
// (a single long unbroken token), hard-cut at the cap.
|
|
64
|
+
const head = str.slice(0, maxLen - 3);
|
|
65
|
+
const sentenceBreak = Math.max(head.lastIndexOf(". "), head.lastIndexOf("! "), head.lastIndexOf("? "));
|
|
66
|
+
if (sentenceBreak >= maxLen / 2)
|
|
67
|
+
return head.slice(0, sentenceBreak + 1) + "…";
|
|
68
|
+
const spaceBreak = head.lastIndexOf(" ");
|
|
69
|
+
if (spaceBreak >= maxLen / 2)
|
|
70
|
+
return head.slice(0, spaceBreak) + " …";
|
|
71
|
+
return head + "…";
|
|
48
72
|
}
|
|
49
73
|
function participantAlias(participant) {
|
|
50
74
|
const id = asString(participant.id);
|
|
51
75
|
return id ? deterministicAlias(ALIAS_PREFIX.participant, id) : null;
|
|
52
76
|
}
|
|
53
|
-
function studyHeader(filtered) {
|
|
54
|
-
const study = filtered.study;
|
|
55
|
-
const id = asString(study.id);
|
|
56
|
-
return {
|
|
57
|
-
alias: id ? deterministicAlias(ALIAS_PREFIX.study, id) : null,
|
|
58
|
-
name: asString(study.name) ?? null,
|
|
59
|
-
modality: asString(study.modality) ?? null,
|
|
60
|
-
};
|
|
61
|
-
}
|
|
62
77
|
function readSentimentLabel(interaction) {
|
|
63
78
|
const s = asRecord(interaction.sentiment);
|
|
64
79
|
return s ? asString(s.label) : null;
|
|
@@ -111,14 +126,11 @@ function collectParticipantAlias(bucket, seen, participant) {
|
|
|
111
126
|
bucket.push(alias);
|
|
112
127
|
}
|
|
113
128
|
/**
|
|
114
|
-
* `--group-by iteration` — one slice per iteration
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
* with `totals_unfiltered` + `warnings` because per-iteration is the
|
|
120
|
-
* "default agent slice" — the one most likely to be piped directly without
|
|
121
|
-
* the surface re-wrapping.
|
|
129
|
+
* `--group-by iteration` — one slice per declared iteration, in the same
|
|
130
|
+
* order as `ish study get`. Iterations with zero surviving participants
|
|
131
|
+
* still appear with `participant_count: 0` so the consumer sees the full
|
|
132
|
+
* matrix at stable size. Returns a bare array; the surface wraps it in
|
|
133
|
+
* the uniform `SliceResponse` envelope.
|
|
122
134
|
*/
|
|
123
135
|
export function buildStudyResultsPerIteration(filtered) {
|
|
124
136
|
const iterations = asArray(filtered.study.iterations);
|
|
@@ -176,16 +188,7 @@ export function buildStudyResultsPerIteration(filtered) {
|
|
|
176
188
|
.map(([action_type, count]) => ({ action_type, count }));
|
|
177
189
|
byIteration.get(iterId).top_actions = rows;
|
|
178
190
|
}
|
|
179
|
-
|
|
180
|
-
// matrix at stable size. Iterations with zero surviving rows still appear
|
|
181
|
-
// with `participant_count: 0` — useful for "matched X / Y" framing.
|
|
182
|
-
const slices = order.map((o) => byIteration.get(o.id));
|
|
183
|
-
return {
|
|
184
|
-
study: studyHeader(filtered),
|
|
185
|
-
slices,
|
|
186
|
-
totals_unfiltered: filtered.totals_unfiltered,
|
|
187
|
-
warnings: filtered.warnings,
|
|
188
|
-
};
|
|
191
|
+
return order.map((o) => byIteration.get(o.id));
|
|
189
192
|
}
|
|
190
193
|
/**
|
|
191
194
|
* `--group-by frame` — one slice per Frame that had a surviving interaction.
|
package/dist/lib/types.d.ts
CHANGED
|
@@ -357,6 +357,10 @@ export interface SimulationStatus {
|
|
|
357
357
|
create_time?: string;
|
|
358
358
|
completion_time?: string;
|
|
359
359
|
error?: string;
|
|
360
|
+
error_kind?: string | null;
|
|
361
|
+
started_at?: string | null;
|
|
362
|
+
last_heartbeat_at?: string | null;
|
|
363
|
+
age_seconds?: number | null;
|
|
360
364
|
}
|
|
361
365
|
export interface SimulationCancelResponse {
|
|
362
366
|
job_id: string;
|