@ishlabs/cli 0.14.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/commands/profile.js +237 -1
- package/dist/commands/study-run.js +269 -32
- package/dist/commands/workspace.js +3 -2
- package/dist/lib/api-client.d.ts +7 -0
- package/dist/lib/api-client.js +9 -0
- package/dist/lib/command-helpers.js +1 -1
- package/dist/lib/docs.js +320 -1
- package/dist/lib/enums.d.ts +8 -0
- package/dist/lib/enums.js +12 -0
- package/dist/lib/skill-content.js +112 -7
- package/dist/lib/study-events.d.ts +46 -0
- package/dist/lib/study-events.js +126 -0
- package/dist/lib/types.d.ts +56 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -308,6 +308,16 @@ ish ask run --new --name "newsletter" \
|
|
|
308
308
|
|
|
309
309
|
**Audience flags** (`--profile`, `--sample`, `--all-simulatable`, `--country`, `--gender`, `--min-age`, `--max-age`, `--search`, `--visibility`, `--name`, `--description`, `--workspace`) only apply with `--new` — the audience is fixed at ask creation.
|
|
310
310
|
|
|
311
|
+
**`--visibility` values** (same set everywhere it's accepted):
|
|
312
|
+
|
|
313
|
+
| Value | Selects |
|
|
314
|
+
|---|---|
|
|
315
|
+
| `workspace` | Profiles owned by your workspace (default scope). |
|
|
316
|
+
| `shared` | Community-published profiles visible across workspaces. |
|
|
317
|
+
| `platform` | Admin-curated profiles from the platform pool. |
|
|
318
|
+
|
|
319
|
+
Old values `private` (now `workspace`) and `public` (now `platform`) keep working until the next release; the server logs a deprecation warning and maps them to the new vocabulary.
|
|
320
|
+
|
|
311
321
|
**Other ask verbs:**
|
|
312
322
|
|
|
313
323
|
```bash
|
package/dist/commands/profile.js
CHANGED
|
@@ -7,7 +7,7 @@ import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
|
7
7
|
import { formatTesterProfileList, formatGeneratedProfileList, output, } from "../lib/output.js";
|
|
8
8
|
import { resolveTextContent } from "../lib/upload.js";
|
|
9
9
|
import { isUuid, resolveSourceRef } from "../lib/profile-sources.js";
|
|
10
|
-
import { assertEnumValue, EDUCATION_LEVELS, HOUSEHOLDS, LOCALE_TYPES, INCOME_LEVELS, EMPLOYMENT_STATUSES, } from "../lib/enums.js";
|
|
10
|
+
import { assertEnumValue, EDUCATION_LEVELS, EVIDENCE_SOURCES, HOUSEHOLDS, LOCALE_TYPES, INCOME_LEVELS, EMPLOYMENT_STATUSES, } from "../lib/enums.js";
|
|
11
11
|
import { validateAccessibilityProfile } from "../lib/accessibility-profile.js";
|
|
12
12
|
function collect(value, prev) {
|
|
13
13
|
return prev.concat(value);
|
|
@@ -393,6 +393,242 @@ Schema: https://ishlabs.io/spec/accessibility-profile-schema.v1.json`)
|
|
|
393
393
|
output({ id: rid, alias: tagAlias(ALIAS_PREFIX.testerProfile, rid), message: "Profile deleted" }, globals.json, { writePath: true });
|
|
394
394
|
});
|
|
395
395
|
});
|
|
396
|
+
profile
|
|
397
|
+
.command("suggest-scenarios")
|
|
398
|
+
.description("Ask the LLM for scenario probes to craft a specific simulated tester")
|
|
399
|
+
.option("--workspace <id>", "Workspace (product) ID; falls back to active workspace")
|
|
400
|
+
.option("--context <text>", "What you already know about this tester. Use @path to read from file.")
|
|
401
|
+
.option("--context-file <path>", "Read --context from a file")
|
|
402
|
+
.option("--count <n>", "Number of scenarios to return (1-10, default 5)")
|
|
403
|
+
.option("--previous-answers <json-or-@path>", "Answers already collected this session. Inline JSON, @/path/to.json, or - for stdin. Array of {type, prompt, answer}; max 40.")
|
|
404
|
+
.option("--already-surfaced <json-or-@path>", "Prompt labels already shown this session, so the LLM avoids paraphrasing them. Inline JSON, @/path, or -. Array of strings; max 40.")
|
|
405
|
+
.addHelpText("after", `
|
|
406
|
+
Examples:
|
|
407
|
+
# Bare invocation: 5 scenarios from a free-form context blob
|
|
408
|
+
$ ish profile suggest-scenarios --context "Mid-career engineer who handles oncall for a Stripe-using fintech"
|
|
409
|
+
|
|
410
|
+
# Load context from a file, ask for 3 scenarios
|
|
411
|
+
$ ish profile suggest-scenarios --context-file ./persona-notes.md --count 3
|
|
412
|
+
|
|
413
|
+
# Follow-up probe: skip prompts already shown, build on prior answers
|
|
414
|
+
$ ish profile suggest-scenarios \\
|
|
415
|
+
--context "$(cat notes.md)" \\
|
|
416
|
+
--count 3 \\
|
|
417
|
+
--already-surfaced '["How do you triage 02:00 pages?"]' \\
|
|
418
|
+
--previous-answers @./answers.json
|
|
419
|
+
|
|
420
|
+
# Capture just the first scenario's type
|
|
421
|
+
$ ish profile suggest-scenarios --context "..." --count 1 --get scenarios[0].type
|
|
422
|
+
|
|
423
|
+
The loop: suggest → answer locally → persist via \`ish profile evidence add <id>\` (read back with \`evidence list\`).
|
|
424
|
+
See \`ish docs get-page guides/build-specific-tester\` for the full workflow.`)
|
|
425
|
+
.action(async (opts, cmd) => {
|
|
426
|
+
await withClient(cmd, async (client, globals) => {
|
|
427
|
+
const productId = resolveWorkspace(opts.workspace);
|
|
428
|
+
let context;
|
|
429
|
+
if (opts.context)
|
|
430
|
+
context = resolveTextContent(opts.context);
|
|
431
|
+
if (opts.contextFile)
|
|
432
|
+
context = resolveTextContent(`@${opts.contextFile}`);
|
|
433
|
+
if (!context) {
|
|
434
|
+
throw new Error("Provide --context (text or @path) or --context-file.");
|
|
435
|
+
}
|
|
436
|
+
const trimmed = context.trim();
|
|
437
|
+
if (trimmed.length === 0) {
|
|
438
|
+
throw new Error("--context cannot be empty.");
|
|
439
|
+
}
|
|
440
|
+
if (trimmed.length > 20_000) {
|
|
441
|
+
throw new Error(`--context is ${trimmed.length} chars; backend max is 20000. Trim before retrying.`);
|
|
442
|
+
}
|
|
443
|
+
const body = {
|
|
444
|
+
product_id: productId,
|
|
445
|
+
context: trimmed,
|
|
446
|
+
};
|
|
447
|
+
if (opts.count !== undefined) {
|
|
448
|
+
const n = parseInt(opts.count, 10);
|
|
449
|
+
if (Number.isNaN(n) || n < 1 || n > 10) {
|
|
450
|
+
throw new Error("--count must be an integer between 1 and 10.");
|
|
451
|
+
}
|
|
452
|
+
body.count = n;
|
|
453
|
+
}
|
|
454
|
+
if (opts.previousAnswers) {
|
|
455
|
+
const parsed = await parseJsonFlag(opts.previousAnswers, "--previous-answers");
|
|
456
|
+
if (!Array.isArray(parsed)) {
|
|
457
|
+
throw new Error("--previous-answers must be a JSON array of {type, prompt, answer}.");
|
|
458
|
+
}
|
|
459
|
+
if (parsed.length > 40) {
|
|
460
|
+
throw new Error(`--previous-answers max 40 entries (got ${parsed.length}).`);
|
|
461
|
+
}
|
|
462
|
+
for (let i = 0; i < parsed.length; i++) {
|
|
463
|
+
const row = parsed[i];
|
|
464
|
+
if (!row || typeof row !== "object") {
|
|
465
|
+
throw new Error(`--previous-answers[${i}]: expected object {type, prompt, answer}.`);
|
|
466
|
+
}
|
|
467
|
+
if (typeof row.type !== "string" || typeof row.prompt !== "string" || typeof row.answer !== "string") {
|
|
468
|
+
throw new Error(`--previous-answers[${i}]: missing or non-string {type, prompt, answer}.`);
|
|
469
|
+
}
|
|
470
|
+
assertEnumValue(row.type, EVIDENCE_SOURCES, `--previous-answers[${i}].type`);
|
|
471
|
+
}
|
|
472
|
+
body.previous_answers = parsed;
|
|
473
|
+
}
|
|
474
|
+
if (opts.alreadySurfaced) {
|
|
475
|
+
const parsed = await parseJsonFlag(opts.alreadySurfaced, "--already-surfaced");
|
|
476
|
+
if (!Array.isArray(parsed) || parsed.some((s) => typeof s !== "string")) {
|
|
477
|
+
throw new Error("--already-surfaced must be a JSON array of strings.");
|
|
478
|
+
}
|
|
479
|
+
if (parsed.length > 40) {
|
|
480
|
+
throw new Error(`--already-surfaced max 40 entries (got ${parsed.length}).`);
|
|
481
|
+
}
|
|
482
|
+
body.already_surfaced_prompts = parsed;
|
|
483
|
+
}
|
|
484
|
+
if (!globals.quiet) {
|
|
485
|
+
const target = body.count ?? 5;
|
|
486
|
+
console.error(` suggesting ${target} scenario${target === 1 ? "" : "s"}...`);
|
|
487
|
+
}
|
|
488
|
+
const data = await client.post("/tester-profiles/suggest-scenarios", body, { timeout: 180_000 });
|
|
489
|
+
output(data, globals.json);
|
|
490
|
+
});
|
|
491
|
+
});
|
|
492
|
+
const evidence = profile
|
|
493
|
+
.command("evidence")
|
|
494
|
+
.description("Manage scenario-answer evidence on a tester profile")
|
|
495
|
+
.addHelpText("after", `
|
|
496
|
+
Evidence rows persist answers to \`suggest-scenarios\` probes onto a
|
|
497
|
+
specific profile. The \`source\` field on every trace is the same enum
|
|
498
|
+
as the \`type\` field on a suggested scenario — copy verbatim when
|
|
499
|
+
building a traces.json.
|
|
500
|
+
|
|
501
|
+
Guide: ish docs get-page guides/build-specific-tester`);
|
|
502
|
+
evidence
|
|
503
|
+
.command("add")
|
|
504
|
+
.description("Persist scenario answers as structured evidence on a profile")
|
|
505
|
+
.argument("<id>", "Profile ID (alias or UUID)")
|
|
506
|
+
.option("--traces <json-or-@path>", `Array of {text, source, scenario_prompt?, raw_response?} where source ∈ ${EVIDENCE_SOURCES.join("|")}. Inline JSON, @/path/to.json, or - for stdin.`)
|
|
507
|
+
.option("--traces-file <path>", "Read --traces from a JSON file")
|
|
508
|
+
.addHelpText("after", `
|
|
509
|
+
Examples:
|
|
510
|
+
# Inline JSON for a single trace
|
|
511
|
+
$ ish profile evidence add tp-d4e --traces '[{"text":"I would page my staff engineer first.","source":"situation","scenario_prompt":"PagerDuty fires at 02:00."}]'
|
|
512
|
+
|
|
513
|
+
# From a file
|
|
514
|
+
$ ish profile evidence add tp-d4e --traces-file ./answers.json
|
|
515
|
+
|
|
516
|
+
# From stdin (pipe-friendly)
|
|
517
|
+
$ jq -c '.traces' session.json | ish profile evidence add tp-d4e --traces -
|
|
518
|
+
|
|
519
|
+
# Project the response
|
|
520
|
+
$ ish profile evidence add tp-d4e --traces-file ./answers.json --fields id,source,created_at
|
|
521
|
+
|
|
522
|
+
Valid source values: ${EVIDENCE_SOURCES.join(", ")}.
|
|
523
|
+
\`source\` on a trace = \`type\` on a suggested scenario — same enum.
|
|
524
|
+
Pair with \`ish profile suggest-scenarios\` to drive the iterative probe → answer loop.
|
|
525
|
+
Verify with \`ish profile evidence list <id>\`.`)
|
|
526
|
+
.action(async (id, opts, cmd) => {
|
|
527
|
+
await withClient(cmd, async (client, globals) => {
|
|
528
|
+
if (opts.traces && opts.tracesFile) {
|
|
529
|
+
throw new Error("Pass either --traces or --traces-file, not both.");
|
|
530
|
+
}
|
|
531
|
+
if (!opts.traces && !opts.tracesFile) {
|
|
532
|
+
throw new Error("Provide --traces (JSON, @path, or -) or --traces-file <path>.");
|
|
533
|
+
}
|
|
534
|
+
const parsed = opts.tracesFile
|
|
535
|
+
? await readJsonFileOrStdin(opts.tracesFile)
|
|
536
|
+
: await parseJsonFlag(opts.traces, "--traces");
|
|
537
|
+
if (!Array.isArray(parsed) || parsed.length === 0) {
|
|
538
|
+
throw new Error("traces must be a non-empty JSON array.");
|
|
539
|
+
}
|
|
540
|
+
const traces = [];
|
|
541
|
+
for (let i = 0; i < parsed.length; i++) {
|
|
542
|
+
const row = parsed[i];
|
|
543
|
+
if (!row || typeof row !== "object") {
|
|
544
|
+
throw new Error(`traces[${i}]: expected object {text, source, ...}.`);
|
|
545
|
+
}
|
|
546
|
+
if (typeof row.text !== "string" || row.text.length === 0) {
|
|
547
|
+
throw new Error(`traces[${i}].text: required non-empty string.`);
|
|
548
|
+
}
|
|
549
|
+
if (typeof row.source !== "string") {
|
|
550
|
+
throw new Error(`traces[${i}].source: required string.`);
|
|
551
|
+
}
|
|
552
|
+
assertEnumValue(row.source, EVIDENCE_SOURCES, `traces[${i}].source`);
|
|
553
|
+
if (row.scenario_prompt !== undefined && typeof row.scenario_prompt !== "string") {
|
|
554
|
+
throw new Error(`traces[${i}].scenario_prompt: must be a string if provided.`);
|
|
555
|
+
}
|
|
556
|
+
traces.push({
|
|
557
|
+
text: row.text,
|
|
558
|
+
source: row.source,
|
|
559
|
+
scenario_prompt: row.scenario_prompt ?? "",
|
|
560
|
+
...(row.raw_response !== undefined ? { raw_response: row.raw_response } : {}),
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
const rid = resolveId(id);
|
|
564
|
+
const body = { traces };
|
|
565
|
+
if (!globals.quiet) {
|
|
566
|
+
console.error(` persisting ${traces.length} evidence trace${traces.length === 1 ? "" : "s"} on ${tagAlias(ALIAS_PREFIX.testerProfile, rid)}...`);
|
|
567
|
+
}
|
|
568
|
+
const data = await client.post(`/tester-profiles/${rid}/scenarios`, body);
|
|
569
|
+
if (globals.json) {
|
|
570
|
+
output({ items: data, total: data.length }, true);
|
|
571
|
+
}
|
|
572
|
+
else {
|
|
573
|
+
output(data, false);
|
|
574
|
+
}
|
|
575
|
+
});
|
|
576
|
+
});
|
|
577
|
+
evidence
|
|
578
|
+
.command("list")
|
|
579
|
+
.description("List evidence traces persisted on a tester profile (newest first)")
|
|
580
|
+
.argument("<id>", "Profile ID (alias or UUID)")
|
|
581
|
+
.addHelpText("after", `
|
|
582
|
+
Examples:
|
|
583
|
+
# Read back every trace on a profile
|
|
584
|
+
$ ish profile evidence list tp-d4e
|
|
585
|
+
|
|
586
|
+
# Project per-row fields
|
|
587
|
+
$ ish profile evidence list tp-d4e --fields id,source,scenario_prompt
|
|
588
|
+
|
|
589
|
+
# Capture just the sources, one per line (auto-descends into items)
|
|
590
|
+
$ ish profile evidence list tp-d4e --get source
|
|
591
|
+
|
|
592
|
+
Returns a {items, total} envelope. Use the same id you passed to \`evidence add\`.`)
|
|
593
|
+
.action(async (id, _opts, cmd) => {
|
|
594
|
+
await withClient(cmd, async (client, globals) => {
|
|
595
|
+
const rid = resolveId(id);
|
|
596
|
+
const data = await client.get(`/tester-profiles/${rid}/scenarios`);
|
|
597
|
+
if (globals.json) {
|
|
598
|
+
output({ items: data, total: data.length }, true);
|
|
599
|
+
}
|
|
600
|
+
else {
|
|
601
|
+
output(data, false);
|
|
602
|
+
}
|
|
603
|
+
});
|
|
604
|
+
});
|
|
605
|
+
}
|
|
606
|
+
/**
|
|
607
|
+
* Parse a flag that accepts inline JSON, an @path reference, or `-` for stdin.
|
|
608
|
+
* Mirrors the @path convention on --description (resolveTextContent) and the
|
|
609
|
+
* stdin convention on --file. Used by --traces, --previous-answers, etc.
|
|
610
|
+
*/
|
|
611
|
+
async function parseJsonFlag(value, flagName) {
|
|
612
|
+
if (value === "-") {
|
|
613
|
+
return readJsonFileOrStdin();
|
|
614
|
+
}
|
|
615
|
+
if (value.startsWith("@")) {
|
|
616
|
+
const path = value.slice(1);
|
|
617
|
+
if (!path) {
|
|
618
|
+
throw new Error(`Missing file path after @ in ${flagName}. Usage: ${flagName} @/path/to/file.json`);
|
|
619
|
+
}
|
|
620
|
+
return readJsonFileOrStdin(path);
|
|
621
|
+
}
|
|
622
|
+
const trimmed = value.trim();
|
|
623
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
624
|
+
try {
|
|
625
|
+
return JSON.parse(trimmed);
|
|
626
|
+
}
|
|
627
|
+
catch (e) {
|
|
628
|
+
throw new Error(`Invalid JSON in ${flagName}: ${e.message}`);
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
throw new Error(`${flagName} expects inline JSON (starting with { or [), an @path reference (@/path/to/file.json), or '-' for stdin.`);
|
|
396
632
|
}
|
|
397
633
|
/**
|
|
398
634
|
* If the value matches the testerProfileSource alias pattern (e.g. "tps-3a4"),
|
|
@@ -8,9 +8,10 @@
|
|
|
8
8
|
* Lower-level: `study poll`, `study cancel`.
|
|
9
9
|
*/
|
|
10
10
|
import * as readline from "node:readline/promises";
|
|
11
|
-
import { withClient, getWebUrl, terminalLink, resolveWorkspace, resolveStudy, parseWaitTimeout, resolveAudienceProfileIds, addAudienceFilterFlags, hasAudienceFlags, } from "../lib/command-helpers.js";
|
|
11
|
+
import { withClient, getWebUrl, terminalLink, resolveWorkspace, resolveStudy, parseWaitTimeout, resolveAudienceProfileIds, addAudienceFilterFlags, hasAudienceFlags, readFileOrStdin, } from "../lib/command-helpers.js";
|
|
12
12
|
import { resolveId, tagAlias, ALIAS_PREFIX } from "../lib/alias-store.js";
|
|
13
13
|
import { output, formatSimulationPoll } from "../lib/output.js";
|
|
14
|
+
import { streamStudyEvents } from "../lib/study-events.js";
|
|
14
15
|
import { isMediaModality, isChatModality, iterationHasContent, describeRequiredContentFlag, readChatMode, readTesterPairConfig, summarizeRoleCriteria, } from "../lib/modality.js";
|
|
15
16
|
import { runLocalSimulations } from "../lib/local-sim/loop.js";
|
|
16
17
|
import { ensureBrowser } from "../lib/local-sim/install.js";
|
|
@@ -93,6 +94,11 @@ function dedupeSimulations(simResults) {
|
|
|
93
94
|
];
|
|
94
95
|
}
|
|
95
96
|
const POLL_INTERVAL_MS = 5_000;
|
|
97
|
+
// When the backend SSE stream is connected we drop to a slow backstop —
|
|
98
|
+
// events wake us up promptly, so re-fetching every 5s is wasteful. If the
|
|
99
|
+
// stream dies (older backend, broker offline, token mint failure) the loop
|
|
100
|
+
// transparently reverts to POLL_INTERVAL_MS.
|
|
101
|
+
const SSE_BACKSTOP_INTERVAL_MS = 30_000;
|
|
96
102
|
const TERMINAL_STATUSES = new Set(["completed", "errored", "failed", "cancelled", "canceled"]);
|
|
97
103
|
function flattenTesterStatuses(iterations, only) {
|
|
98
104
|
const rows = [];
|
|
@@ -118,38 +124,90 @@ function flattenTesterStatuses(iterations, only) {
|
|
|
118
124
|
async function pollStudyUntilDone(client, opts) {
|
|
119
125
|
const start = Date.now();
|
|
120
126
|
let lastReported = "";
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
}
|
|
127
|
+
// Open the SSE event stream as a wake source. When the backend supports
|
|
128
|
+
// it (enable_realtime=true + broker live), tester status changes wake
|
|
129
|
+
// the loop the moment they happen — no need to wait for the next poll
|
|
130
|
+
// tick. When it doesn't, the iterator exits silently on the first read
|
|
131
|
+
// and we fall through to pure polling at POLL_INTERVAL_MS.
|
|
132
|
+
const ac = new AbortController();
|
|
133
|
+
const eventIter = streamStudyEvents(client, opts.studyId, ac.signal);
|
|
134
|
+
// Optimistic: we assume the stream will deliver until the first read
|
|
135
|
+
// confirms otherwise. The first `await pendingEvent` settles within
|
|
136
|
+
// milliseconds for an unavailable stream (token mint 503 / endpoint 503)
|
|
137
|
+
// and is essentially a no-op for the latency budget.
|
|
138
|
+
let sseAlive = true;
|
|
139
|
+
let pendingEvent = eventIter.next();
|
|
140
|
+
try {
|
|
141
|
+
while (true) {
|
|
142
|
+
const study = await client.get(`/studies/${opts.studyId}`, undefined, { timeout: 60_000 });
|
|
143
|
+
const isMedia = isMediaModality(study.modality);
|
|
144
|
+
let iterations = study.iterations;
|
|
145
|
+
if (opts.iterationId) {
|
|
146
|
+
iterations = (iterations ?? []).filter((it) => it.id === opts.iterationId);
|
|
147
|
+
}
|
|
148
|
+
const rows = flattenTesterStatuses(iterations, opts.testerIds);
|
|
149
|
+
const total = rows.length;
|
|
150
|
+
const done = rows.filter((r) => TERMINAL_STATUSES.has(r.status)).length;
|
|
151
|
+
const errored = rows.filter((r) => r.status === "errored" || r.status === "failed").length;
|
|
152
|
+
const summary = `${done}/${total} done${errored > 0 ? `, ${errored} errored/failed` : ""}`;
|
|
153
|
+
if (!opts.quiet && summary !== lastReported) {
|
|
154
|
+
process.stderr.write(` ${summary}\n`);
|
|
155
|
+
lastReported = summary;
|
|
156
|
+
}
|
|
157
|
+
if (total > 0 && done === total) {
|
|
158
|
+
return { rows, isMedia };
|
|
159
|
+
}
|
|
160
|
+
if (Date.now() - start > opts.timeoutMs) {
|
|
161
|
+
throw new WaitTimeoutError(`Timed out after ${Math.round(opts.timeoutMs / 1000)}s waiting for simulations. ` +
|
|
162
|
+
`${done}/${total} done. Run \`ish study poll --study ${opts.studyId}\` to check status.`, {
|
|
163
|
+
study_id: opts.studyId,
|
|
164
|
+
...(opts.iterationId && { iteration_id: opts.iterationId }),
|
|
165
|
+
timeout_seconds: Math.round(opts.timeoutMs / 1000),
|
|
166
|
+
done,
|
|
167
|
+
total,
|
|
168
|
+
pending: total - done,
|
|
169
|
+
rows,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
// Wait for either the next backend event or the next tick. Use the
|
|
173
|
+
// backstop interval while the stream is delivering (events are the
|
|
174
|
+
// primary wake source); fall back to POLL_INTERVAL_MS once the
|
|
175
|
+
// stream is dead (older backend or broker dropped mid-run).
|
|
176
|
+
const interval = sseAlive ? SSE_BACKSTOP_INTERVAL_MS : POLL_INTERVAL_MS;
|
|
177
|
+
if (sseAlive && pendingEvent !== null) {
|
|
178
|
+
let timerHandle;
|
|
179
|
+
const timer = new Promise((resolve) => {
|
|
180
|
+
timerHandle = setTimeout(() => resolve({ kind: "timer" }), interval);
|
|
181
|
+
});
|
|
182
|
+
const eventOrEnd = pendingEvent.then((r) => ({ kind: "event", result: r }), () => ({ kind: "event", result: { value: undefined, done: true } }));
|
|
183
|
+
const winner = await Promise.race([eventOrEnd, timer]);
|
|
184
|
+
if (timerHandle !== undefined)
|
|
185
|
+
clearTimeout(timerHandle);
|
|
186
|
+
if (winner.kind === "event") {
|
|
187
|
+
if (winner.result.done) {
|
|
188
|
+
// Stream ended; revert to pure polling for the rest of the run.
|
|
189
|
+
sseAlive = false;
|
|
190
|
+
pendingEvent = null;
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
// Re-arm for the next event. We deliberately don't act on
|
|
194
|
+
// the event payload here — the next loop iteration re-fetches
|
|
195
|
+
// status and that's the truth source.
|
|
196
|
+
pendingEvent = eventIter.next();
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
// Timer winning doesn't replace pendingEvent — it stays parked
|
|
200
|
+
// and resolves on whichever event arrives next.
|
|
201
|
+
}
|
|
202
|
+
else {
|
|
203
|
+
await new Promise((resolve) => setTimeout(resolve, interval));
|
|
204
|
+
}
|
|
151
205
|
}
|
|
152
|
-
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
// Stop the SSE fetch + reader so we don't leave a dangling HTTP
|
|
209
|
+
// connection to the backend after returning / throwing.
|
|
210
|
+
ac.abort();
|
|
153
211
|
}
|
|
154
212
|
}
|
|
155
213
|
function readIterationDetails(details) {
|
|
@@ -1065,4 +1123,183 @@ Examples:
|
|
|
1065
1123
|
output(data, globals.json);
|
|
1066
1124
|
});
|
|
1067
1125
|
});
|
|
1126
|
+
// --- Extend ---
|
|
1127
|
+
//
|
|
1128
|
+
// Resume a terminal tester with `additional_steps` more turns — the
|
|
1129
|
+
// "start" half of the cancel + extend pair. The backend spawns a NEW
|
|
1130
|
+
// tester under the same iteration, branched from the source's last
|
|
1131
|
+
// interaction; the source row is left untouched. When --instruction is
|
|
1132
|
+
// set, the new tester treats it as overriding direction (the backend
|
|
1133
|
+
// surfaces it in a dedicated <user_added_instructions> block on every
|
|
1134
|
+
// prompt — see app-simulation Fix 1).
|
|
1135
|
+
study
|
|
1136
|
+
.command("extend")
|
|
1137
|
+
.description("Extend a terminal tester with more steps (and optionally a mid-run instruction)")
|
|
1138
|
+
.argument("<tester_id>", "Tester to extend (alias or UUID). Must be in a terminal state (completed/failed/cancelled).")
|
|
1139
|
+
.option("--add-steps <n>", "Extra interactions past the source's original cap (1-50; backend caps server-side)", "10")
|
|
1140
|
+
.option("--instruction <text>", "User message to inject as the new tester resumes. Accepts inline text, `@/path/to/file`, or `-` for stdin.")
|
|
1141
|
+
.option("--wait", "Block until the new tester reaches a terminal state")
|
|
1142
|
+
.option("--timeout <s>", "Wait timeout in seconds (default 300; only with --wait)")
|
|
1143
|
+
.option("--dispatch-timeout <s>", "Per-POST timeout in seconds for the dispatch call (default 120)")
|
|
1144
|
+
.option("--workspace <id>", "Workspace ID; accepted for consistency (workspace is inferred from <tester_id>)")
|
|
1145
|
+
.addHelpText("after", `
|
|
1146
|
+
The source tester is left untouched; a new tester is spawned under the
|
|
1147
|
+
same iteration and branched from the source's last interaction. Get the
|
|
1148
|
+
new tester ID from \`.tester_id\` / \`.tester_alias\` on the JSON output.
|
|
1149
|
+
|
|
1150
|
+
Examples:
|
|
1151
|
+
# Add 5 more steps to a completed run (no new instruction):
|
|
1152
|
+
$ ish study extend t-072 --add-steps 5
|
|
1153
|
+
|
|
1154
|
+
# Inject a mid-run instruction and wait for completion:
|
|
1155
|
+
$ ish study extend t-072 \\
|
|
1156
|
+
--instruction "Open the language selector and switch to German." \\
|
|
1157
|
+
--wait
|
|
1158
|
+
|
|
1159
|
+
# Long instruction from a file:
|
|
1160
|
+
$ ish study extend t-072 --instruction @/tmp/prompt.txt --wait --timeout 600
|
|
1161
|
+
|
|
1162
|
+
# Instruction from stdin (pipe-friendly):
|
|
1163
|
+
$ echo "Try the search bar instead." | ish study extend t-072 --instruction -
|
|
1164
|
+
|
|
1165
|
+
Get tester IDs from \`ish study run --json\` (.tester_aliases[] / .tester_ids[]).
|
|
1166
|
+
See \`ish docs get-page concepts/extending-a-simulation\` for the full mental model.`)
|
|
1167
|
+
.action(async (testerId, opts, cmd) => {
|
|
1168
|
+
await withClient(cmd, async (client, globals) => {
|
|
1169
|
+
// --add-steps: client-side parser fails fast before the network
|
|
1170
|
+
// call. Bound mirrors the backend's `le=50` cap; if the backend
|
|
1171
|
+
// moves the bound, this message will lag — the backend remains
|
|
1172
|
+
// authoritative and any 422 is surfaced verbatim.
|
|
1173
|
+
const addStepsRaw = opts.addSteps ?? "10";
|
|
1174
|
+
const addSteps = parseInt(addStepsRaw, 10);
|
|
1175
|
+
if (Number.isNaN(addSteps) || addSteps < 1 || addSteps > 50) {
|
|
1176
|
+
throw new Error(`--add-steps must be an integer between 1 and 50, got "${addStepsRaw}".`);
|
|
1177
|
+
}
|
|
1178
|
+
// --instruction: inline text | `@path` | `-` (stdin).
|
|
1179
|
+
let instruction;
|
|
1180
|
+
if (opts.instruction !== undefined) {
|
|
1181
|
+
let raw;
|
|
1182
|
+
if (opts.instruction === "-") {
|
|
1183
|
+
raw = await readFileOrStdin("-");
|
|
1184
|
+
}
|
|
1185
|
+
else if (opts.instruction.startsWith("@")) {
|
|
1186
|
+
raw = await readFileOrStdin(opts.instruction.slice(1));
|
|
1187
|
+
}
|
|
1188
|
+
else {
|
|
1189
|
+
raw = opts.instruction;
|
|
1190
|
+
}
|
|
1191
|
+
const trimmed = raw.trim();
|
|
1192
|
+
if (trimmed.length === 0) {
|
|
1193
|
+
throw new Error("--instruction must be non-empty (after trimming).");
|
|
1194
|
+
}
|
|
1195
|
+
instruction = trimmed;
|
|
1196
|
+
}
|
|
1197
|
+
const dispatchTimeoutMs = opts.dispatchTimeout
|
|
1198
|
+
? Math.max(1, parseInt(opts.dispatchTimeout, 10)) * 1000
|
|
1199
|
+
: 120_000;
|
|
1200
|
+
if (opts.dispatchTimeout &&
|
|
1201
|
+
(Number.isNaN(parseInt(opts.dispatchTimeout, 10)) ||
|
|
1202
|
+
parseInt(opts.dispatchTimeout, 10) < 1)) {
|
|
1203
|
+
throw new Error(`--dispatch-timeout must be a positive integer (seconds), got "${opts.dispatchTimeout}".`);
|
|
1204
|
+
}
|
|
1205
|
+
const sourceId = resolveId(testerId);
|
|
1206
|
+
const sourceAlias = tagAlias(ALIAS_PREFIX.tester, sourceId);
|
|
1207
|
+
if (!globals.quiet) {
|
|
1208
|
+
const stepNote = `${addSteps} step${addSteps === 1 ? "" : "s"}`;
|
|
1209
|
+
const instrNote = instruction ? " and a new instruction" : "";
|
|
1210
|
+
console.error(`Extending ${sourceAlias} with ${stepNote}${instrNote}...`);
|
|
1211
|
+
}
|
|
1212
|
+
const body = {
|
|
1213
|
+
source_tester_id: sourceId,
|
|
1214
|
+
additional_steps: addSteps,
|
|
1215
|
+
};
|
|
1216
|
+
if (instruction)
|
|
1217
|
+
body.user_message = instruction;
|
|
1218
|
+
const data = await client.post("/simulation/interactive/extend", body, { timeout: dispatchTimeoutMs });
|
|
1219
|
+
const newTesterId = String(data.tester_id);
|
|
1220
|
+
const newAlias = tagAlias(ALIAS_PREFIX.tester, newTesterId);
|
|
1221
|
+
// UUIDs preserved on the output — `study extend` is a write-path
|
|
1222
|
+
// dispatch command and the new `tester_id` is the load-bearing
|
|
1223
|
+
// return value (mirrors how `study run` keeps tester_ids in lean
|
|
1224
|
+
// output via the writePath option).
|
|
1225
|
+
const baseEnvelope = {
|
|
1226
|
+
tester_id: newTesterId,
|
|
1227
|
+
tester_alias: newAlias,
|
|
1228
|
+
source_tester_id: sourceId,
|
|
1229
|
+
source_alias: sourceAlias,
|
|
1230
|
+
study_id: data.study_id,
|
|
1231
|
+
job_id: data.job_id,
|
|
1232
|
+
additional_steps: addSteps,
|
|
1233
|
+
...(instruction && { instruction }),
|
|
1234
|
+
message: data.message,
|
|
1235
|
+
};
|
|
1236
|
+
if (!opts.wait) {
|
|
1237
|
+
if (globals.json) {
|
|
1238
|
+
output(baseEnvelope, true, { writePath: true });
|
|
1239
|
+
}
|
|
1240
|
+
else {
|
|
1241
|
+
console.error(` New tester: ${newAlias}`);
|
|
1242
|
+
if (data.message)
|
|
1243
|
+
console.error(` ${data.message}`);
|
|
1244
|
+
console.error(` Run \`ish study wait ${newAlias} --timeout 600\` to block until it finishes.`);
|
|
1245
|
+
}
|
|
1246
|
+
return;
|
|
1247
|
+
}
|
|
1248
|
+
// --wait: poll the new tester until it reaches a terminal state.
|
|
1249
|
+
// Mirrors the per-tester wait block in `study wait <tester_id>`
|
|
1250
|
+
// above — same WaitTimeoutError shape (exit 5, retryable) so the
|
|
1251
|
+
// failure envelope is consistent across commands.
|
|
1252
|
+
const timeoutMs = parseWaitTimeout(opts.timeout);
|
|
1253
|
+
if (!globals.quiet) {
|
|
1254
|
+
console.error(`Waiting for ${newAlias} to finish...`);
|
|
1255
|
+
}
|
|
1256
|
+
const start = Date.now();
|
|
1257
|
+
let lastStatus = "";
|
|
1258
|
+
while (true) {
|
|
1259
|
+
const status = await client.get(`/simulation/status/${newTesterId}`, undefined, { timeout: 60_000 });
|
|
1260
|
+
const s = String(status.status ?? "unknown");
|
|
1261
|
+
if (!globals.quiet && s !== lastStatus) {
|
|
1262
|
+
process.stderr.write(` ${s}\n`);
|
|
1263
|
+
lastStatus = s;
|
|
1264
|
+
}
|
|
1265
|
+
if (TERMINAL_STATUSES.has(s)) {
|
|
1266
|
+
const result = {
|
|
1267
|
+
status: s,
|
|
1268
|
+
...(typeof status.interaction_count === "number" && {
|
|
1269
|
+
interaction_count: status.interaction_count,
|
|
1270
|
+
}),
|
|
1271
|
+
...(status.tester_name && { tester_name: status.tester_name }),
|
|
1272
|
+
...(status.error && { error: status.error }),
|
|
1273
|
+
};
|
|
1274
|
+
if (globals.json) {
|
|
1275
|
+
output({ ...baseEnvelope, result }, true, { writePath: true });
|
|
1276
|
+
}
|
|
1277
|
+
else {
|
|
1278
|
+
console.error(` ${newAlias} finished: ${s}`);
|
|
1279
|
+
if (status.error)
|
|
1280
|
+
console.error(` error: ${status.error}`);
|
|
1281
|
+
}
|
|
1282
|
+
return;
|
|
1283
|
+
}
|
|
1284
|
+
if (Date.now() - start > timeoutMs) {
|
|
1285
|
+
throw new WaitTimeoutError(`Timed out after ${Math.round(timeoutMs / 1000)}s waiting for tester ${newAlias}. Last status: ${s}.`, {
|
|
1286
|
+
study_id: newTesterId,
|
|
1287
|
+
timeout_seconds: Math.round(timeoutMs / 1000),
|
|
1288
|
+
done: 0,
|
|
1289
|
+
total: 1,
|
|
1290
|
+
pending: 1,
|
|
1291
|
+
rows: [
|
|
1292
|
+
{
|
|
1293
|
+
id: newTesterId,
|
|
1294
|
+
status: s,
|
|
1295
|
+
tester_name: String(status.tester_name ?? "Unknown"),
|
|
1296
|
+
interaction_count: typeof status.interaction_count === "number" ? status.interaction_count : 0,
|
|
1297
|
+
},
|
|
1298
|
+
],
|
|
1299
|
+
});
|
|
1300
|
+
}
|
|
1301
|
+
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
|
1302
|
+
}
|
|
1303
|
+
});
|
|
1304
|
+
});
|
|
1068
1305
|
}
|
|
@@ -214,11 +214,12 @@ async function collectWorkspaceUsage(client, workspaceId) {
|
|
|
214
214
|
.get(`/products/${workspaceId}/studies`)
|
|
215
215
|
.catch(() => []);
|
|
216
216
|
// testers_used: paginated list returns { total, items, ... }. Backend
|
|
217
|
-
// gates `maxCustomTesterProfiles` on visibility=
|
|
217
|
+
// gates `maxCustomTesterProfiles` on visibility=workspace (rows owned by
|
|
218
|
+
// this workspace; was `private` before the visibility rename).
|
|
218
219
|
const testersPromise = client
|
|
219
220
|
.get("/tester-profiles", {
|
|
220
221
|
product_id: workspaceId,
|
|
221
|
-
visibility: "
|
|
222
|
+
visibility: "workspace",
|
|
222
223
|
type: "ai",
|
|
223
224
|
limit: "1",
|
|
224
225
|
offset: "0",
|
package/dist/lib/api-client.d.ts
CHANGED
|
@@ -20,6 +20,13 @@ export declare class ApiClient {
|
|
|
20
20
|
token: string;
|
|
21
21
|
});
|
|
22
22
|
get accessToken(): string;
|
|
23
|
+
/** Resolved base URL including the ``/api/v1`` prefix.
|
|
24
|
+
*
|
|
25
|
+
* Exposed so helpers that bypass the JSON-decoding ``fetch`` wrappers
|
|
26
|
+
* (e.g. the SSE stream in ``lib/study-events.ts``) can build their own
|
|
27
|
+
* requests against the same backend.
|
|
28
|
+
*/
|
|
29
|
+
get apiBase(): string;
|
|
23
30
|
private headers;
|
|
24
31
|
get<T = unknown>(path: string, params?: Record<string, string | string[]>, opts?: RequestOpts): Promise<T>;
|
|
25
32
|
post<T = unknown>(path: string, body?: unknown, opts?: RequestOpts): Promise<T>;
|
package/dist/lib/api-client.js
CHANGED
|
@@ -83,6 +83,15 @@ export class ApiClient {
|
|
|
83
83
|
get accessToken() {
|
|
84
84
|
return this.token;
|
|
85
85
|
}
|
|
86
|
+
/** Resolved base URL including the ``/api/v1`` prefix.
|
|
87
|
+
*
|
|
88
|
+
* Exposed so helpers that bypass the JSON-decoding ``fetch`` wrappers
|
|
89
|
+
* (e.g. the SSE stream in ``lib/study-events.ts``) can build their own
|
|
90
|
+
* requests against the same backend.
|
|
91
|
+
*/
|
|
92
|
+
get apiBase() {
|
|
93
|
+
return this.baseUrl;
|
|
94
|
+
}
|
|
86
95
|
headers() {
|
|
87
96
|
return {
|
|
88
97
|
Authorization: `Bearer ${this.token}`,
|
|
@@ -245,7 +245,7 @@ export function addAudienceFilterFlags(cmd, opts = {}) {
|
|
|
245
245
|
.option("--country <code>", "Filter by 2-letter country code (repeatable)", collectRepeatable, [])
|
|
246
246
|
.option("--min-age <n>", "Minimum age (inclusive)")
|
|
247
247
|
.option("--max-age <n>", "Maximum age (inclusive)")
|
|
248
|
-
.option("--visibility <v>", "Filter by visibility (private
|
|
248
|
+
.option("--visibility <v>", "Filter by visibility: workspace (your workspace), shared (community-published), platform (admin-curated). Old values `private` / `public` still accepted as aliases for `workspace` / `platform` until the next release; server logs a deprecation warning.");
|
|
249
249
|
}
|
|
250
250
|
export function getGlobals(cmd) {
|
|
251
251
|
let globals;
|