@ishlabs/cli 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/ask.js +52 -0
- package/dist/lib/docs.js +43 -0
- package/dist/lib/output.js +4 -1
- package/dist/lib/skill-content.js +49 -1
- package/dist/lib/types.d.ts +4 -0
- package/package.json +1 -1
package/dist/commands/ask.js
CHANGED
|
@@ -347,6 +347,7 @@ Examples:
|
|
|
347
347
|
allFlagName: "--all-simulatable",
|
|
348
348
|
allFlagDescription: "Use every simulatable AI profile matching the filters",
|
|
349
349
|
})
|
|
350
|
+
.option("--no-dispatch", "Create the ask in DRAFT status without billing or dispatching the round. Hand the draft id back to the user, then start it with `ish ask dispatch <id>`. Audience flags are still required because the testers are materialized at create time. Mutually exclusive with --wait (nothing to wait for).")
|
|
350
351
|
.option("--wait", "Wait until the first round completes (or errors)")
|
|
351
352
|
.option("--timeout <s>", "Wait timeout in seconds (default 300)")
|
|
352
353
|
.addHelpText("after", `
|
|
@@ -389,6 +390,9 @@ Picks come back with a \`pick_confidence\` (0..1) score per tester when
|
|
|
389
390
|
`)
|
|
390
391
|
.action(async (opts, cmd) => {
|
|
391
392
|
await withClient(cmd, async (client, globals) => {
|
|
393
|
+
if (opts.dispatch === false && opts.wait) {
|
|
394
|
+
throw new Error("--no-dispatch and --wait are incompatible — a draft ask has nothing to wait for. Drop --wait, or run `ish ask dispatch <id> --wait` after the draft is created.");
|
|
395
|
+
}
|
|
392
396
|
const wid = resolveWorkspace(opts.workspace);
|
|
393
397
|
const testerIds = await resolveAudienceProfileIds(client, wid, audienceFlags(opts), { requireSimulatable: true, allFlagName: "--all-simulatable" });
|
|
394
398
|
const round = await buildRoundInput(client, wid, opts, !!globals.quiet);
|
|
@@ -398,6 +402,7 @@ Picks come back with a \`pick_confidence\` (0..1) score per tester when
|
|
|
398
402
|
language: opts.language,
|
|
399
403
|
tester_profile_ids: testerIds,
|
|
400
404
|
first_round: round,
|
|
405
|
+
...(opts.dispatch === false && { dispatch: false }),
|
|
401
406
|
};
|
|
402
407
|
let data = await client.post(`/products/${wid}/asks`, body);
|
|
403
408
|
if (data.id) {
|
|
@@ -414,11 +419,58 @@ Picks come back with a \`pick_confidence\` (0..1) score per tester when
|
|
|
414
419
|
result.alias = tagAlias(ALIAS_PREFIX.ask, String(result.id));
|
|
415
420
|
formatAskDetail(result, globals.json);
|
|
416
421
|
if (!globals.json && data.id) {
|
|
422
|
+
if (opts.dispatch === false) {
|
|
423
|
+
const askAlias = tagAlias(ALIAS_PREFIX.ask, data.id);
|
|
424
|
+
process.stderr.write(`\n Draft created. Start it with: ish ask dispatch ${askAlias}\n`);
|
|
425
|
+
}
|
|
417
426
|
const url = getWebUrl(globals, `/${wid}/asks/${data.id}`);
|
|
418
427
|
process.stderr.write(`\n ${terminalLink(url, "Open in browser ↗")}\n\n`);
|
|
419
428
|
}
|
|
420
429
|
});
|
|
421
430
|
});
|
|
431
|
+
// ---- dispatch -----------------------------------------------------------
|
|
432
|
+
// Pattern B-dispatch: flip a DRAFT ask to RUNNING and enqueue the worker.
|
|
433
|
+
// Idempotent on the server (409 on non-DRAFT) — surface that as a usage
|
|
434
|
+
// error rather than a transient failure.
|
|
435
|
+
ask
|
|
436
|
+
.command("dispatch")
|
|
437
|
+
.description("Dispatch a draft ask — bills credits and starts the round")
|
|
438
|
+
.argument("[id]", "Ask alias or UUID (defaults to active ask)")
|
|
439
|
+
.option("--ask <id>", "Ask ID; alternative to positional argument")
|
|
440
|
+
.option("--wait", "Wait until the first round completes (or errors)")
|
|
441
|
+
.option("--timeout <s>", "Wait timeout in seconds (default 300)")
|
|
442
|
+
.addHelpText("after", `
|
|
443
|
+
Use after \`ish ask create --no-dispatch\` to start a draft once the user has
|
|
444
|
+
reviewed it. The dispatch is BILLABLE — credits are charged when responses
|
|
445
|
+
land, the same as a normal create.
|
|
446
|
+
|
|
447
|
+
Examples:
|
|
448
|
+
# Dispatch the active draft and wait for results:
|
|
449
|
+
$ ish ask dispatch --wait
|
|
450
|
+
|
|
451
|
+
# Dispatch a specific draft, JSON output:
|
|
452
|
+
$ ish ask dispatch a-6ec --json
|
|
453
|
+
|
|
454
|
+
A non-DRAFT ask returns a 409 (\`already dispatched\`). The CLI maps that to a
|
|
455
|
+
usage error so re-running this command is safe — no duplicate run.`)
|
|
456
|
+
.action(async (id, opts, cmd) => {
|
|
457
|
+
await withClient(cmd, async (client, globals) => {
|
|
458
|
+
const aid = resolveAsk(pickAskRef(id, opts.ask));
|
|
459
|
+
let data = await client.post(`/asks/${aid}/dispatch`, {});
|
|
460
|
+
if (opts.wait) {
|
|
461
|
+
const timeoutMs = parseWaitTimeout(opts.timeout);
|
|
462
|
+
data = await pollUntilRoundDone(client, aid, 0, timeoutMs, !!globals.quiet);
|
|
463
|
+
}
|
|
464
|
+
const result = data;
|
|
465
|
+
if (result.id)
|
|
466
|
+
result.alias = tagAlias(ALIAS_PREFIX.ask, String(result.id));
|
|
467
|
+
formatAskDetail(result, globals.json);
|
|
468
|
+
if (!globals.json && data.product_id) {
|
|
469
|
+
const url = getWebUrl(globals, `/${data.product_id}/asks/${aid}`);
|
|
470
|
+
process.stderr.write(`\n ${terminalLink(url, "Open in browser ↗")}\n\n`);
|
|
471
|
+
}
|
|
472
|
+
});
|
|
473
|
+
});
|
|
422
474
|
// ---- get ----------------------------------------------------------------
|
|
423
475
|
ask
|
|
424
476
|
.command("get")
|
package/dist/lib/docs.js
CHANGED
|
@@ -574,6 +574,49 @@ ish ask results a-6ec
|
|
|
574
574
|
ish ask results a-6ec --json | jq '.rounds[0].aggregates'
|
|
575
575
|
\`\`\`
|
|
576
576
|
|
|
577
|
+
## Status field
|
|
578
|
+
|
|
579
|
+
Asks carry a top-level \`status\`:
|
|
580
|
+
|
|
581
|
+
- \`draft\` — created but not dispatched yet. No credits charged. Created
|
|
582
|
+
by \`ish ask create --no-dispatch\`.
|
|
583
|
+
- \`running\` — dispatched; the round is executing or queued.
|
|
584
|
+
- \`completed\` — round 1 (or the most recent round) finished.
|
|
585
|
+
- \`cancelled\` — terminated explicitly.
|
|
586
|
+
|
|
587
|
+
Surfaces in \`ish ask list\` (table column) and \`ish ask get\` (header
|
|
588
|
+
metadata line and JSON \`status\` field). Lean JSON keeps the field
|
|
589
|
+
intact — no \`--verbose\` needed to see it.
|
|
590
|
+
|
|
591
|
+
## Stage-then-dispatch (draft asks)
|
|
592
|
+
|
|
593
|
+
When you want a human to review the audience and prompt **before** any
|
|
594
|
+
credits are spent, separate creation from dispatch:
|
|
595
|
+
|
|
596
|
+
\`\`\`
|
|
597
|
+
# 1. Stage — materializes testers, no worker enqueue, no bill yet
|
|
598
|
+
ish ask create --workspace w-6ec --name "tagline AB" \\
|
|
599
|
+
--prompt "Which sounds better?" \\
|
|
600
|
+
--variant text:"Short and punchy." \\
|
|
601
|
+
--variant text:"A longer, descriptive line." \\
|
|
602
|
+
--sample 30 --wants-pick \\
|
|
603
|
+
--no-dispatch
|
|
604
|
+
|
|
605
|
+
# Returns an ask with status="draft". Hand the alias back to the user.
|
|
606
|
+
|
|
607
|
+
# 2. Dispatch — flips DRAFT → RUNNING and enqueues the round (BILLABLE)
|
|
608
|
+
ish ask dispatch a-6ec --wait
|
|
609
|
+
\`\`\`
|
|
610
|
+
|
|
611
|
+
\`--no-dispatch\` requires audience flags (testers are still materialized
|
|
612
|
+
at create time — only the worker enqueue and billing are deferred). It
|
|
613
|
+
is incompatible with \`--wait\` since there is nothing to wait for.
|
|
614
|
+
|
|
615
|
+
\`ish ask dispatch\` is idempotent on the server: a non-DRAFT ask returns
|
|
616
|
+
HTTP 409 (\`already dispatched\`) which the CLI maps to a usage error, so
|
|
617
|
+
re-running the command is safe. The user who calls \`dispatch\` is the
|
|
618
|
+
billing principal — keep that in mind for shared workspaces.
|
|
619
|
+
|
|
577
620
|
## Reading the verdict
|
|
578
621
|
|
|
579
622
|
For \`--wants-pick\` / \`--wants-ratings\` rounds, \`ask results --json\`
|
package/dist/lib/output.js
CHANGED
|
@@ -1452,9 +1452,10 @@ export function formatAskList(asks, json) {
|
|
|
1452
1452
|
return;
|
|
1453
1453
|
}
|
|
1454
1454
|
const aliasMap = getAliasMap(ALIAS_PREFIX.ask);
|
|
1455
|
-
printTable(["#", "NAME", "AUDIENCE", "ROUNDS", "LAST ROUND", "ARCHIVED"], asks.map((a) => [
|
|
1455
|
+
printTable(["#", "NAME", "STATUS", "AUDIENCE", "ROUNDS", "LAST ROUND", "ARCHIVED"], asks.map((a) => [
|
|
1456
1456
|
aliasMap.get(String(a.id)) || String(a.id || ""),
|
|
1457
1457
|
String(a.name || ""),
|
|
1458
|
+
String(a.status || "-"),
|
|
1458
1459
|
String(a.audience_count ?? "0"),
|
|
1459
1460
|
String(a.round_count ?? "0"),
|
|
1460
1461
|
formatDate(a.last_round_at),
|
|
@@ -1533,6 +1534,8 @@ export function formatAskDetail(ask, json) {
|
|
|
1533
1534
|
if (ask.description)
|
|
1534
1535
|
console.log(String(ask.description));
|
|
1535
1536
|
const meta = [];
|
|
1537
|
+
if (ask.status)
|
|
1538
|
+
meta.push(String(ask.status));
|
|
1536
1539
|
if (ask.is_archived)
|
|
1537
1540
|
meta.push("archived");
|
|
1538
1541
|
meta.push(formatDate(ask.created_at));
|
|
@@ -138,6 +138,11 @@ ish study create --modality chat --endpoint my-bot --assignment "Sign up:Try to
|
|
|
138
138
|
ish study run --sample 5 --country SE --wait
|
|
139
139
|
ish ask run --new --name "..." --prompt "..." --variant text:"A" --variant text:"B" --sample 30 --wants-pick --wait
|
|
140
140
|
|
|
141
|
+
# Stage an ask for human review, then dispatch (no credits charged on stage)
|
|
142
|
+
ish ask create --name "..." --prompt "..." --variant text:"A" --variant text:"B" \
|
|
143
|
+
--sample 30 --wants-pick --no-dispatch
|
|
144
|
+
ish ask dispatch a-6ec --wait
|
|
145
|
+
|
|
141
146
|
# Results
|
|
142
147
|
ish study results
|
|
143
148
|
ish ask results a-6ec --round 1
|
|
@@ -236,6 +241,15 @@ implies \`--quiet\` so the bare value is the only thing on stdout.
|
|
|
236
241
|
follow-up question to a completed round preserves prior comments,
|
|
237
242
|
picks, and ratings; only the new question is dispatched. Pass
|
|
238
243
|
\`--redispatch-all\` for the legacy reset-and-rerun behavior.
|
|
244
|
+
- **\`ask create --no-dispatch\` stages a draft, no bill yet.** Pair
|
|
245
|
+
with \`ish ask dispatch <id>\` to flip DRAFT → RUNNING and start
|
|
246
|
+
the round. Use this when the user wants to review the audience or
|
|
247
|
+
prompt before any credits are charged. Audience flags are still
|
|
248
|
+
required (testers materialize at create time); only the worker
|
|
249
|
+
enqueue and billing are deferred. Asks now carry a top-level
|
|
250
|
+
\`status\` (\`draft | running | completed | cancelled\`) visible in
|
|
251
|
+
\`ask list\` and \`ask get\`. \`dispatch\` is idempotent — a
|
|
252
|
+
non-DRAFT ask returns 409 mapped to a usage error.
|
|
239
253
|
- **\`ask results --json\` adds \`cross_round_summary\` for 2+ rounds.**
|
|
240
254
|
Top-level field with per-round picks/winner snapshots and
|
|
241
255
|
\`picks_delta\` (R1 → last). Don't diff two \`ask results\` calls by
|
|
@@ -608,7 +622,41 @@ you can branch on plan caps before \`study create\` returns
|
|
|
608
622
|
The full reference is at \`ish docs get-page guides/chat\`,
|
|
609
623
|
secrets are at \`ish docs get-page concepts/secret\`.
|
|
610
624
|
|
|
611
|
-
## 8.
|
|
625
|
+
## 8. Stage an ask for human review, then dispatch
|
|
626
|
+
|
|
627
|
+
Goal: prepare a billable A/B but let the user inspect and approve the
|
|
628
|
+
audience + prompt before any credits are spent. Two-step flow with a
|
|
629
|
+
DRAFT status in between.
|
|
630
|
+
|
|
631
|
+
\`\`\`bash
|
|
632
|
+
# 1. Stage. No worker enqueued, no bill. Audience flags are still
|
|
633
|
+
# required — testers materialize at create time.
|
|
634
|
+
ASK=$(ish ask create --name "tagline AB" \\
|
|
635
|
+
--prompt "Which sounds better?" \\
|
|
636
|
+
--variant text:"Short and punchy." \\
|
|
637
|
+
--variant text:"A longer, descriptive line." \\
|
|
638
|
+
--sample 30 --wants-pick \\
|
|
639
|
+
--no-dispatch \\
|
|
640
|
+
--get alias)
|
|
641
|
+
|
|
642
|
+
# Hand the alias back to the user. They can inspect it:
|
|
643
|
+
# ish ask get "$ASK" # status: draft
|
|
644
|
+
# ish ask get "$ASK" --json | jq '.testers | length'
|
|
645
|
+
|
|
646
|
+
# 2. Dispatch once approved (BILLABLE). Idempotent: a non-DRAFT ask
|
|
647
|
+
# returns 409 mapped to exit 2, so re-running is safe.
|
|
648
|
+
ish ask dispatch "$ASK" --wait
|
|
649
|
+
\`\`\`
|
|
650
|
+
|
|
651
|
+
The \`status\` field on the ask reflects lifecycle (\`draft\` → \`running\`
|
|
652
|
+
→ \`completed\`); \`is_archived\` is orthogonal. \`ish ask list\` shows
|
|
653
|
+
status as a column.
|
|
654
|
+
|
|
655
|
+
\`--no-dispatch\` is incompatible with \`--wait\` — there is nothing to
|
|
656
|
+
wait for. Pass \`--wait\` to \`ish ask dispatch\` instead if you want to
|
|
657
|
+
block until the round settles.
|
|
658
|
+
|
|
659
|
+
## 9. Display-vs-capture: a script that does both
|
|
612
660
|
|
|
613
661
|
Goal: drive an A/B in a script, capture aliases without \`jq\`, and
|
|
614
662
|
still show the human a readable result table at the end.
|
package/dist/lib/types.d.ts
CHANGED
|
@@ -245,6 +245,7 @@ export interface SimulationConfig {
|
|
|
245
245
|
}
|
|
246
246
|
export type AskVariantKind = "image" | "text" | "audio" | "video" | "document";
|
|
247
247
|
export declare const ASK_VARIANT_KINDS: AskVariantKind[];
|
|
248
|
+
export type AskStatus = "draft" | "running" | "completed" | "cancelled";
|
|
248
249
|
export type AskRoundStatus = "running" | "completed" | "errored";
|
|
249
250
|
export type AskResponseStatus = "pending" | "completed" | "errored";
|
|
250
251
|
export interface AskVariant {
|
|
@@ -293,6 +294,7 @@ export interface AskCreateInput {
|
|
|
293
294
|
language?: string;
|
|
294
295
|
tester_profile_ids: string[];
|
|
295
296
|
first_round: AskRoundInput;
|
|
297
|
+
dispatch?: boolean;
|
|
296
298
|
}
|
|
297
299
|
export interface AskUpdateInput {
|
|
298
300
|
name?: string;
|
|
@@ -355,6 +357,7 @@ export interface Ask {
|
|
|
355
357
|
name: string;
|
|
356
358
|
description?: string | null;
|
|
357
359
|
is_archived: boolean;
|
|
360
|
+
status?: AskStatus;
|
|
358
361
|
testers: AskAudienceTester[];
|
|
359
362
|
rounds: AskRound[];
|
|
360
363
|
created_at: string;
|
|
@@ -367,6 +370,7 @@ export interface AskListItem {
|
|
|
367
370
|
name: string;
|
|
368
371
|
description?: string | null;
|
|
369
372
|
is_archived: boolean;
|
|
373
|
+
status?: AskStatus;
|
|
370
374
|
audience_count: number;
|
|
371
375
|
round_count: number;
|
|
372
376
|
last_round_at?: string | null;
|