@roarkanalytics/cli 0.61.1 → 0.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -0
- package/README.md +45 -0
- package/commands.js +575 -0
- package/completions.js +4 -4
- package/man/roark-calendar-connection-check.1 +93 -0
- package/man/roark-calendar-connection-create.1 +102 -0
- package/man/roark-calendar-connection-delete.1 +93 -0
- package/man/roark-calendar-connection-get.1 +93 -0
- package/man/roark-calendar-connection-list.1 +93 -0
- package/man/roark-calendar-connection-target-get.1 +96 -0
- package/man/roark-calendar-connection-target-list.1 +93 -0
- package/man/roark-calendar-connection-update.1 +103 -0
- package/man/roark-metric-definition-update.1 +3 -0
- package/man/roark-metric-variant-update.1 +3 -0
- package/man/roark-tool-create.1 +123 -0
- package/man/roark-tool-delete.1 +93 -0
- package/man/roark-tool-get.1 +93 -0
- package/man/roark-tool-list.1 +96 -0
- package/man/roark-tool-update.1 +127 -0
- package/man/roark.1 +39 -0
- package/npm-shrinkwrap.json +6 -6
- package/package.json +15 -2
- package/runtime/commands/config-as-code.js +5 -23
- package/runtime/commands/get-started.d.ts +34 -0
- package/runtime/commands/get-started.js +350 -0
- package/runtime/config-bundle.d.ts +10 -0
- package/runtime/config-bundle.js +14 -3
- package/runtime/config-changes.d.ts +15 -0
- package/runtime/config-changes.js +32 -0
- package/runtime/get-started/account.d.ts +18 -0
- package/runtime/get-started/account.js +35 -0
- package/runtime/get-started/agent.d.ts +17 -0
- package/runtime/get-started/agent.js +116 -0
- package/runtime/get-started/answers.d.ts +15 -0
- package/runtime/get-started/answers.js +42 -0
- package/runtime/get-started/apply.d.ts +35 -0
- package/runtime/get-started/apply.js +54 -0
- package/runtime/get-started/existing.d.ts +15 -0
- package/runtime/get-started/existing.js +47 -0
- package/runtime/get-started/ladder.d.ts +37 -0
- package/runtime/get-started/ladder.js +69 -0
- package/runtime/get-started/module.d.ts +20 -0
- package/runtime/get-started/module.js +159 -0
- package/runtime/get-started/placeholder.d.ts +9 -0
- package/runtime/get-started/placeholder.js +36 -0
- package/runtime/get-started/prompt.d.ts +32 -0
- package/runtime/get-started/prompt.js +132 -0
- package/runtime/get-started/resources.d.ts +106 -0
- package/runtime/get-started/resources.js +184 -0
- package/runtime/get-started/run.d.ts +42 -0
- package/runtime/get-started/run.js +101 -0
- package/runtime/get-started/scaffold-only.d.ts +19 -0
- package/runtime/get-started/scaffold-only.js +43 -0
- package/runtime/get-started/scaffold.d.ts +15 -0
- package/runtime/get-started/scaffold.js +35 -0
- package/runtime/get-started/starter-metrics.d.ts +33 -0
- package/runtime/get-started/starter-metrics.js +104 -0
- package/runtime/get-started/steps.d.ts +33 -0
- package/runtime/get-started/steps.js +112 -0
- package/runtime/get-started/target.d.ts +47 -0
- package/runtime/get-started/target.js +81 -0
- package/runtime/get-started/types.d.ts +18 -0
- package/runtime/get-started/types.js +8 -0
- package/runtime/get-started/watch.d.ts +57 -0
- package/runtime/get-started/watch.js +103 -0
- package/runtime/program.js +5 -0
- package/version.d.ts +1 -1
- package/version.js +1 -1
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Step 7: start the run and watch it to a verdict.
|
|
3
|
+
*
|
|
4
|
+
* Where the interesting failure modes live: a plan id that has to be resolved two ways, a
|
|
5
|
+
* billable action that needs its own confirmation, and a wait that can end three different ways.
|
|
6
|
+
*/
|
|
7
|
+
import type Roark from '@roarkanalytics/sdk';
|
|
8
|
+
import type { AgentSetup } from './resources';
|
|
9
|
+
import type { ConfigChange, ResolveOutput } from './types';
|
|
10
|
+
import type { Prompter } from './prompt';
|
|
11
|
+
interface RunPhase {
|
|
12
|
+
client: Roark;
|
|
13
|
+
prompter: Prompter;
|
|
14
|
+
say: (text: string) => void;
|
|
15
|
+
binaryName: string;
|
|
16
|
+
color: boolean;
|
|
17
|
+
quiet: boolean;
|
|
18
|
+
options: Record<string, unknown>;
|
|
19
|
+
planName: string;
|
|
20
|
+
/** What to call the config in the closing "what next" block. */
|
|
21
|
+
dir: string;
|
|
22
|
+
/** The apply command for that config, which differs between YAML and a config module. */
|
|
23
|
+
apply: string;
|
|
24
|
+
/** The line the run will dial, quoted back in the confirmation. */
|
|
25
|
+
dial: AgentSetup['dial'];
|
|
26
|
+
/** The apply result, when there was one. Absent when the project already matched the files. */
|
|
27
|
+
applied: ConfigChange[] | undefined;
|
|
28
|
+
resolveOutput: ResolveOutput;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Step 7: start the run and watch it to a verdict.
|
|
32
|
+
*
|
|
33
|
+
* Its own function because it is the one phase that stands alone (everything it needs is a plan
|
|
34
|
+
* id and a line), and because it is where the interesting failure modes live: a plan id that has
|
|
35
|
+
* to be resolved two ways, a billable action that needs its own confirmation, and a wait that can
|
|
36
|
+
* end three different ways.
|
|
37
|
+
*
|
|
38
|
+
* Returns true when it has reported the outcome and the command is finished, false when it could
|
|
39
|
+
* not start and the caller should print its own closing text.
|
|
40
|
+
*/
|
|
41
|
+
export declare const startRun: (phase: RunPhase) => Promise<boolean>;
|
|
42
|
+
export {};
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Step 7: start the run and watch it to a verdict.
|
|
4
|
+
*
|
|
5
|
+
* Where the interesting failure modes live: a plan id that has to be resolved two ways, a
|
|
6
|
+
* billable action that needs its own confirmation, and a wait that can end three different ways.
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.startRun = void 0;
|
|
10
|
+
const output_1 = require("../output");
|
|
11
|
+
const ladder_1 = require("./ladder");
|
|
12
|
+
const watch_1 = require("./watch");
|
|
13
|
+
/** How long to wait for the first run, and how often to ask. A simulated call takes a minute or two. */
|
|
14
|
+
const WATCH_TIMEOUT_MS = 6 * 60_000;
|
|
15
|
+
const POLL_MS = 5_000;
|
|
16
|
+
/**
|
|
17
|
+
* Step 7: start the run and watch it to a verdict.
|
|
18
|
+
*
|
|
19
|
+
* Its own function because it is the one phase that stands alone (everything it needs is a plan
|
|
20
|
+
* id and a line), and because it is where the interesting failure modes live: a plan id that has
|
|
21
|
+
* to be resolved two ways, a billable action that needs its own confirmation, and a wait that can
|
|
22
|
+
* end three different ways.
|
|
23
|
+
*
|
|
24
|
+
* Returns true when it has reported the outcome and the command is finished, false when it could
|
|
25
|
+
* not start and the caller should print its own closing text.
|
|
26
|
+
*/
|
|
27
|
+
const startRun = async (phase) => {
|
|
28
|
+
const { client, prompter, say, binaryName, color, quiet, options, planName } = phase;
|
|
29
|
+
say((0, ladder_1.stepHeader)(6, color));
|
|
30
|
+
// Two ways to the id, because there are two ways to get here: an apply that just created the
|
|
31
|
+
// plan carries it, and a re-run over an already-applied directory has no apply result at all.
|
|
32
|
+
const planId = phase.applied?.find((entry) => entry.configKey === `simulationPlan/${planName}`)?.id ??
|
|
33
|
+
(await findPlanId(client, planName));
|
|
34
|
+
if (planId === undefined) {
|
|
35
|
+
say((0, ladder_1.note)(`Could not work out the plan's id. Find it with \`${binaryName} simulation plan list\` and run \`${binaryName} simulation run --data '{"planId":"<id>"}'\`.`, color, 'yellow'));
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
// A run places a real call on a real line and is billed, so it is confirmed on its own rather
|
|
39
|
+
// than riding on the apply. `--yes` covers both, `--no-run` skips it.
|
|
40
|
+
const dialled = phase.dial !== undefined && 'value' in phase.dial ? phase.dial.value : 'your agent';
|
|
41
|
+
const start = options['yes'] === true ||
|
|
42
|
+
(await prompter.confirm(`Start the run? It places a real call to ${dialled}.`, true));
|
|
43
|
+
if (!start) {
|
|
44
|
+
say((0, ladder_1.stepSkipped)(6, 'not started', color));
|
|
45
|
+
say('');
|
|
46
|
+
say(`The plan is saved. Start it whenever you like: \`${binaryName} simulation run --data '{"planId":"${planId}"}'\`.`);
|
|
47
|
+
if (!quiet)
|
|
48
|
+
(0, output_1.write)(planId);
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
const run = await client.simulation.run({ planId });
|
|
52
|
+
const jobId = run.data.simulationRunPlanJobId;
|
|
53
|
+
say((0, ladder_1.echoCommand)(`${binaryName} simulation run --data '{"planId":"${planId}"}'`, color));
|
|
54
|
+
say(`Started ${(0, output_1.paint)(jobId, 'cyan', color)}: ${run.data.simulationJobCount} call(s). Waiting for the verdict...`);
|
|
55
|
+
const watched = await (0, watch_1.watchJob)({
|
|
56
|
+
jobId,
|
|
57
|
+
getJob: async (id) => (await client.simulationRunPlanJob.getByID(id)).data,
|
|
58
|
+
onProgress: (line) => say(` ${(0, output_1.paint)(line, 'dim', color)}`),
|
|
59
|
+
timeoutMs: WATCH_TIMEOUT_MS,
|
|
60
|
+
pollMs: POLL_MS,
|
|
61
|
+
sleep: watch_1.sleep,
|
|
62
|
+
now: () => Date.now(),
|
|
63
|
+
});
|
|
64
|
+
if (watched.outcome === 'settled') {
|
|
65
|
+
say((0, ladder_1.stepDone)(6, (0, output_1.paint)((0, watch_1.verdictLine)(watched.snapshot), (0, watch_1.verdictStyle)(watched.snapshot), color), color));
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
// Neither of these is a failed run: the run is fine, it is this command that stopped watching,
|
|
69
|
+
// so it is reported as a skipped step rather than as a verdict.
|
|
70
|
+
say((0, ladder_1.stepSkipped)(6, watched.outcome === 'unreachable' ?
|
|
71
|
+
'could not read the run from the API. It is still going'
|
|
72
|
+
: `still running after ${Math.round(WATCH_TIMEOUT_MS / 60_000)} minutes. It will finish on its own`, color));
|
|
73
|
+
}
|
|
74
|
+
say((0, ladder_1.nextSteps)({
|
|
75
|
+
binaryName,
|
|
76
|
+
dir: phase.dir,
|
|
77
|
+
apply: phase.apply,
|
|
78
|
+
lines: [
|
|
79
|
+
`${(0, output_1.paint)(`${binaryName} simulation plan job get ${jobId}`, 'cyan', color)} every call, with its metrics`,
|
|
80
|
+
`${(0, output_1.paint)(`${binaryName} simulation run --data '{"planId":"${planId}"}'`, 'cyan', color)} run it again`,
|
|
81
|
+
],
|
|
82
|
+
}, color));
|
|
83
|
+
if (!quiet) {
|
|
84
|
+
const output = phase.resolveOutput(options);
|
|
85
|
+
(0, output_1.write)(output.format === 'json' || options['json'] === true ?
|
|
86
|
+
(0, output_1.render)({ planId, jobId, status: watched.snapshot.status }, output)
|
|
87
|
+
: jobId);
|
|
88
|
+
}
|
|
89
|
+
return true;
|
|
90
|
+
};
|
|
91
|
+
exports.startRun = startRun;
|
|
92
|
+
/** The plan's id when the apply result did not carry one, e.g. because nothing about it changed. */
|
|
93
|
+
const findPlanId = async (client, name) => {
|
|
94
|
+
try {
|
|
95
|
+
const response = await client.simulationRunPlan.list({ searchText: name, limit: 50 });
|
|
96
|
+
return response.data.find((plan) => plan.name === name)?.id;
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
};
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `--scaffold-only`: write the config with placeholders and stop.
|
|
3
|
+
*
|
|
4
|
+
* A complete alternative mode rather than a step of the guided flow: it asks nothing, calls no
|
|
5
|
+
* endpoint, and ends with a directory someone fills in by hand. The path for a script, a
|
|
6
|
+
* dotfiles repo, or anyone who would rather read the config format than answer questions about
|
|
7
|
+
* it, which is why it has to work with no terminal and no credential.
|
|
8
|
+
*/
|
|
9
|
+
import { type Target } from './target';
|
|
10
|
+
export interface ScaffoldOnlyRun {
|
|
11
|
+
binaryName: string;
|
|
12
|
+
target: Target;
|
|
13
|
+
color: boolean;
|
|
14
|
+
quiet: boolean;
|
|
15
|
+
say: (text: string) => void;
|
|
16
|
+
/** Said here too: a config module nobody can import is worth flagging before it is written. */
|
|
17
|
+
warnIfUnimportable: () => void;
|
|
18
|
+
}
|
|
19
|
+
export declare const scaffoldOnlyRun: (run: ScaffoldOnlyRun) => Promise<void>;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* `--scaffold-only`: write the config with placeholders and stop.
|
|
4
|
+
*
|
|
5
|
+
* A complete alternative mode rather than a step of the guided flow: it asks nothing, calls no
|
|
6
|
+
* endpoint, and ends with a directory someone fills in by hand. The path for a script, a
|
|
7
|
+
* dotfiles repo, or anyone who would rather read the config format than answer questions about
|
|
8
|
+
* it, which is why it has to work with no terminal and no credential.
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.scaffoldOnlyRun = void 0;
|
|
12
|
+
const output_1 = require("../output");
|
|
13
|
+
const target_1 = require("./target");
|
|
14
|
+
const ladder_1 = require("./ladder");
|
|
15
|
+
const module_1 = require("./module");
|
|
16
|
+
const placeholder_1 = require("./placeholder");
|
|
17
|
+
const resources_1 = require("./resources");
|
|
18
|
+
const scaffold_1 = require("./scaffold");
|
|
19
|
+
const scaffoldOnlyRun = async (run) => {
|
|
20
|
+
const { binaryName, target, color, quiet, say } = run;
|
|
21
|
+
run.warnIfUnimportable();
|
|
22
|
+
const placeholder = (0, placeholder_1.placeholderSetup)();
|
|
23
|
+
const files = target.format === 'ts' ? [(0, module_1.setupModule)(placeholder)] : (0, resources_1.setupFiles)(placeholder);
|
|
24
|
+
const result = await (0, scaffold_1.writeScaffold)(target.dir, files);
|
|
25
|
+
for (const path of result.written)
|
|
26
|
+
say(`${(0, output_1.paint)('+', 'green', color)} ${path}`);
|
|
27
|
+
for (const path of result.skipped) {
|
|
28
|
+
say(`${(0, output_1.paint)('·', 'yellow', color)} ${path} (already there, left alone)`);
|
|
29
|
+
}
|
|
30
|
+
say('');
|
|
31
|
+
for (const todo of (0, placeholder_1.placeholderTodos)(target.format))
|
|
32
|
+
say((0, ladder_1.note)(`TODO: ${todo}`, color, 'yellow'));
|
|
33
|
+
const diff = `${binaryName} config diff${target.argument === '' ? '' : ` ${target.argument}`}`;
|
|
34
|
+
say((0, ladder_1.nextSteps)({
|
|
35
|
+
binaryName,
|
|
36
|
+
dir: target.label,
|
|
37
|
+
apply: (0, target_1.applyCommand)(binaryName, target),
|
|
38
|
+
lines: [`${(0, output_1.paint)(diff, 'cyan', color)} preview what it would create`],
|
|
39
|
+
}, color));
|
|
40
|
+
if (!quiet)
|
|
41
|
+
(0, output_1.write)(target.label);
|
|
42
|
+
};
|
|
43
|
+
exports.scaffoldOnlyRun = scaffoldOnlyRun;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Writing the config directory.
|
|
3
|
+
*
|
|
4
|
+
* A file that already exists is left exactly as it is and reported as skipped, never merged and
|
|
5
|
+
* never overwritten: someone who has edited their config and runs `get-started` again is asking
|
|
6
|
+
* to fill the gaps, not to lose their edits. That is also what makes the command safe to re-run,
|
|
7
|
+
* which is how a half-finished setup gets finished.
|
|
8
|
+
*/
|
|
9
|
+
import type { ScaffoldFile } from './resources';
|
|
10
|
+
export interface ScaffoldResult {
|
|
11
|
+
/** Paths as given to the command, so they are copy-pasteable. */
|
|
12
|
+
written: string[];
|
|
13
|
+
skipped: string[];
|
|
14
|
+
}
|
|
15
|
+
export declare const writeScaffold: (dir: string, files: readonly ScaffoldFile[]) => Promise<ScaffoldResult>;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Writing the config directory.
|
|
4
|
+
*
|
|
5
|
+
* A file that already exists is left exactly as it is and reported as skipped, never merged and
|
|
6
|
+
* never overwritten: someone who has edited their config and runs `get-started` again is asking
|
|
7
|
+
* to fill the gaps, not to lose their edits. That is also what makes the command safe to re-run,
|
|
8
|
+
* which is how a half-finished setup gets finished.
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.writeScaffold = void 0;
|
|
12
|
+
const promises_1 = require("node:fs/promises");
|
|
13
|
+
const node_path_1 = require("node:path");
|
|
14
|
+
const writeScaffold = async (dir, files) => {
|
|
15
|
+
const root = (0, node_path_1.resolve)(dir);
|
|
16
|
+
const result = { written: [], skipped: [] };
|
|
17
|
+
for (const file of files) {
|
|
18
|
+
const absolute = (0, node_path_1.join)(root, file.path);
|
|
19
|
+
await (0, promises_1.mkdir)((0, node_path_1.dirname)(absolute), { recursive: true });
|
|
20
|
+
const shown = (0, node_path_1.join)(dir, file.path);
|
|
21
|
+
try {
|
|
22
|
+
// `wx` rather than an exists check: the check-then-write race is exactly how a concurrent
|
|
23
|
+
// run would clobber a file this function promises never to touch.
|
|
24
|
+
await (0, promises_1.writeFile)(absolute, file.contents, { encoding: 'utf8', flag: 'wx' });
|
|
25
|
+
result.written.push(shown);
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
if (error.code !== 'EEXIST')
|
|
29
|
+
throw error;
|
|
30
|
+
result.skipped.push(shown);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return result;
|
|
34
|
+
};
|
|
35
|
+
exports.writeScaffold = writeScaffold;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which metrics a first run is graded by.
|
|
3
|
+
*
|
|
4
|
+
* Roark ships system metrics that work on any agent with no setup, so the opening question is a
|
|
5
|
+
* pick list rather than an essay. The list below is curated: these are the ones that say something
|
|
6
|
+
* useful about a call nobody has configured anything for yet.
|
|
7
|
+
*
|
|
8
|
+
* The project's real metrics are fetched and the curated list is intersected with them, so a slug
|
|
9
|
+
* that would be rejected on apply is never offered. If that intersection comes back thin (an older
|
|
10
|
+
* stage, a trimmed project), the project's own metrics are offered instead of a list of names that
|
|
11
|
+
* do not exist there.
|
|
12
|
+
*/
|
|
13
|
+
import type { Choice } from './prompt';
|
|
14
|
+
export interface StarterMetric {
|
|
15
|
+
slug: string;
|
|
16
|
+
label: string;
|
|
17
|
+
hint: string;
|
|
18
|
+
/** Pre-marked, so pressing Enter gives a run that is actually graded. */
|
|
19
|
+
recommended: boolean;
|
|
20
|
+
}
|
|
21
|
+
export declare const STARTER_METRICS: readonly StarterMetric[];
|
|
22
|
+
export interface AvailableMetric {
|
|
23
|
+
metricId?: string | null;
|
|
24
|
+
name?: string | null;
|
|
25
|
+
}
|
|
26
|
+
export interface MetricOptions {
|
|
27
|
+
choices: Choice<string>[];
|
|
28
|
+
/** Pre-marked slugs. Always at least one when there is anything to offer. */
|
|
29
|
+
recommended: string[];
|
|
30
|
+
/** False when the list could not be checked against the project, so it may contain a slug it lacks. */
|
|
31
|
+
verified: boolean;
|
|
32
|
+
}
|
|
33
|
+
export declare const metricOptions: (available: readonly AvailableMetric[] | undefined) => MetricOptions;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Which metrics a first run is graded by.
|
|
4
|
+
*
|
|
5
|
+
* Roark ships system metrics that work on any agent with no setup, so the opening question is a
|
|
6
|
+
* pick list rather than an essay. The list below is curated: these are the ones that say something
|
|
7
|
+
* useful about a call nobody has configured anything for yet.
|
|
8
|
+
*
|
|
9
|
+
* The project's real metrics are fetched and the curated list is intersected with them, so a slug
|
|
10
|
+
* that would be rejected on apply is never offered. If that intersection comes back thin (an older
|
|
11
|
+
* stage, a trimmed project), the project's own metrics are offered instead of a list of names that
|
|
12
|
+
* do not exist there.
|
|
13
|
+
*/
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.metricOptions = exports.STARTER_METRICS = void 0;
|
|
16
|
+
exports.STARTER_METRICS = [
|
|
17
|
+
{
|
|
18
|
+
slug: 'agent_responsive',
|
|
19
|
+
label: 'Agent responsive',
|
|
20
|
+
hint: 'did the agent hold up its end of the call',
|
|
21
|
+
recommended: true,
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
slug: 'call_ended_cleanly',
|
|
25
|
+
label: 'Call ended cleanly',
|
|
26
|
+
hint: 'closed deliberately, not in dead air or an error',
|
|
27
|
+
recommended: true,
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
slug: 'scenario_adherence',
|
|
31
|
+
label: 'Flow adherence',
|
|
32
|
+
hint: 'how closely the call followed the scenario',
|
|
33
|
+
recommended: true,
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
slug: 'call_outcome',
|
|
37
|
+
label: 'Call outcome',
|
|
38
|
+
hint: 'what the call actually achieved',
|
|
39
|
+
recommended: false,
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
slug: 'instruction_follow',
|
|
43
|
+
label: 'Instruction following',
|
|
44
|
+
hint: 'did the agent do what it was told',
|
|
45
|
+
recommended: false,
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
slug: 'frustration_score',
|
|
49
|
+
label: 'Caller frustration',
|
|
50
|
+
hint: 'how frustrated the caller got',
|
|
51
|
+
recommended: false,
|
|
52
|
+
},
|
|
53
|
+
{ slug: 'sentiment_score', label: 'Sentiment', hint: 'the tone of the conversation', recommended: false },
|
|
54
|
+
{
|
|
55
|
+
slug: 'response_time',
|
|
56
|
+
label: 'Response time',
|
|
57
|
+
hint: 'how long the agent took to answer',
|
|
58
|
+
recommended: false,
|
|
59
|
+
},
|
|
60
|
+
];
|
|
61
|
+
/** Below this many curated matches, the project's own metrics are better than a curated guess. */
|
|
62
|
+
const MIN_CURATED_MATCHES = 2;
|
|
63
|
+
/** How many of the project's metrics to offer when falling back. A pick list has to stay readable. */
|
|
64
|
+
const FALLBACK_LIMIT = 6;
|
|
65
|
+
const toChoice = (metric) => ({
|
|
66
|
+
value: metric.slug,
|
|
67
|
+
label: metric.label,
|
|
68
|
+
hint: metric.hint,
|
|
69
|
+
});
|
|
70
|
+
const metricOptions = (available) => {
|
|
71
|
+
// The list could not be read (offline, or the credential lacks metric:read). Offer the curated
|
|
72
|
+
// set unverified rather than nothing: apply will reject a slug the project lacks, clearly.
|
|
73
|
+
if (available === undefined) {
|
|
74
|
+
return {
|
|
75
|
+
choices: exports.STARTER_METRICS.map(toChoice),
|
|
76
|
+
recommended: exports.STARTER_METRICS.filter((metric) => metric.recommended).map((metric) => metric.slug),
|
|
77
|
+
verified: false,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
const slugs = new Set(available.map((metric) => metric.metricId).filter((slug) => typeof slug === 'string'));
|
|
81
|
+
const curated = exports.STARTER_METRICS.filter((metric) => slugs.has(metric.slug));
|
|
82
|
+
if (curated.length >= MIN_CURATED_MATCHES) {
|
|
83
|
+
const recommended = curated.filter((metric) => metric.recommended).map((metric) => metric.slug);
|
|
84
|
+
return {
|
|
85
|
+
choices: curated.map(toChoice),
|
|
86
|
+
// Never empty: a plan needs at least one metric, and Enter has to produce a valid plan.
|
|
87
|
+
recommended: recommended.length > 0 ? recommended : [curated[0].slug],
|
|
88
|
+
verified: true,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
const fallback = available
|
|
92
|
+
.filter((metric) => typeof metric.metricId === 'string')
|
|
93
|
+
.slice(0, FALLBACK_LIMIT);
|
|
94
|
+
return {
|
|
95
|
+
choices: fallback.map((metric) => ({
|
|
96
|
+
value: metric.metricId,
|
|
97
|
+
label: typeof metric.name === 'string' && metric.name.length > 0 ? metric.name : metric.metricId,
|
|
98
|
+
hint: metric.metricId,
|
|
99
|
+
})),
|
|
100
|
+
recommended: fallback.slice(0, MIN_CURATED_MATCHES).map((metric) => metric.metricId),
|
|
101
|
+
verified: true,
|
|
102
|
+
};
|
|
103
|
+
};
|
|
104
|
+
exports.metricOptions = metricOptions;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Steps 3 to 5: the caller, the scenario, and what grades the call.
|
|
3
|
+
*
|
|
4
|
+
* One function per step, each taking the same `Steps` context and returning its own slice of the
|
|
5
|
+
* `Setup`. Step 2 is its own module (`agent.ts`) only because it is several times the size of
|
|
6
|
+
* these; it has the same shape.
|
|
7
|
+
*/
|
|
8
|
+
import type Roark from '@roarkanalytics/sdk';
|
|
9
|
+
import type { AgentSetup, CustomMetricSetup, FlowSetup, PersonaSetup } from './resources';
|
|
10
|
+
import type { Prompter } from './prompt';
|
|
11
|
+
export interface Steps {
|
|
12
|
+
prompter: Prompter;
|
|
13
|
+
say: (text: string) => void;
|
|
14
|
+
color: boolean;
|
|
15
|
+
binaryName: string;
|
|
16
|
+
/** Resources the config already declares, which is what a step skips itself for. */
|
|
17
|
+
existing: Record<string, unknown>[];
|
|
18
|
+
/** What to call the config when pointing at it. */
|
|
19
|
+
label: string;
|
|
20
|
+
}
|
|
21
|
+
/** A step's answer, plus whether this run authored it or read it back out of the config. */
|
|
22
|
+
export type Authored<T> = T & {
|
|
23
|
+
authored: boolean;
|
|
24
|
+
};
|
|
25
|
+
/** Step 3: who calls the agent. */
|
|
26
|
+
export declare const chooseCaller: (steps: Steps) => Promise<Authored<PersonaSetup>>;
|
|
27
|
+
/** Step 4: what the call is about. */
|
|
28
|
+
export declare const chooseScenario: (steps: Steps, agent: AgentSetup, agentIsNew: boolean) => Promise<Authored<FlowSetup>>;
|
|
29
|
+
/** Step 5: how the call is graded. */
|
|
30
|
+
export declare const chooseMetrics: (steps: Steps, client: Roark, declaredPlan: string | undefined) => Promise<{
|
|
31
|
+
metrics: string[];
|
|
32
|
+
customMetric?: CustomMetricSetup;
|
|
33
|
+
}>;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Steps 3 to 5: the caller, the scenario, and what grades the call.
|
|
4
|
+
*
|
|
5
|
+
* One function per step, each taking the same `Steps` context and returning its own slice of the
|
|
6
|
+
* `Setup`. Step 2 is its own module (`agent.ts`) only because it is several times the size of
|
|
7
|
+
* these; it has the same shape.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.chooseMetrics = exports.chooseScenario = exports.chooseCaller = void 0;
|
|
11
|
+
const ladder_1 = require("./ladder");
|
|
12
|
+
const answers_1 = require("./answers");
|
|
13
|
+
const existing_1 = require("./existing");
|
|
14
|
+
const starter_metrics_1 = require("./starter-metrics");
|
|
15
|
+
/** A plan needs at least one metric, so an empty pick list still has to produce a valid plan. */
|
|
16
|
+
const FALLBACK_METRIC = 'agent_responsive';
|
|
17
|
+
/** Step 3: who calls the agent. */
|
|
18
|
+
const chooseCaller = async (steps) => {
|
|
19
|
+
const { prompter, say, color, binaryName } = steps;
|
|
20
|
+
const declared = (0, existing_1.namesOfKind)(steps.existing, 'persona')[0];
|
|
21
|
+
if (declared !== undefined) {
|
|
22
|
+
say((0, ladder_1.stepSkipped)(2, `${declared}, already declared in ${steps.label}`, color));
|
|
23
|
+
// The declared file is the source of truth for the rest of its fields; nothing here reads
|
|
24
|
+
// them, because nothing here writes that file again.
|
|
25
|
+
return { name: declared, language: 'EN', accent: 'US', gender: 'FEMALE', authored: false };
|
|
26
|
+
}
|
|
27
|
+
say((0, ladder_1.stepHeader)(2, color));
|
|
28
|
+
const name = await prompter.ask('What should we call the caller?', {
|
|
29
|
+
default: 'curious-caller',
|
|
30
|
+
validate: (0, answers_1.nameRules)('name'),
|
|
31
|
+
});
|
|
32
|
+
const language = await (0, answers_1.pickCode)(prompter, 'What language do they speak?', answers_1.LANGUAGES, 'EN');
|
|
33
|
+
const accent = await (0, answers_1.pickCode)(prompter, 'With which accent?', answers_1.ACCENTS, 'US');
|
|
34
|
+
const gender = await prompter.select('Voice?', answers_1.GENDERS, 'FEMALE');
|
|
35
|
+
say((0, ladder_1.stepDone)(2, `${name} (${language}/${accent}/${gender.toLowerCase()})`, color));
|
|
36
|
+
say((0, ladder_1.echoCommand)(`${binaryName} simulation persona create --name ${name} --language ${language} --accent ${accent} --gender ${gender}`, color));
|
|
37
|
+
return { name, language, accent, gender, authored: true };
|
|
38
|
+
};
|
|
39
|
+
exports.chooseCaller = chooseCaller;
|
|
40
|
+
/** Step 4: what the call is about. */
|
|
41
|
+
const chooseScenario = async (steps, agent, agentIsNew) => {
|
|
42
|
+
const { prompter, say, color } = steps;
|
|
43
|
+
const declared = (0, existing_1.namesOfKind)(steps.existing, 'flow')[0];
|
|
44
|
+
if (declared !== undefined) {
|
|
45
|
+
say((0, ladder_1.stepSkipped)(3, `${declared}, already declared in ${steps.label}`, color));
|
|
46
|
+
// A flow names the agents it is written for. Reusing one that does not name the agent we
|
|
47
|
+
// just declared produces a plan whose flow targets somebody else, which apply accepts and
|
|
48
|
+
// the run then does something unexpected with.
|
|
49
|
+
const resource = steps.existing.find((entry) => entry['kind'] === 'flow' && entry['name'] === declared);
|
|
50
|
+
const targets = Array.isArray(resource?.['agents']) ? resource['agents'] : [];
|
|
51
|
+
if (agentIsNew && !targets.includes(agent.name)) {
|
|
52
|
+
say((0, ladder_1.note)(`${declared} does not list ${agent.name} under \`agents\`. Add it, or the run will exercise a scenario written for a different agent.`, color, 'yellow'));
|
|
53
|
+
}
|
|
54
|
+
return { name: declared, goal: '', authored: false };
|
|
55
|
+
}
|
|
56
|
+
say((0, ladder_1.stepHeader)(3, color));
|
|
57
|
+
const goal = await prompter.ask('What is the caller calling about? Say it as you would to them');
|
|
58
|
+
const name = await prompter.ask('Short name for that scenario?', {
|
|
59
|
+
// Not derived from the sentence above: a slug of "You call to rebook a cancelled cleaning"
|
|
60
|
+
// reads as `you-call-to-rebook-a`, which is worse than a plain default.
|
|
61
|
+
default: 'first-scenario',
|
|
62
|
+
validate: (0, answers_1.nameRules)('name'),
|
|
63
|
+
});
|
|
64
|
+
const written = await prompter.ask('What must the agent do for this call to pass? (Enter to skip)', {
|
|
65
|
+
default: '-',
|
|
66
|
+
});
|
|
67
|
+
say((0, ladder_1.stepDone)(3, name, color));
|
|
68
|
+
return { name, goal, ...(written === '-' ? {} : { expectation: written }), authored: true };
|
|
69
|
+
};
|
|
70
|
+
exports.chooseScenario = chooseScenario;
|
|
71
|
+
/** Step 5: how the call is graded. */
|
|
72
|
+
const chooseMetrics = async (steps, client, declaredPlan) => {
|
|
73
|
+
const { prompter, say, color } = steps;
|
|
74
|
+
// A plan names the metrics, so when one is already declared there is nothing these answers
|
|
75
|
+
// could attach to: that plan is left alone, and the picks would be collected and then quietly
|
|
76
|
+
// dropped. Not asking is better than asking and ignoring.
|
|
77
|
+
if (declaredPlan !== undefined) {
|
|
78
|
+
say((0, ladder_1.stepSkipped)(4, `${declaredPlan} already names the metrics it grades on`, color));
|
|
79
|
+
return { metrics: [] };
|
|
80
|
+
}
|
|
81
|
+
say((0, ladder_1.stepHeader)(4, color));
|
|
82
|
+
let available;
|
|
83
|
+
try {
|
|
84
|
+
const response = await client.metric.listDefinitions({ limit: 500 });
|
|
85
|
+
available = response.data.map((metric) => ({ metricId: metric.metricId, name: metric.name }));
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
available = undefined;
|
|
89
|
+
}
|
|
90
|
+
const offered = (0, starter_metrics_1.metricOptions)(available);
|
|
91
|
+
if (!offered.verified) {
|
|
92
|
+
say((0, ladder_1.note)("Could not read this project's metrics, so these are Roark's usual ones.", color, 'yellow'));
|
|
93
|
+
}
|
|
94
|
+
const metrics = await prompter.multiSelect('Which metrics should grade the run?', offered.choices, offered.recommended);
|
|
95
|
+
let customMetric;
|
|
96
|
+
if (await prompter.confirm('Add a metric of your own?', false)) {
|
|
97
|
+
const name = await prompter.ask('Short name for it', {
|
|
98
|
+
default: 'my_metric',
|
|
99
|
+
validate: (0, answers_1.nameRules)('slug'),
|
|
100
|
+
});
|
|
101
|
+
const question = await prompter.ask('The yes/no question Roark should judge, e.g. "did the agent confirm the booking date?"');
|
|
102
|
+
customMetric = { name, question };
|
|
103
|
+
metrics.push(name);
|
|
104
|
+
}
|
|
105
|
+
if (metrics.length === 0) {
|
|
106
|
+
metrics.push(FALLBACK_METRIC);
|
|
107
|
+
say((0, ladder_1.note)(`A run needs at least one metric, so ${FALLBACK_METRIC} was kept.`, color, 'yellow'));
|
|
108
|
+
}
|
|
109
|
+
say((0, ladder_1.stepDone)(4, metrics.join(', '), color));
|
|
110
|
+
return { metrics, ...(customMetric === undefined ? {} : { customMetric }) };
|
|
111
|
+
};
|
|
112
|
+
exports.chooseMetrics = chooseMetrics;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where the config is written, and in which form.
|
|
3
|
+
*
|
|
4
|
+
* One object rather than a loose directory string, because the format decides three things that
|
|
5
|
+
* must agree: where files land, what to call the config when pointing at it, and what
|
|
6
|
+
* `config diff` / `config apply` should be given. They were separate once, and a config module
|
|
7
|
+
* chosen at the prompt got written to one path and loaded from another.
|
|
8
|
+
*/
|
|
9
|
+
import type { Prompter } from './prompt';
|
|
10
|
+
import type { ConfigFormat } from './resources';
|
|
11
|
+
export declare const DEFAULT_DIR = "./roark";
|
|
12
|
+
/**
|
|
13
|
+
* Where the config is written, and in which form.
|
|
14
|
+
*
|
|
15
|
+
* YAML needs nothing installed, so it is the default and the only thing offered to someone who
|
|
16
|
+
* reached the CLI through a global install. TypeScript gives types and editor completion, but its
|
|
17
|
+
* `define*` helpers are imported from the SDK, so it is only usable in a project that has the SDK
|
|
18
|
+
* to import from - which is why it is offered rather than assumed.
|
|
19
|
+
*/
|
|
20
|
+
export interface Target {
|
|
21
|
+
format: ConfigFormat;
|
|
22
|
+
/** The directory for YAML, the containing directory for a config module. */
|
|
23
|
+
dir: string;
|
|
24
|
+
/** What to print when pointing at the config: a directory, or the file. */
|
|
25
|
+
label: string;
|
|
26
|
+
/** What `config diff` / `config apply` should be given. */
|
|
27
|
+
argument: string;
|
|
28
|
+
}
|
|
29
|
+
export declare const targetFor: (format: ConfigFormat, dir: string) => Target;
|
|
30
|
+
/** The command that applies this target, spelled the way a person would type it. */
|
|
31
|
+
export declare const applyCommand: (binaryName: string, target: Target) => string;
|
|
32
|
+
/**
|
|
33
|
+
* Whether a config module could actually be loaded from here.
|
|
34
|
+
*
|
|
35
|
+
* `roark.config.ts` imports its helpers from the SDK, and jiti resolves that from the config
|
|
36
|
+
* file's own project. Offering TypeScript where the import cannot resolve would hand someone a
|
|
37
|
+
* config that fails on the next command, so the question is only asked where it can work.
|
|
38
|
+
*/
|
|
39
|
+
export declare const sdkIsInstalled: (dir: string) => boolean;
|
|
40
|
+
/**
|
|
41
|
+
* Which form to write, when the flags did not say.
|
|
42
|
+
*
|
|
43
|
+
* Asked only where it is a real choice: a project that cannot import the helpers cannot load a
|
|
44
|
+
* config module, so offering one would hand someone a config that fails on the very next step,
|
|
45
|
+
* after every question had been answered.
|
|
46
|
+
*/
|
|
47
|
+
export declare const chooseFormat: (prompter: Prompter, chosenDir: string | undefined, current: Target) => Promise<Target>;
|