@roarkanalytics/cli 0.67.1 → 0.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +22 -0
- package/completions.js +3 -3
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/runtime/commands/config-as-code.js +5 -23
- package/runtime/commands/get-started.d.ts +34 -0
- package/runtime/commands/get-started.js +350 -0
- package/runtime/config-bundle.d.ts +10 -0
- package/runtime/config-bundle.js +14 -3
- package/runtime/config-changes.d.ts +15 -0
- package/runtime/config-changes.js +32 -0
- package/runtime/get-started/account.d.ts +18 -0
- package/runtime/get-started/account.js +35 -0
- package/runtime/get-started/agent.d.ts +17 -0
- package/runtime/get-started/agent.js +116 -0
- package/runtime/get-started/answers.d.ts +15 -0
- package/runtime/get-started/answers.js +42 -0
- package/runtime/get-started/apply.d.ts +35 -0
- package/runtime/get-started/apply.js +54 -0
- package/runtime/get-started/existing.d.ts +15 -0
- package/runtime/get-started/existing.js +47 -0
- package/runtime/get-started/ladder.d.ts +37 -0
- package/runtime/get-started/ladder.js +69 -0
- package/runtime/get-started/module.d.ts +20 -0
- package/runtime/get-started/module.js +159 -0
- package/runtime/get-started/placeholder.d.ts +9 -0
- package/runtime/get-started/placeholder.js +36 -0
- package/runtime/get-started/prompt.d.ts +32 -0
- package/runtime/get-started/prompt.js +132 -0
- package/runtime/get-started/resources.d.ts +106 -0
- package/runtime/get-started/resources.js +184 -0
- package/runtime/get-started/run.d.ts +42 -0
- package/runtime/get-started/run.js +101 -0
- package/runtime/get-started/scaffold-only.d.ts +19 -0
- package/runtime/get-started/scaffold-only.js +43 -0
- package/runtime/get-started/scaffold.d.ts +15 -0
- package/runtime/get-started/scaffold.js +35 -0
- package/runtime/get-started/starter-metrics.d.ts +33 -0
- package/runtime/get-started/starter-metrics.js +104 -0
- package/runtime/get-started/steps.d.ts +33 -0
- package/runtime/get-started/steps.js +112 -0
- package/runtime/get-started/target.d.ts +47 -0
- package/runtime/get-started/target.js +81 -0
- package/runtime/get-started/types.d.ts +18 -0
- package/runtime/get-started/types.js +8 -0
- package/runtime/get-started/watch.d.ts +57 -0
- package/runtime/get-started/watch.js +103 -0
- package/runtime/program.js +5 -0
- package/version.d.ts +1 -1
- package/version.js +1 -1
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* What `--scaffold-only` writes: a complete, valid config with nothing invented about you.
|
|
4
|
+
*
|
|
5
|
+
* Every value here is a placeholder someone is expected to replace, which is why the command
|
|
6
|
+
* prints a TODO per file rather than pretending the result is ready to apply.
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.placeholderTodos = exports.placeholderSetup = void 0;
|
|
10
|
+
const module_1 = require("./module");
|
|
11
|
+
const resources_1 = require("./resources");
|
|
12
|
+
const placeholderSetup = () => ({
|
|
13
|
+
agent: {
|
|
14
|
+
name: 'my-agent',
|
|
15
|
+
endpoints: [{ value: '+15551234567', direction: 'INCOMING' }],
|
|
16
|
+
dial: { value: '+15551234567', direction: 'INCOMING' },
|
|
17
|
+
},
|
|
18
|
+
persona: { name: 'curious-caller', language: 'EN', accent: 'US', gender: 'FEMALE' },
|
|
19
|
+
flow: {
|
|
20
|
+
name: 'first-scenario',
|
|
21
|
+
goal: 'You call to ask a question you need answered before you can book.',
|
|
22
|
+
expectation: 'The agent answers the question and offers to help further',
|
|
23
|
+
},
|
|
24
|
+
metrics: ['agent_responsive', 'call_ended_cleanly', 'scenario_adherence'],
|
|
25
|
+
plan: { name: 'first-run', maxDurationSeconds: resources_1.DEFAULT_MAX_DURATION_SECONDS },
|
|
26
|
+
});
|
|
27
|
+
exports.placeholderSetup = placeholderSetup;
|
|
28
|
+
const placeholderTodos = (format) => {
|
|
29
|
+
const agent = format === 'ts' ? module_1.CONFIG_MODULE_FILE : 'agents/my-agent.yaml';
|
|
30
|
+
const flow = format === 'ts' ? module_1.CONFIG_MODULE_FILE : 'flows/first-scenario.yaml';
|
|
31
|
+
return [
|
|
32
|
+
`Replace the endpoint value in ${agent} with the number Roark should call.`,
|
|
33
|
+
`Rewrite the scenario prompt in ${flow} so it describes a real call.`,
|
|
34
|
+
];
|
|
35
|
+
};
|
|
36
|
+
exports.placeholderTodos = placeholderTodos;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The questions `get-started` asks.
|
|
3
|
+
*
|
|
4
|
+
* `confirm.ts` covers the only question the rest of the CLI needs ("are you sure?"). A guided
|
|
5
|
+
* setup needs three more shapes - a value, one choice, a set of choices - so they live here
|
|
6
|
+
* instead of growing that file. Same rules as everywhere else: the conversation is written to
|
|
7
|
+
* stderr so stdout stays reserved for the ids a script captures, and it is all line-oriented
|
|
8
|
+
* (no cursor control), so it behaves the same over ssh, in tmux, and in a dumb terminal.
|
|
9
|
+
*
|
|
10
|
+
* `Prompter` is an interface because scripted answers are the only practical way to test a
|
|
11
|
+
* wizard: a real readline needs a TTY, which CI does not have.
|
|
12
|
+
*/
|
|
13
|
+
export interface Choice<T extends string> {
|
|
14
|
+
value: T;
|
|
15
|
+
label: string;
|
|
16
|
+
/** A few words on what picking this means; dimmed after the label. */
|
|
17
|
+
hint?: string;
|
|
18
|
+
}
|
|
19
|
+
export interface AskOptions {
|
|
20
|
+
default?: string;
|
|
21
|
+
/** Returns a message when the answer is unusable, so the same question can be asked again. */
|
|
22
|
+
validate?: (answer: string) => string | undefined;
|
|
23
|
+
}
|
|
24
|
+
export interface Prompter {
|
|
25
|
+
ask: (question: string, options?: AskOptions) => Promise<string>;
|
|
26
|
+
select: <T extends string>(question: string, choices: readonly Choice<T>[], fallback: T) => Promise<T>;
|
|
27
|
+
/** Returns the chosen values; an empty array is a legitimate answer ("none"). */
|
|
28
|
+
multiSelect: <T extends string>(question: string, choices: readonly Choice<T>[], fallback: readonly T[]) => Promise<T[]>;
|
|
29
|
+
confirm: (question: string, fallback: boolean) => Promise<boolean>;
|
|
30
|
+
close: () => void;
|
|
31
|
+
}
|
|
32
|
+
export declare const createPrompter: () => Prompter;
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The questions `get-started` asks.
|
|
4
|
+
*
|
|
5
|
+
* `confirm.ts` covers the only question the rest of the CLI needs ("are you sure?"). A guided
|
|
6
|
+
* setup needs three more shapes - a value, one choice, a set of choices - so they live here
|
|
7
|
+
* instead of growing that file. Same rules as everywhere else: the conversation is written to
|
|
8
|
+
* stderr so stdout stays reserved for the ids a script captures, and it is all line-oriented
|
|
9
|
+
* (no cursor control), so it behaves the same over ssh, in tmux, and in a dumb terminal.
|
|
10
|
+
*
|
|
11
|
+
* `Prompter` is an interface because scripted answers are the only practical way to test a
|
|
12
|
+
* wizard: a real readline needs a TTY, which CI does not have.
|
|
13
|
+
*/
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.createPrompter = void 0;
|
|
16
|
+
const node_readline_1 = require("node:readline");
|
|
17
|
+
const errors_1 = require("../errors");
|
|
18
|
+
const output_1 = require("../output");
|
|
19
|
+
/**
|
|
20
|
+
* How many times one question is re-asked before giving up. Looping forever on a pipe that will
|
|
21
|
+
* never answer is how a wizard hangs a CI job; three tries is enough for a typo and no more.
|
|
22
|
+
*/
|
|
23
|
+
const ATTEMPTS = 3;
|
|
24
|
+
const exhausted = () => {
|
|
25
|
+
throw new errors_1.UsageError('No usable answer after three attempts, so nothing was changed.');
|
|
26
|
+
};
|
|
27
|
+
const createPrompter = () => {
|
|
28
|
+
const out = process.stderr;
|
|
29
|
+
const color = (0, output_1.supportsColor)(out);
|
|
30
|
+
const rl = (0, node_readline_1.createInterface)({ input: process.stdin, output: out });
|
|
31
|
+
const line = (text) => (0, output_1.write)(text, out);
|
|
32
|
+
const question = (text) => new Promise((resolve) => rl.question(text, resolve));
|
|
33
|
+
const problem = (text) => line((0, output_1.paint)(` ${text}`, 'yellow', color));
|
|
34
|
+
const asked = (text) => `${(0, output_1.paint)('?', 'cyan', color)} ${text}`;
|
|
35
|
+
const listChoices = (choices, marked) => {
|
|
36
|
+
choices.forEach((choice, index) => {
|
|
37
|
+
const mark = marked.includes(choice.value) ? (0, output_1.paint)('>', 'green', color) : ' ';
|
|
38
|
+
const hint = choice.hint === undefined ? '' : (0, output_1.paint)(` ${choice.hint}`, 'dim', color);
|
|
39
|
+
line(` ${mark} ${index + 1}) ${choice.label}${hint}`);
|
|
40
|
+
});
|
|
41
|
+
};
|
|
42
|
+
const ask = async (text, options = {}) => {
|
|
43
|
+
const fallback = options.default;
|
|
44
|
+
const suffix = fallback === undefined ? '' : ` ${(0, output_1.paint)(`[${fallback}]`, 'dim', color)}`;
|
|
45
|
+
for (let attempt = 0; attempt < ATTEMPTS; attempt += 1) {
|
|
46
|
+
const raw = (await question(`${asked(text)}${suffix} `)).trim();
|
|
47
|
+
const answer = raw.length > 0 ? raw : fallback ?? '';
|
|
48
|
+
if (answer.length === 0) {
|
|
49
|
+
problem('A value is needed here.');
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
const invalid = options.validate?.(answer);
|
|
53
|
+
if (invalid !== undefined) {
|
|
54
|
+
problem(invalid);
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
return answer;
|
|
58
|
+
}
|
|
59
|
+
return exhausted();
|
|
60
|
+
};
|
|
61
|
+
const select = async (text, choices, fallback) => {
|
|
62
|
+
if (choices.length === 0)
|
|
63
|
+
throw new errors_1.UsageError(`nothing to choose from for: ${text}`);
|
|
64
|
+
const fallbackIndex = choices.findIndex((choice) => choice.value === fallback);
|
|
65
|
+
for (let attempt = 0; attempt < ATTEMPTS; attempt += 1) {
|
|
66
|
+
line(asked(text));
|
|
67
|
+
listChoices(choices, [fallback]);
|
|
68
|
+
const hint = fallbackIndex < 0 ? '' : (0, output_1.paint)(`[${fallbackIndex + 1}]`, 'dim', color);
|
|
69
|
+
const raw = (await question(` ${hint} `)).trim();
|
|
70
|
+
if (raw.length === 0 && fallbackIndex >= 0)
|
|
71
|
+
return fallback;
|
|
72
|
+
const index = Number(raw);
|
|
73
|
+
if (Number.isInteger(index) && index >= 1 && index <= choices.length)
|
|
74
|
+
return choices[index - 1].value;
|
|
75
|
+
// Typing the answer out is what someone who has run this before will do. Either side counts:
|
|
76
|
+
// the value is what a second run remembers, the label is what is on screen.
|
|
77
|
+
const typed = raw.toLowerCase();
|
|
78
|
+
const named = choices.find((choice) => choice.value.toLowerCase() === typed || choice.label.toLowerCase() === typed);
|
|
79
|
+
if (named)
|
|
80
|
+
return named.value;
|
|
81
|
+
problem(`Pick a number from 1 to ${choices.length}.`);
|
|
82
|
+
}
|
|
83
|
+
return exhausted();
|
|
84
|
+
};
|
|
85
|
+
const multiSelect = async (text, choices, fallback) => {
|
|
86
|
+
if (choices.length === 0)
|
|
87
|
+
return [];
|
|
88
|
+
for (let attempt = 0; attempt < ATTEMPTS; attempt += 1) {
|
|
89
|
+
line(asked(text));
|
|
90
|
+
listChoices(choices, fallback);
|
|
91
|
+
line((0, output_1.paint)(' Numbers, comma separated. Enter keeps the marked ones; "none" picks nothing.', 'dim', color));
|
|
92
|
+
const raw = (await question(' ').then((answer) => answer.trim())).toLowerCase();
|
|
93
|
+
if (raw.length === 0)
|
|
94
|
+
return [...fallback];
|
|
95
|
+
if (raw === 'none')
|
|
96
|
+
return [];
|
|
97
|
+
const parts = raw.split(/[\s,]+/).filter((part) => part.length > 0);
|
|
98
|
+
const picked = [];
|
|
99
|
+
let bad;
|
|
100
|
+
for (const part of parts) {
|
|
101
|
+
const index = Number(part);
|
|
102
|
+
if (!Number.isInteger(index) || index < 1 || index > choices.length) {
|
|
103
|
+
bad = part;
|
|
104
|
+
break;
|
|
105
|
+
}
|
|
106
|
+
const value = choices[index - 1].value;
|
|
107
|
+
if (!picked.includes(value))
|
|
108
|
+
picked.push(value);
|
|
109
|
+
}
|
|
110
|
+
if (bad === undefined)
|
|
111
|
+
return picked;
|
|
112
|
+
problem(`"${bad}" is not one of 1 to ${choices.length}.`);
|
|
113
|
+
}
|
|
114
|
+
return exhausted();
|
|
115
|
+
};
|
|
116
|
+
const confirmPrompt = async (text, fallback) => {
|
|
117
|
+
const hint = (0, output_1.paint)(fallback ? '[Y/n]' : '[y/N]', 'dim', color);
|
|
118
|
+
for (let attempt = 0; attempt < ATTEMPTS; attempt += 1) {
|
|
119
|
+
const raw = (await question(`${asked(text)} ${hint} `)).trim();
|
|
120
|
+
if (raw.length === 0)
|
|
121
|
+
return fallback;
|
|
122
|
+
if (/^y(es)?$/i.test(raw))
|
|
123
|
+
return true;
|
|
124
|
+
if (/^no?$/i.test(raw))
|
|
125
|
+
return false;
|
|
126
|
+
problem('Answer y or n.');
|
|
127
|
+
}
|
|
128
|
+
return exhausted();
|
|
129
|
+
};
|
|
130
|
+
return { ask, select, multiSelect, confirm: confirmPrompt, close: () => rl.close() };
|
|
131
|
+
};
|
|
132
|
+
exports.createPrompter = createPrompter;
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The config-as-code files `get-started` writes.
|
|
3
|
+
*
|
|
4
|
+
* The wizard's output is a config directory, not a sequence of create calls. That is deliberate:
|
|
5
|
+
* the YAML is the thing we want someone to own and commit, `config apply` is idempotent so a
|
|
6
|
+
* second run is safe, and the whole flow then rides on one endpoint pair instead of six resource
|
|
7
|
+
* APIs. The wizard is disposable; the files are not.
|
|
8
|
+
*
|
|
9
|
+
* Everything here is pure - answers in, file contents out - so the shape of what we write is
|
|
10
|
+
* tested without a terminal or a network.
|
|
11
|
+
*/
|
|
12
|
+
/** Editor autocomplete and validation for anyone who opens these files afterwards. */
|
|
13
|
+
export declare const SCHEMA_HEADER = "# yaml-language-server: $schema=https://roark.ai/roark-config.schema.json";
|
|
14
|
+
/**
|
|
15
|
+
* A Roark-managed environment that exists in every project, so a first run needs no environment
|
|
16
|
+
* of its own. Flows reference an environment by its display name.
|
|
17
|
+
*/
|
|
18
|
+
export declare const DEFAULT_ENVIRONMENT = "Quiet line";
|
|
19
|
+
/** How long a simulated call may run before Roark ends it. */
|
|
20
|
+
export declare const DEFAULT_MAX_DURATION_SECONDS = 300;
|
|
21
|
+
/** The config DSL's local key: lowercase, and the same regex the server validates. */
|
|
22
|
+
export declare const NAME_PATTERN: RegExp;
|
|
23
|
+
/** E.164, which is what a PHONE endpoint's value has to be. */
|
|
24
|
+
export declare const PHONE_PATTERN: RegExp;
|
|
25
|
+
export type EndpointDirection = 'INCOMING' | 'OUTGOING' | 'INCOMING_AND_OUTGOING';
|
|
26
|
+
export interface EndpointSetup {
|
|
27
|
+
value: string;
|
|
28
|
+
direction: EndpointDirection;
|
|
29
|
+
}
|
|
30
|
+
export interface AgentSetup {
|
|
31
|
+
name: string;
|
|
32
|
+
/** The agent's name in Roark when it is not slug-shaped. Set when adopting an existing agent. */
|
|
33
|
+
displayName?: string;
|
|
34
|
+
/**
|
|
35
|
+
* Endpoints to declare. Left out when adopting an agent whose endpoints Roark already holds:
|
|
36
|
+
* an absent `endpoints` key leaves them untouched, where an empty list would detach every one.
|
|
37
|
+
*/
|
|
38
|
+
endpoints?: readonly EndpointSetup[];
|
|
39
|
+
/** The line the run dials. Absent when the agent has nothing reachable yet. */
|
|
40
|
+
dial?: EndpointSetup | {
|
|
41
|
+
direction: EndpointDirection;
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
export interface PersonaSetup {
|
|
45
|
+
name: string;
|
|
46
|
+
language: string;
|
|
47
|
+
accent: string;
|
|
48
|
+
gender: string;
|
|
49
|
+
}
|
|
50
|
+
export interface FlowSetup {
|
|
51
|
+
name: string;
|
|
52
|
+
/** What the caller is calling about, in their own terms. */
|
|
53
|
+
goal: string;
|
|
54
|
+
/** One thing the agent has to do for the call to count as a pass. */
|
|
55
|
+
expectation?: string;
|
|
56
|
+
}
|
|
57
|
+
export interface CustomMetricSetup {
|
|
58
|
+
name: string;
|
|
59
|
+
/** A yes/no question about the call, which becomes the grading prompt. */
|
|
60
|
+
question: string;
|
|
61
|
+
}
|
|
62
|
+
export interface PlanSetup {
|
|
63
|
+
name: string;
|
|
64
|
+
maxDurationSeconds: number;
|
|
65
|
+
}
|
|
66
|
+
export interface Setup {
|
|
67
|
+
agent: AgentSetup;
|
|
68
|
+
persona: PersonaSetup;
|
|
69
|
+
flow: FlowSetup;
|
|
70
|
+
/** Metric slugs the run is graded by: Roark's own, plus the custom one when there is one. */
|
|
71
|
+
metrics: readonly string[];
|
|
72
|
+
customMetric?: CustomMetricSetup;
|
|
73
|
+
/** Absent when there is no endpoint to dial, so there is nothing a plan could run against. */
|
|
74
|
+
plan?: PlanSetup;
|
|
75
|
+
}
|
|
76
|
+
export interface ScaffoldFile {
|
|
77
|
+
/** Relative to the config directory, always posix-style. */
|
|
78
|
+
path: string;
|
|
79
|
+
contents: string;
|
|
80
|
+
}
|
|
81
|
+
/** Best-effort config name from free text, so a name can be suggested rather than demanded. */
|
|
82
|
+
export declare const slugify: (text: string) => string;
|
|
83
|
+
/** `refund_policy_accuracy` -> `Refund Policy Accuracy`, for a readable display name. */
|
|
84
|
+
export declare const titleize: (name: string) => string;
|
|
85
|
+
/** A run is outbound only when the line it dials is outgoing-only; everything else is inbound. */
|
|
86
|
+
export declare const directionFor: (endpointDirection: EndpointDirection | undefined) => "INBOUND" | "OUTBOUND";
|
|
87
|
+
/** Which serialization the config is written in. The resources are the same either way. */
|
|
88
|
+
export type ConfigFormat = 'yaml' | 'ts';
|
|
89
|
+
export interface ResourceEntry {
|
|
90
|
+
/** The config `kind`, which the TypeScript helpers supply themselves. */
|
|
91
|
+
kind: string;
|
|
92
|
+
/** Where the YAML form puts it, relative to the config directory. */
|
|
93
|
+
path: string;
|
|
94
|
+
resource: Record<string, unknown>;
|
|
95
|
+
/** Comments to write above it. A few differ by format, because the escape hatches do. */
|
|
96
|
+
notes: string[];
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Every resource this setup implies, in apply order (which is also a sensible reading order).
|
|
100
|
+
*
|
|
101
|
+
* Shared by both serializations, so the YAML and the TypeScript forms cannot describe different
|
|
102
|
+
* projects: they are the same objects, written two ways.
|
|
103
|
+
*/
|
|
104
|
+
export declare const resourcesFor: (setup: Setup, format?: ConfigFormat) => ResourceEntry[];
|
|
105
|
+
/** The YAML form: one file per resource, under the config directory. */
|
|
106
|
+
export declare const setupFiles: (setup: Setup) => ScaffoldFile[];
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The config-as-code files `get-started` writes.
|
|
4
|
+
*
|
|
5
|
+
* The wizard's output is a config directory, not a sequence of create calls. That is deliberate:
|
|
6
|
+
* the YAML is the thing we want someone to own and commit, `config apply` is idempotent so a
|
|
7
|
+
* second run is safe, and the whole flow then rides on one endpoint pair instead of six resource
|
|
8
|
+
* APIs. The wizard is disposable; the files are not.
|
|
9
|
+
*
|
|
10
|
+
* Everything here is pure - answers in, file contents out - so the shape of what we write is
|
|
11
|
+
* tested without a terminal or a network.
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.setupFiles = exports.resourcesFor = exports.directionFor = exports.titleize = exports.slugify = exports.PHONE_PATTERN = exports.NAME_PATTERN = exports.DEFAULT_MAX_DURATION_SECONDS = exports.DEFAULT_ENVIRONMENT = exports.SCHEMA_HEADER = void 0;
|
|
15
|
+
const yaml_1 = require("yaml");
|
|
16
|
+
/** Editor autocomplete and validation for anyone who opens these files afterwards. */
|
|
17
|
+
exports.SCHEMA_HEADER = '# yaml-language-server: $schema=https://roark.ai/roark-config.schema.json';
|
|
18
|
+
/**
|
|
19
|
+
* A Roark-managed environment that exists in every project, so a first run needs no environment
|
|
20
|
+
* of its own. Flows reference an environment by its display name.
|
|
21
|
+
*/
|
|
22
|
+
exports.DEFAULT_ENVIRONMENT = 'Quiet line';
|
|
23
|
+
/** How long a simulated call may run before Roark ends it. */
|
|
24
|
+
exports.DEFAULT_MAX_DURATION_SECONDS = 300;
|
|
25
|
+
/** The config DSL's local key: lowercase, and the same regex the server validates. */
|
|
26
|
+
exports.NAME_PATTERN = /^[a-z0-9][a-z0-9-_.]*$/;
|
|
27
|
+
/** E.164, which is what a PHONE endpoint's value has to be. */
|
|
28
|
+
exports.PHONE_PATTERN = /^\+[1-9]\d{6,14}$/;
|
|
29
|
+
/** Best-effort config name from free text, so a name can be suggested rather than demanded. */
|
|
30
|
+
const slugify = (text) => text
|
|
31
|
+
.toLowerCase()
|
|
32
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
33
|
+
.replace(/^-+|-+$/g, '')
|
|
34
|
+
.slice(0, 60);
|
|
35
|
+
exports.slugify = slugify;
|
|
36
|
+
/** `refund_policy_accuracy` -> `Refund Policy Accuracy`, for a readable display name. */
|
|
37
|
+
const titleize = (name) => name
|
|
38
|
+
.split(/[-_.]+/)
|
|
39
|
+
.filter((word) => word.length > 0)
|
|
40
|
+
.map((word) => `${word.charAt(0).toUpperCase()}${word.slice(1)}`)
|
|
41
|
+
.join(' ');
|
|
42
|
+
exports.titleize = titleize;
|
|
43
|
+
/** A run is outbound only when the line it dials is outgoing-only; everything else is inbound. */
|
|
44
|
+
const directionFor = (endpointDirection) => endpointDirection === 'OUTGOING' ? 'OUTBOUND' : 'INBOUND';
|
|
45
|
+
exports.directionFor = directionFor;
|
|
46
|
+
const document = (notes, resource) => [
|
|
47
|
+
exports.SCHEMA_HEADER,
|
|
48
|
+
...notes.map((note) => `# ${note}`),
|
|
49
|
+
'',
|
|
50
|
+
// No line wrapping: a folded prompt is valid YAML but miserable to edit.
|
|
51
|
+
(0, yaml_1.stringify)(resource, { lineWidth: 0 }),
|
|
52
|
+
].join('\n');
|
|
53
|
+
const agentEntry = (agent, format) => ({
|
|
54
|
+
kind: 'agent',
|
|
55
|
+
path: `agents/${agent.name}.yaml`,
|
|
56
|
+
notes: [
|
|
57
|
+
'The agent under test. Roark does not run it; this is how Roark reaches it.',
|
|
58
|
+
...(agent.endpoints === undefined ?
|
|
59
|
+
['Its endpoints live in Roark already, so they are not declared here.']
|
|
60
|
+
: format === 'ts' ?
|
|
61
|
+
["Add `prompt: readFileSync('./prompts/<name>.md', 'utf8')` to keep its system prompt here too."]
|
|
62
|
+
: ['Add `prompt: file://prompts/<name>.md` to keep its system prompt in git too.']),
|
|
63
|
+
],
|
|
64
|
+
resource: {
|
|
65
|
+
kind: 'agent',
|
|
66
|
+
name: agent.name,
|
|
67
|
+
...(agent.displayName === undefined ? {} : { displayName: agent.displayName }),
|
|
68
|
+
...(agent.endpoints === undefined ?
|
|
69
|
+
{}
|
|
70
|
+
: {
|
|
71
|
+
endpoints: agent.endpoints.map((endpoint) => ({
|
|
72
|
+
value: endpoint.value,
|
|
73
|
+
direction: endpoint.direction,
|
|
74
|
+
})),
|
|
75
|
+
}),
|
|
76
|
+
},
|
|
77
|
+
});
|
|
78
|
+
const personaEntry = (persona) => ({
|
|
79
|
+
kind: 'persona',
|
|
80
|
+
path: `personas/${persona.name}.yaml`,
|
|
81
|
+
notes: [
|
|
82
|
+
'The simulated caller. Everything else about them has a sensible default:',
|
|
83
|
+
'baseEmotion, speechPace, backgroundNoise, interruption and more.',
|
|
84
|
+
],
|
|
85
|
+
resource: {
|
|
86
|
+
kind: 'persona',
|
|
87
|
+
name: persona.name,
|
|
88
|
+
language: persona.language,
|
|
89
|
+
accent: persona.accent,
|
|
90
|
+
gender: persona.gender,
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
const flowEntry = (flow, agentName, personaName) => ({
|
|
94
|
+
kind: 'flow',
|
|
95
|
+
path: `flows/${flow.name}.yaml`,
|
|
96
|
+
notes: [
|
|
97
|
+
'What the call is about. `improv` means the caller is given a goal and talks freely;',
|
|
98
|
+
'a scripted flow is the other kind, for an exact turn-by-turn script.',
|
|
99
|
+
'Add `edgeCases` to test the same call going wrong in a specific way.',
|
|
100
|
+
],
|
|
101
|
+
resource: {
|
|
102
|
+
kind: 'flow',
|
|
103
|
+
type: 'improv',
|
|
104
|
+
name: flow.name,
|
|
105
|
+
agents: [agentName],
|
|
106
|
+
happyPath: {
|
|
107
|
+
persona: personaName,
|
|
108
|
+
environment: exports.DEFAULT_ENVIRONMENT,
|
|
109
|
+
prompt: flow.goal,
|
|
110
|
+
},
|
|
111
|
+
...(flow.expectation === undefined ? {} : { expectations: [flow.expectation] }),
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
const metricEntry = (metric) => ({
|
|
115
|
+
kind: 'metric',
|
|
116
|
+
path: `metrics/${metric.name}.yaml`,
|
|
117
|
+
notes: [
|
|
118
|
+
'A custom metric, graded by an LLM against the transcript.',
|
|
119
|
+
'`name` is the slug a plan references. `type` and `name` are fixed once created.',
|
|
120
|
+
],
|
|
121
|
+
resource: {
|
|
122
|
+
kind: 'metric',
|
|
123
|
+
name: metric.name,
|
|
124
|
+
displayName: (0, exports.titleize)(metric.name).slice(0, 100),
|
|
125
|
+
type: 'BOOLEAN',
|
|
126
|
+
// The grading prompt has to reference a transcript source and the world context, so the
|
|
127
|
+
// question is wrapped rather than used raw: a prompt that names neither is rejected.
|
|
128
|
+
prompt: `Using {{transcript}} and {{world_context}}, ${metric.question}`,
|
|
129
|
+
trueLabel: 'Yes',
|
|
130
|
+
falseLabel: 'No',
|
|
131
|
+
},
|
|
132
|
+
});
|
|
133
|
+
const planEntry = (setup, plan) => {
|
|
134
|
+
const dial = setup.agent.dial;
|
|
135
|
+
// The line is named only when it could be ambiguous. An agent whose config declares exactly one
|
|
136
|
+
// endpoint has nothing to disambiguate, and naming it there would mean the same number lived in
|
|
137
|
+
// two places that could drift apart. An adopted agent is the other case: Roark may hold several
|
|
138
|
+
// endpoints for it, and apply refuses to guess rather than dial the wrong one.
|
|
139
|
+
const declaresOneEndpoint = setup.agent.endpoints?.length === 1;
|
|
140
|
+
const value = !declaresOneEndpoint && dial !== undefined && 'value' in dial ? dial.value : undefined;
|
|
141
|
+
return {
|
|
142
|
+
kind: 'simulationPlan',
|
|
143
|
+
path: `simulation-plans/${plan.name}.yaml`,
|
|
144
|
+
notes: [
|
|
145
|
+
'The saved run: which line to call, which flows to run, and what grades them.',
|
|
146
|
+
'Applying this declares the plan. Starting a run is a separate, deliberate step.',
|
|
147
|
+
'Add `schedule` to run it on a cadence, or `iterations` to run each case more than once.',
|
|
148
|
+
],
|
|
149
|
+
resource: {
|
|
150
|
+
kind: 'simulationPlan',
|
|
151
|
+
name: plan.name,
|
|
152
|
+
direction: (0, exports.directionFor)(dial?.direction),
|
|
153
|
+
agentEndpoints: [
|
|
154
|
+
{
|
|
155
|
+
agent: setup.agent.name,
|
|
156
|
+
...(value === undefined ? {} : { value, direction: dial?.direction }),
|
|
157
|
+
},
|
|
158
|
+
],
|
|
159
|
+
metrics: [...setup.metrics],
|
|
160
|
+
flows: [{ flow: setup.flow.name }],
|
|
161
|
+
maxDurationSeconds: plan.maxDurationSeconds,
|
|
162
|
+
},
|
|
163
|
+
};
|
|
164
|
+
};
|
|
165
|
+
/**
|
|
166
|
+
* Every resource this setup implies, in apply order (which is also a sensible reading order).
|
|
167
|
+
*
|
|
168
|
+
* Shared by both serializations, so the YAML and the TypeScript forms cannot describe different
|
|
169
|
+
* projects: they are the same objects, written two ways.
|
|
170
|
+
*/
|
|
171
|
+
const resourcesFor = (setup, format = 'yaml') => [
|
|
172
|
+
...(setup.customMetric === undefined ? [] : [metricEntry(setup.customMetric)]),
|
|
173
|
+
personaEntry(setup.persona),
|
|
174
|
+
agentEntry(setup.agent, format),
|
|
175
|
+
flowEntry(setup.flow, setup.agent.name, setup.persona.name),
|
|
176
|
+
...(setup.plan === undefined ? [] : [planEntry(setup, setup.plan)]),
|
|
177
|
+
];
|
|
178
|
+
exports.resourcesFor = resourcesFor;
|
|
179
|
+
/** The YAML form: one file per resource, under the config directory. */
|
|
180
|
+
const setupFiles = (setup) => (0, exports.resourcesFor)(setup, 'yaml').map((entry) => ({
|
|
181
|
+
path: entry.path,
|
|
182
|
+
contents: document(entry.notes, entry.resource),
|
|
183
|
+
}));
|
|
184
|
+
exports.setupFiles = setupFiles;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Step 7: start the run and watch it to a verdict.
|
|
3
|
+
*
|
|
4
|
+
* Where the interesting failure modes live: a plan id that has to be resolved two ways, a
|
|
5
|
+
* billable action that needs its own confirmation, and a wait that can end three different ways.
|
|
6
|
+
*/
|
|
7
|
+
import type Roark from '@roarkanalytics/sdk';
|
|
8
|
+
import type { AgentSetup } from './resources';
|
|
9
|
+
import type { ConfigChange, ResolveOutput } from './types';
|
|
10
|
+
import type { Prompter } from './prompt';
|
|
11
|
+
interface RunPhase {
|
|
12
|
+
client: Roark;
|
|
13
|
+
prompter: Prompter;
|
|
14
|
+
say: (text: string) => void;
|
|
15
|
+
binaryName: string;
|
|
16
|
+
color: boolean;
|
|
17
|
+
quiet: boolean;
|
|
18
|
+
options: Record<string, unknown>;
|
|
19
|
+
planName: string;
|
|
20
|
+
/** What to call the config in the closing "what next" block. */
|
|
21
|
+
dir: string;
|
|
22
|
+
/** The apply command for that config, which differs between YAML and a config module. */
|
|
23
|
+
apply: string;
|
|
24
|
+
/** The line the run will dial, quoted back in the confirmation. */
|
|
25
|
+
dial: AgentSetup['dial'];
|
|
26
|
+
/** The apply result, when there was one. Absent when the project already matched the files. */
|
|
27
|
+
applied: ConfigChange[] | undefined;
|
|
28
|
+
resolveOutput: ResolveOutput;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Step 7: start the run and watch it to a verdict.
|
|
32
|
+
*
|
|
33
|
+
* Its own function because it is the one phase that stands alone (everything it needs is a plan
|
|
34
|
+
* id and a line), and because it is where the interesting failure modes live: a plan id that has
|
|
35
|
+
* to be resolved two ways, a billable action that needs its own confirmation, and a wait that can
|
|
36
|
+
* end three different ways.
|
|
37
|
+
*
|
|
38
|
+
* Returns true when it has reported the outcome and the command is finished, false when it could
|
|
39
|
+
* not start and the caller should print its own closing text.
|
|
40
|
+
*/
|
|
41
|
+
export declare const startRun: (phase: RunPhase) => Promise<boolean>;
|
|
42
|
+
export {};
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Step 7: start the run and watch it to a verdict.
|
|
4
|
+
*
|
|
5
|
+
* Where the interesting failure modes live: a plan id that has to be resolved two ways, a
|
|
6
|
+
* billable action that needs its own confirmation, and a wait that can end three different ways.
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.startRun = void 0;
|
|
10
|
+
const output_1 = require("../output");
|
|
11
|
+
const ladder_1 = require("./ladder");
|
|
12
|
+
const watch_1 = require("./watch");
|
|
13
|
+
/** How long to wait for the first run, and how often to ask. A simulated call takes a minute or two. */
|
|
14
|
+
const WATCH_TIMEOUT_MS = 6 * 60_000;
|
|
15
|
+
const POLL_MS = 5_000;
|
|
16
|
+
/**
|
|
17
|
+
* Step 7: start the run and watch it to a verdict.
|
|
18
|
+
*
|
|
19
|
+
* Its own function because it is the one phase that stands alone (everything it needs is a plan
|
|
20
|
+
* id and a line), and because it is where the interesting failure modes live: a plan id that has
|
|
21
|
+
* to be resolved two ways, a billable action that needs its own confirmation, and a wait that can
|
|
22
|
+
* end three different ways.
|
|
23
|
+
*
|
|
24
|
+
* Returns true when it has reported the outcome and the command is finished, false when it could
|
|
25
|
+
* not start and the caller should print its own closing text.
|
|
26
|
+
*/
|
|
27
|
+
const startRun = async (phase) => {
|
|
28
|
+
const { client, prompter, say, binaryName, color, quiet, options, planName } = phase;
|
|
29
|
+
say((0, ladder_1.stepHeader)(6, color));
|
|
30
|
+
// Two ways to the id, because there are two ways to get here: an apply that just created the
|
|
31
|
+
// plan carries it, and a re-run over an already-applied directory has no apply result at all.
|
|
32
|
+
const planId = phase.applied?.find((entry) => entry.configKey === `simulationPlan/${planName}`)?.id ??
|
|
33
|
+
(await findPlanId(client, planName));
|
|
34
|
+
if (planId === undefined) {
|
|
35
|
+
say((0, ladder_1.note)(`Could not work out the plan's id. Find it with \`${binaryName} simulation plan list\` and run \`${binaryName} simulation run --data '{"planId":"<id>"}'\`.`, color, 'yellow'));
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
// A run places a real call on a real line and is billed, so it is confirmed on its own rather
|
|
39
|
+
// than riding on the apply. `--yes` covers both, `--no-run` skips it.
|
|
40
|
+
const dialled = phase.dial !== undefined && 'value' in phase.dial ? phase.dial.value : 'your agent';
|
|
41
|
+
const start = options['yes'] === true ||
|
|
42
|
+
(await prompter.confirm(`Start the run? It places a real call to ${dialled}.`, true));
|
|
43
|
+
if (!start) {
|
|
44
|
+
say((0, ladder_1.stepSkipped)(6, 'not started', color));
|
|
45
|
+
say('');
|
|
46
|
+
say(`The plan is saved. Start it whenever you like: \`${binaryName} simulation run --data '{"planId":"${planId}"}'\`.`);
|
|
47
|
+
if (!quiet)
|
|
48
|
+
(0, output_1.write)(planId);
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
const run = await client.simulation.run({ planId });
|
|
52
|
+
const jobId = run.data.simulationRunPlanJobId;
|
|
53
|
+
say((0, ladder_1.echoCommand)(`${binaryName} simulation run --data '{"planId":"${planId}"}'`, color));
|
|
54
|
+
say(`Started ${(0, output_1.paint)(jobId, 'cyan', color)}: ${run.data.simulationJobCount} call(s). Waiting for the verdict...`);
|
|
55
|
+
const watched = await (0, watch_1.watchJob)({
|
|
56
|
+
jobId,
|
|
57
|
+
getJob: async (id) => (await client.simulationRunPlanJob.getByID(id)).data,
|
|
58
|
+
onProgress: (line) => say(` ${(0, output_1.paint)(line, 'dim', color)}`),
|
|
59
|
+
timeoutMs: WATCH_TIMEOUT_MS,
|
|
60
|
+
pollMs: POLL_MS,
|
|
61
|
+
sleep: watch_1.sleep,
|
|
62
|
+
now: () => Date.now(),
|
|
63
|
+
});
|
|
64
|
+
if (watched.outcome === 'settled') {
|
|
65
|
+
say((0, ladder_1.stepDone)(6, (0, output_1.paint)((0, watch_1.verdictLine)(watched.snapshot), (0, watch_1.verdictStyle)(watched.snapshot), color), color));
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
// Neither of these is a failed run: the run is fine, it is this command that stopped watching,
|
|
69
|
+
// so it is reported as a skipped step rather than as a verdict.
|
|
70
|
+
say((0, ladder_1.stepSkipped)(6, watched.outcome === 'unreachable' ?
|
|
71
|
+
'could not read the run from the API. It is still going'
|
|
72
|
+
: `still running after ${Math.round(WATCH_TIMEOUT_MS / 60_000)} minutes. It will finish on its own`, color));
|
|
73
|
+
}
|
|
74
|
+
say((0, ladder_1.nextSteps)({
|
|
75
|
+
binaryName,
|
|
76
|
+
dir: phase.dir,
|
|
77
|
+
apply: phase.apply,
|
|
78
|
+
lines: [
|
|
79
|
+
`${(0, output_1.paint)(`${binaryName} simulation plan job get ${jobId}`, 'cyan', color)} every call, with its metrics`,
|
|
80
|
+
`${(0, output_1.paint)(`${binaryName} simulation run --data '{"planId":"${planId}"}'`, 'cyan', color)} run it again`,
|
|
81
|
+
],
|
|
82
|
+
}, color));
|
|
83
|
+
if (!quiet) {
|
|
84
|
+
const output = phase.resolveOutput(options);
|
|
85
|
+
(0, output_1.write)(output.format === 'json' || options['json'] === true ?
|
|
86
|
+
(0, output_1.render)({ planId, jobId, status: watched.snapshot.status }, output)
|
|
87
|
+
: jobId);
|
|
88
|
+
}
|
|
89
|
+
return true;
|
|
90
|
+
};
|
|
91
|
+
exports.startRun = startRun;
|
|
92
|
+
/** The plan's id when the apply result did not carry one, e.g. because nothing about it changed. */
|
|
93
|
+
const findPlanId = async (client, name) => {
|
|
94
|
+
try {
|
|
95
|
+
const response = await client.simulationRunPlan.list({ searchText: name, limit: 50 });
|
|
96
|
+
return response.data.find((plan) => plan.name === name)?.id;
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
};
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `--scaffold-only`: write the config with placeholders and stop.
|
|
3
|
+
*
|
|
4
|
+
* A complete alternative mode rather than a step of the guided flow: it asks nothing, calls no
|
|
5
|
+
* endpoint, and ends with a directory someone fills in by hand. The path for a script, a
|
|
6
|
+
* dotfiles repo, or anyone who would rather read the config format than answer questions about
|
|
7
|
+
* it, which is why it has to work with no terminal and no credential.
|
|
8
|
+
*/
|
|
9
|
+
import { type Target } from './target';
|
|
10
|
+
export interface ScaffoldOnlyRun {
|
|
11
|
+
binaryName: string;
|
|
12
|
+
target: Target;
|
|
13
|
+
color: boolean;
|
|
14
|
+
quiet: boolean;
|
|
15
|
+
say: (text: string) => void;
|
|
16
|
+
/** Said here too: a config module nobody can import is worth flagging before it is written. */
|
|
17
|
+
warnIfUnimportable: () => void;
|
|
18
|
+
}
|
|
19
|
+
export declare const scaffoldOnlyRun: (run: ScaffoldOnlyRun) => Promise<void>;
|