@roarkanalytics/cli 0.5.0 → 0.7.0

package/commands.js

@@ -13,6 +13,7 @@ exports.GROUPS = {
     call: 'Manage calls',
     'call metric': 'Read call metrics',
     'call sentiment-run': 'Read call sentiment runs',
+    'call tool-invocation': 'Manage call tool invocations',
     'call transcript': 'Read call transcript',
     config: 'Read configs',
     'customer-flow': 'Manage customer flows',
@@ -24,12 +25,14 @@ exports.GROUPS = {
     'metric collection-job': 'Manage metric collection jobs',
     'metric definition': 'Manage metric definitions',
     'metric policy': 'Manage metric policies',
+    'metric variant': 'Manage metric variants',
     simulation: 'Manage simulations',
     'simulation environment': 'Read simulation environments',
     'simulation job': 'Read simulation jobs',
     'simulation persona': 'Manage simulation personas',
     'simulation plan': 'Manage simulation plans',
     'simulation plan job': 'Manage simulation plan jobs',
+    'simulation template': 'Read simulation templates',
     webhook: 'Manage webhooks',
 };
 exports.COMMANDS = [
@@ -777,6 +780,47 @@ exports.COMMANDS = [
         acceptsBody: false,
         requiresAuth: true,
     },
+    {
+        commandPath: ['call', 'tool-invocation', 'append'],
+        clientProperty: 'call',
+        methodName: 'appendToolInvocations',
+        httpMethod: 'post',
+        httpPath: '/v1/call/{callId}/tool-invocations',
+        summary: 'Append tool invocations to a call',
+        description: 'Attach tool invocations that fired during a call to an already-existing call, asynchronously after the call was created. Use this when the tool-call data becomes available later than the call itself (e.g. a Roark simulation, or a call submitted via POST /v1/call before its tools were ready). Writes are idempotent: re-sending an invocation already present on the call (same tool name and timing) is skipped, so retries converge instead of double-counting. Optionally pass `metrics` to (re)score metrics over the newly attached tools, which triggers a billed metric collection job and requires the `metric:create` permission.',
+        positionals: [
+            {
+                name: 'call-id',
+                paramKey: 'callId',
+            },
+        ],
+        flags: [
+            {
+                name: 'tool-invocations',
+                path: ['toolInvocations'],
+                location: 'body',
+                required: true,
+                description: 'Tool invocations that fired during the call, to attach to it. Max 500 per request. Re-sending an invocation already present on the call (same tool name and timing) is skipped, so retries are safe.',
+                valueKind: 'array',
+                repeatable: false,
+            },
+            {
+                name: 'metrics',
+                path: ['metrics'],
+                location: 'body',
+                required: false,
+                description: "Optional. Metric definitions to (re)score on this call after the tool invocations are attached, e.g. the Tool Invocation Analysis metrics. Triggers a metric collection job (billed, credit-gated) and requires the 'metric:create' permission. Omit to attach tools without scoring. Max 20.",
+                valueKind: 'array',
+                repeatable: false,
+            },
+        ],
+        hasParams: true,
+        paramsAllOptional: false,
+        bodyOpaque: false,
+        bodyVariants: [],
+        acceptsBody: true,
+        requiresAuth: true,
+    },
     {
         commandPath: ['call', 'transcript', 'get'],
         clientProperty: 'call',
@@ -1804,6 +1848,245 @@ exports.COMMANDS = [
         acceptsBody: false,
         requiresAuth: true,
     },
+    {
+        commandPath: ['metric', 'definition', 'update'],
+        clientProperty: 'metric',
+        methodName: 'updateDefinition',
+        httpMethod: 'put',
+        httpPath: '/v1/metric/definitions/{idOrSlug}',
+        summary: 'Update a metric definition',
+        description: "Update the editable subset of a custom metric definition, addressed by its UUID or its stable `slug`. Only the supplied fields are changed; omitted fields are left unchanged. Every update creates a new immutable version; the response carries the advanced `versionId`. Immutable fields (scope, outputType, calcType, …) are rejected, and which fields are editable depends on the metric (e.g. derived metrics only allow `name`). Roark's own metrics are rejected here: this endpoint edits the shared definition, which every workspace sees. To change one for your workspace alone, edit its variant with PUT /v1/metric/definitions/{idOrSlug}/variants/{variantId}, which forks it for you and leaves every other workspace on the original.",
+        positionals: [
+            {
+                name: 'id-or-slug',
+                paramKey: 'idOrSlug',
+                description: "The metric definition's Roark ID (UUID) or its stable `slug` (e.g. `customer_satisfaction`).",
+            },
+        ],
+        flags: [
+            {
+                name: 'name',
+                path: ['name'],
+                location: 'body',
+                required: false,
+                description: 'New name (only for metrics whose name is editable)',
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'llm-prompt',
+                path: ['llmPrompt'],
+                location: 'body',
+                required: false,
+                description: 'New LLM prompt (only for LLM_JUDGE metrics whose prompt is editable)',
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'scale-min',
+                path: ['scaleMin'],
+                location: 'body',
+                required: false,
+                description: 'New scale minimum (SCALE output only)',
+                valueKind: 'integer',
+                repeatable: false,
+            },
+            {
+                name: 'scale-max',
+                path: ['scaleMax'],
+                location: 'body',
+                required: false,
+                description: 'New scale maximum (SCALE output only)',
+                valueKind: 'integer',
+                repeatable: false,
+            },
+            {
+                name: 'scale-labels',
+                path: ['scaleLabels'],
+                location: 'body',
+                required: false,
+                description: 'Replacement set of scale-range labels (SCALE output only)',
+                valueKind: 'array',
+                repeatable: false,
+            },
+            {
+                name: 'boolean-true-label',
+                path: ['booleanTrueLabel'],
+                location: 'body',
+                required: false,
+                description: 'New label for the true case (BOOLEAN output only)',
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'boolean-false-label',
+                path: ['booleanFalseLabel'],
+                location: 'body',
+                required: false,
+                description: 'New label for the false case (BOOLEAN output only)',
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'classification-options',
+                path: ['classificationOptions'],
+                location: 'body',
+                required: false,
+                description: 'Replacement set of classification options (CLASSIFICATION output only)',
+                valueKind: 'array',
+                repeatable: false,
+            },
+            {
+                name: 'max-classifications',
+                path: ['maxClassifications'],
+                location: 'body',
+                required: false,
+                description: 'New maximum number of classifications (CLASSIFICATION output only)',
+                valueKind: 'integer',
+                repeatable: false,
+            },
+            {
+                name: 'tool-definition-ids',
+                path: ['toolDefinitionIds'],
+                location: 'body',
+                required: false,
+                description: 'Replacement set of scoped tool-definition ids (only for metrics whose tool scoping is editable)',
+                valueKind: 'array',
+                repeatable: true,
+            },
+            {
+                name: 'supported-contexts',
+                path: ['supportedContexts'],
+                location: 'body',
+                required: false,
+                description: 'Replacement set of supported contexts. Omit to leave unchanged.',
+                valueKind: 'array',
+                enumValues: ['CALL', 'SEGMENT', 'TURN'],
+                repeatable: true,
+            },
+            {
+                name: 'formula',
+                path: ['formula'],
+                location: 'body',
+                required: false,
+                description: 'New formula expression (FORMULA only). Pass `sources` alongside if the referenced metrics change.',
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'sources',
+                path: ['sources'],
+                location: 'body',
+                required: false,
+                description: 'Replacement formula sources, required when `formula` changes the referenced metrics (FORMULA only).',
+                valueKind: 'array',
+                repeatable: false,
+            },
+            {
+                name: 'change-reason',
+                path: ['changeReason'],
+                location: 'body',
+                required: false,
+                description: 'Optional free-text audit note recorded on the new version.',
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'slug',
+                path: ['slug'],
+                location: 'body',
+                required: false,
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'metric-id',
+                path: ['metricId'],
+                location: 'body',
+                required: false,
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'source',
+                path: ['source'],
+                location: 'body',
+                required: false,
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'organization-id',
+                path: ['organizationId'],
+                location: 'body',
+                required: false,
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'project-id',
+                path: ['projectId'],
+                location: 'body',
+                required: false,
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'output-type',
+                path: ['outputType'],
+                location: 'body',
+                required: false,
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'scope',
+                path: ['scope'],
+                location: 'body',
+                required: false,
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'participant-role',
+                path: ['participantRole'],
+                location: 'body',
+                required: false,
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'calc-type',
+                path: ['calcType'],
+                location: 'body',
+                required: false,
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'supports-multiple-variants',
+                path: ['supportsMultipleVariants'],
+                location: 'body',
+                required: false,
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'analysis-package-id',
+                path: ['analysisPackageId'],
+                location: 'body',
+                required: false,
+                valueKind: 'string',
+                repeatable: false,
+            },
+        ],
+        hasParams: true,
+        paramsAllOptional: true,
+        bodyOpaque: false,
+        bodyVariants: [],
+        acceptsBody: true,
+        requiresAuth: true,
+    },
     {
         commandPath: ['metric', 'policy', 'create'],
         clientProperty: 'metricPolicy',
@@ -2022,6 +2305,219 @@ exports.COMMANDS = [
         acceptsBody: true,
         requiresAuth: true,
     },
+    {
+        commandPath: ['metric', 'variant', 'create'],
+        clientProperty: 'metricVariant',
+        methodName: 'create',
+        httpMethod: 'post',
+        httpPath: '/v1/metric/definitions/{idOrSlug}/variants',
+        summary: 'Create a metric variant',
+        description: 'Add a configuration of this metric for your organization, seeded from its Default. Edit it with PUT to change what it measures, then pin it where you want it used. Threshold metrics have no variants: their configuration comes from the metric they derive from. Metrics in a package that manages its own variants reject this too.',
+        positionals: [
+            {
+                name: 'id-or-slug',
+                paramKey: 'idOrSlug',
+                description: 'Metric definition UUID or its stable slug.',
+            },
+        ],
+        flags: [
+            {
+                name: 'name',
+                path: ['name'],
+                location: 'body',
+                required: true,
+                description: 'Name for the new variant. Must be unique for this metric within your organization and cannot be `Default`.',
+                valueKind: 'string',
+                repeatable: false,
+            },
+        ],
+        hasParams: true,
+        paramsAllOptional: false,
+        bodyOpaque: false,
+        bodyVariants: [],
+        acceptsBody: true,
+        requiresAuth: true,
+    },
+    {
+        commandPath: ['metric', 'variant', 'delete'],
+        clientProperty: 'metricVariant',
+        methodName: 'delete',
+        httpMethod: 'delete',
+        httpPath: '/v1/metric/definitions/{idOrSlug}/variants/{variantId}',
+        summary: 'Delete a metric variant',
+        description: 'Remove one of your organization’s variants. Anything pinned to it falls back to the Default, so deleting a fork of a Roark variant returns you to Roark’s configuration. Roark’s own variants cannot be deleted, and neither can a Default. Values already collected under the deleted variant are retained.',
+        positionals: [
+            {
+                name: 'id-or-slug',
+                paramKey: 'idOrSlug',
+                description: 'Metric definition UUID or its stable slug.',
+            },
+            {
+                name: 'variant-id',
+                paramKey: 'variantId',
+                description: 'Variant UUID.',
+            },
+        ],
+        flags: [],
+        hasParams: true,
+        paramsAllOptional: false,
+        bodyOpaque: false,
+        bodyVariants: [],
+        acceptsBody: false,
+        requiresAuth: true,
+    },
+    {
+        commandPath: ['metric', 'variant', 'get'],
+        clientProperty: 'metricVariant',
+        methodName: 'getByID',
+        httpMethod: 'get',
+        httpPath: '/v1/metric/definitions/{idOrSlug}/variants/{variantId}',
+        summary: 'Get a metric variant',
+        description: 'One configuration of this metric, by id.',
+        positionals: [
+            {
+                name: 'id-or-slug',
+                paramKey: 'idOrSlug',
+                description: 'Metric definition UUID or its stable slug.',
+            },
+            {
+                name: 'variant-id',
+                paramKey: 'variantId',
+                description: 'Variant UUID.',
+            },
+        ],
+        flags: [],
+        hasParams: true,
+        paramsAllOptional: false,
+        bodyOpaque: false,
+        bodyVariants: [],
+        acceptsBody: false,
+        requiresAuth: true,
+    },
+    {
+        commandPath: ['metric', 'variant', 'list'],
+        clientProperty: 'metricVariant',
+        methodName: 'list',
+        httpMethod: 'get',
+        httpPath: '/v1/metric/definitions/{idOrSlug}/variants',
+        summary: 'List a metric’s variants',
+        description: 'Every configuration of this metric your organization can use: Roark’s own variants and any your organization has added or forked. `isDefault` marks the one the metric is scored with when nothing pins another; pass any variant’s `id` as `sourceVariantId` to pin it on a derived metric. Auto-managed variants (the ones a package materializes for you) are not listed: they are engine state, not configuration you author.',
+        positionals: [
+            {
+                name: 'id-or-slug',
+                paramKey: 'idOrSlug',
+                description: 'Metric definition UUID or its stable slug.',
+            },
+        ],
+        flags: [],
+        hasParams: false,
+        paramsAllOptional: true,
+        bodyOpaque: false,
+        bodyVariants: [],
+        acceptsBody: false,
+        requiresAuth: true,
+    },
+    {
+        commandPath: ['metric', 'variant', 'update'],
+        clientProperty: 'metricVariant',
+        methodName: 'update',
+        httpMethod: 'put',
+        httpPath: '/v1/metric/definitions/{idOrSlug}/variants/{variantId}',
+        summary: 'Update a metric variant',
+        description: 'Rename a variant, change its configuration, or both. Every configuration change creates a new immutable version and advances `versionId`; the response carries the advanced value. **Editing one of Roark’s own variants forks it for your organization.** The response carries the new variant’s `id`, which will differ from the one in the path, and `isSystem` becomes false. Roark’s variant is untouched and other organizations keep it. DELETE your fork to go back to it. Which fields are editable depends on the metric: some Roark metrics lock their prompt or output configuration, and a locked field is rejected rather than ignored.',
+        positionals: [
+            {
+                name: 'id-or-slug',
+                paramKey: 'idOrSlug',
+                description: 'Metric definition UUID or its stable slug.',
+            },
+            {
+                name: 'variant-id',
+                paramKey: 'variantId',
+                description: 'Variant UUID.',
+            },
+        ],
+        flags: [
+            {
+                name: 'name',
+                path: ['name'],
+                location: 'body',
+                required: false,
+                description: 'Rename the variant. Does not change its configuration, so `versionId` is unaffected. `Default` is reserved.',
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'llm-prompt',
+                path: ['llmPrompt'],
+                location: 'body',
+                required: false,
+                description: 'The rubric this variant applies. LLM judge metrics only.',
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'scale-min',
+                path: ['scaleMin'],
+                location: 'body',
+                required: false,
+                description: 'Scale minimum. SCALE output only.',
+                valueKind: 'integer',
+                repeatable: false,
+            },
+            {
+                name: 'scale-max',
+                path: ['scaleMax'],
+                location: 'body',
+                required: false,
+                description: 'Scale maximum. SCALE output only.',
+                valueKind: 'integer',
+                repeatable: false,
+            },
+            {
+                name: 'boolean-true-label',
+                path: ['booleanTrueLabel'],
+                location: 'body',
+                required: false,
+                description: 'What a `true` value means. Given to the judge as its polarity rule, so keep it accurate.',
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'boolean-false-label',
+                path: ['booleanFalseLabel'],
+                location: 'body',
+                required: false,
+                description: 'What a `false` value means. Given to the judge as its polarity rule, so keep it accurate.',
+                valueKind: 'string',
+                repeatable: false,
+            },
+            {
+                name: 'max-classifications',
+                path: ['maxClassifications'],
+                location: 'body',
+                required: false,
+                description: 'Maximum classifications returned. CLASSIFICATION output only.',
+                valueKind: 'integer',
+                repeatable: false,
+            },
+            {
+                name: 'change-reason',
+                path: ['changeReason'],
+                location: 'body',
+                required: false,
+                description: 'Free-text audit note recorded on the new version.',
+                valueKind: 'string',
+                repeatable: false,
+            },
+        ],
+        hasParams: true,
+        paramsAllOptional: false,
+        bodyOpaque: false,
+        bodyVariants: [],
+        acceptsBody: true,
+        requiresAuth: true,
+    },
     {
         commandPath: ['simulation', 'environment', 'get'],
         clientProperty: 'simulationEnvironment',
@@ -2975,6 +3471,15 @@ exports.COMMANDS = [
                 valueKind: 'array',
                 repeatable: false,
             },
+            {
+                name: 'include-flow-metrics',
+                path: ['includeFlowMetrics'],
+                location: 'body',
+                required: false,
+                description: "Also collect each attached flow's own metrics, on top of the `metrics` named here. Default true, which is what you want when you brought your own flows and their graders. Set false for a run whose metric list is meant to be exhaustive: a template like Load Testing or Voicemail deliberately grades a narrow set, and inheriting every flow metric on top multiplies analysis cost across the volume without adding signal. GET /v1/simulation/template returns the value each template expects.",
+                valueKind: 'boolean',
+                repeatable: false,
+            },
             {
                 name: 'enrich-with-live-conversation',
                 path: ['enrichWithLiveConversation'],
@@ -3414,6 +3919,15 @@ exports.COMMANDS = [
                 valueKind: 'boolean',
                 repeatable: false,
             },
+            {
+                name: 'include-flow-metrics',
+                path: ['includeFlowMetrics'],
+                location: 'body',
+                required: false,
+                description: "Whether to also collect each attached flow's own metrics, on top of this plan's list.",
+                valueKind: 'boolean',
+                repeatable: false,
+            },
         ],
         hasParams: true,
         paramsAllOptional: true,
@@ -3429,7 +3943,7 @@ exports.COMMANDS = [
         httpMethod: 'post',
         httpPath: '/v1/simulation/run',
         summary: 'Run a simulation',
-        description: 'Starts a simulation and returns the run. Send `plan` to describe a simulation and run it once. Add `saveAsPlan` to keep that configuration as a reusable run plan. Send `planId` instead to run a plan you already have.',
+        description: 'Starts a simulation and returns the run. Send `template` to run one of the built-in templates: it supplies the metrics and checks, and for some templates the flows too, so the request only names the agent and the direction. Send `plan` to describe a simulation yourself and run it once. Send `planId` to run a plan you already have. `template` and `plan` both resolve to a run plan, returned as `simulationRunPlanId`. Add `saveAsPlan` to keep it, or read it back to see exactly what ran. A plan built from a template is a snapshot: retuning the template later never changes what that plan runs, which is what makes a saved one safe to pin in CI.',
         positionals: [],
         flags: [],
         hasParams: true,
@@ -3444,10 +3958,31 @@ exports.COMMANDS = [
                 name: 'RunSimulationFromPlanId',
                 required: ['planId'],
             },
+            {
+                name: 'RunSimulationFromTemplate',
+                required: ['template', 'agentEndpoints', 'direction'],
+            },
         ],
         acceptsBody: true,
         requiresAuth: true,
     },
+    {
+        commandPath: ['simulation', 'template', 'list'],
+        clientProperty: 'simulationTemplate',
+        methodName: 'list',
+        httpMethod: 'get',
+        httpPath: '/v1/simulation/template',
+        summary: 'List simulation templates',
+        description: 'Returns the built-in simulation templates, each resolved against this project: the metric and check definitions it collects, and the flows it runs with the variants it covers. A template is a preset rather than a stored object, so building a run from one produces an ordinary run plan you own and can edit afterwards. Pass a `slug` as `template` to POST /v1/simulation/run. Every entry is complete, so there is no per-template endpoint to follow up with, and the list is a fixed catalogue rather than a paginated one. Only templates a request can actually run are listed.',
+        positionals: [],
+        flags: [],
+        hasParams: false,
+        paramsAllOptional: true,
+        bodyOpaque: false,
+        bodyVariants: [],
+        acceptsBody: false,
+        requiresAuth: true,
+    },
     {
         commandPath: ['webhook', 'create'],
         clientProperty: 'webhook',