@soat/cli 0.66.0 → 0.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.mjs +15 -3
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -13,7 +13,7 @@ import * as os from "node:os";
|
|
|
13
13
|
import { createHmac, timingSafeEqual } from "node:crypto";
|
|
14
14
|
|
|
15
15
|
//#region package.json
|
|
16
|
-
var version = "0.
|
|
16
|
+
var version = "0.68.0";
|
|
17
17
|
|
|
18
18
|
//#endregion
|
|
19
19
|
//#region src/cli-wrappers/wrappers/formations.ts
|
|
@@ -4316,7 +4316,7 @@ var routes = {
|
|
|
4316
4316
|
"in": "body"
|
|
4317
4317
|
}, {
|
|
4318
4318
|
"name": "scorers",
|
|
4319
|
-
"description": "Scorer configs, a discriminated union on `type`. Each type may appear at most once. Every scorer produces `{ score: 0–1, passed: boolean }`; binary scorers emit 0 or 1.\n\n`exact_match` compares the trimmed output text to `expected_output`. `contains` looks for `value` in the output text. `json_logic` evaluates `expression` over `{ input, output, object, expected, item.metadata }`, where `object` is the structured output (absent when the agent has no `output_schema`). `output_schema` validates the structured output against the scorer's own `schema`, falling back to the agent's; it requires the agent to carry an `output_schema`, because without one the platform emits no structured output and every item would score 0.\n\n`llm_judge` grades the output with a model completion, returning a continuous score plus its `reasoning`. Its `pass_threshold` is required: a continuous score says nothing about where \"good enough\" is, and a defaulted cutoff would silently decide the gate.\n\n`embedding_similarity` embeds the output text and `expected_output` with the platform's configured embedding model (`EMBEDDING_PROVIDER` / `EMBEDDING_MODEL` — the same stack document ingestion uses) and scores their cosine similarity, clamped to 0-1. Its `pass_threshold` is required for the same reason as the judge's. An item without an `expected_output` scores 0; an embedding backend failure marks the **item** errored, never a score of 0.\n\n`tool` runs a custom scoring algorithm: a server-callable project tool the engine invokes once per item with the item's context. Unlike the built-in types it may appear several times, each under a distinct `name` — outcomes and aggregates key on the name.",
|
|
4319
|
+
"description": "Scorer configs, a discriminated union on `type`. Each type may appear at most once. Every scorer produces `{ score: 0–1, passed: boolean }`; binary scorers emit 0 or 1.\n\n`exact_match` compares the trimmed output text to `expected_output`. `contains` looks for `value` in the output text. `json_logic` evaluates `expression` over `{ input, output, object, expected, item.metadata }`, where `object` is the structured output (absent when the agent has no `output_schema`). `output_schema` validates the structured output against the scorer's own `schema`, falling back to the agent's; it requires the agent to carry an `output_schema`, because without one the platform emits no structured output and every item would score 0.\n\n`llm_judge` grades the output with a model completion, returning a continuous score plus its `reasoning`. Its `pass_threshold` is required: a continuous score says nothing about where \"good enough\" is, and a defaulted cutoff would silently decide the gate.\n\n`embedding_similarity` embeds the output text and `expected_output` with the platform's configured embedding model (`EMBEDDING_PROVIDER` / `EMBEDDING_MODEL` — the same stack document ingestion uses) and scores their cosine similarity, clamped to 0-1. Its `pass_threshold` is required for the same reason as the judge's. An item without an `expected_output` scores 0; an embedding backend failure marks the **item** errored, never a score of 0.\n\n`tool` runs a custom scoring algorithm: a server-callable project tool the engine invokes once per item with the item's context. Unlike the built-in types it may appear several times, each under a distinct `name` — outcomes and aggregates key on the name.\n\n`decider` grades each item with a decision of a project decider: its `score` expression reads the decision's `answers`. Like `tool`, it keys on its `name` and may appear several times.",
|
|
4320
4320
|
"required": true,
|
|
4321
4321
|
"type": "array",
|
|
4322
4322
|
"in": "body"
|
|
@@ -4326,6 +4326,12 @@ var routes = {
|
|
|
4326
4326
|
"required": false,
|
|
4327
4327
|
"type": "number",
|
|
4328
4328
|
"in": "body"
|
|
4329
|
+
}, {
|
|
4330
|
+
"name": "group_by",
|
|
4331
|
+
"description": "A key of the items' `metadata`. A run rolls its scores up per string value of that key in `aggregate_scores.grouping`. Null reports no grouping.",
|
|
4332
|
+
"required": false,
|
|
4333
|
+
"type": "string",
|
|
4334
|
+
"in": "body"
|
|
4329
4335
|
}]
|
|
4330
4336
|
},
|
|
4331
4337
|
"get-eval": {
|
|
@@ -4382,7 +4388,7 @@ var routes = {
|
|
|
4382
4388
|
"in": "body"
|
|
4383
4389
|
}, {
|
|
4384
4390
|
"name": "scorers",
|
|
4385
|
-
"description": "Scorer configs, a discriminated union on `type`. Each type may appear at most once. Every scorer produces `{ score: 0–1, passed: boolean }`; binary scorers emit 0 or 1.\n\n`exact_match` compares the trimmed output text to `expected_output`. `contains` looks for `value` in the output text. `json_logic` evaluates `expression` over `{ input, output, object, expected, item.metadata }`, where `object` is the structured output (absent when the agent has no `output_schema`). `output_schema` validates the structured output against the scorer's own `schema`, falling back to the agent's; it requires the agent to carry an `output_schema`, because without one the platform emits no structured output and every item would score 0.\n\n`llm_judge` grades the output with a model completion, returning a continuous score plus its `reasoning`. Its `pass_threshold` is required: a continuous score says nothing about where \"good enough\" is, and a defaulted cutoff would silently decide the gate.\n\n`embedding_similarity` embeds the output text and `expected_output` with the platform's configured embedding model (`EMBEDDING_PROVIDER` / `EMBEDDING_MODEL` — the same stack document ingestion uses) and scores their cosine similarity, clamped to 0-1. Its `pass_threshold` is required for the same reason as the judge's. An item without an `expected_output` scores 0; an embedding backend failure marks the **item** errored, never a score of 0.\n\n`tool` runs a custom scoring algorithm: a server-callable project tool the engine invokes once per item with the item's context. Unlike the built-in types it may appear several times, each under a distinct `name` — outcomes and aggregates key on the name.",
|
|
4391
|
+
"description": "Scorer configs, a discriminated union on `type`. Each type may appear at most once. Every scorer produces `{ score: 0–1, passed: boolean }`; binary scorers emit 0 or 1.\n\n`exact_match` compares the trimmed output text to `expected_output`. `contains` looks for `value` in the output text. `json_logic` evaluates `expression` over `{ input, output, object, expected, item.metadata }`, where `object` is the structured output (absent when the agent has no `output_schema`). `output_schema` validates the structured output against the scorer's own `schema`, falling back to the agent's; it requires the agent to carry an `output_schema`, because without one the platform emits no structured output and every item would score 0.\n\n`llm_judge` grades the output with a model completion, returning a continuous score plus its `reasoning`. Its `pass_threshold` is required: a continuous score says nothing about where \"good enough\" is, and a defaulted cutoff would silently decide the gate.\n\n`embedding_similarity` embeds the output text and `expected_output` with the platform's configured embedding model (`EMBEDDING_PROVIDER` / `EMBEDDING_MODEL` — the same stack document ingestion uses) and scores their cosine similarity, clamped to 0-1. Its `pass_threshold` is required for the same reason as the judge's. An item without an `expected_output` scores 0; an embedding backend failure marks the **item** errored, never a score of 0.\n\n`tool` runs a custom scoring algorithm: a server-callable project tool the engine invokes once per item with the item's context. Unlike the built-in types it may appear several times, each under a distinct `name` — outcomes and aggregates key on the name.\n\n`decider` grades each item with a decision of a project decider: its `score` expression reads the decision's `answers`. Like `tool`, it keys on its `name` and may appear several times.",
|
|
4386
4392
|
"required": false,
|
|
4387
4393
|
"type": "array",
|
|
4388
4394
|
"in": "body"
|
|
@@ -4392,6 +4398,12 @@ var routes = {
|
|
|
4392
4398
|
"required": false,
|
|
4393
4399
|
"type": "number",
|
|
4394
4400
|
"in": "body"
|
|
4401
|
+
}, {
|
|
4402
|
+
"name": "group_by",
|
|
4403
|
+
"description": "A key of the items' `metadata` to group run scores by; null clears it, omitting it leaves it unchanged.",
|
|
4404
|
+
"required": false,
|
|
4405
|
+
"type": "string",
|
|
4406
|
+
"in": "body"
|
|
4395
4407
|
}]
|
|
4396
4408
|
},
|
|
4397
4409
|
"delete-eval": {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soat/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.68.0",
|
|
4
4
|
"description": "CLI for SOAT — open-source infrastructure for production-ready AI agents.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ai-agents",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"dependencies": {
|
|
23
23
|
"@inquirer/input": "^5.1.6",
|
|
24
24
|
"@inquirer/password": "^5.2.2",
|
|
25
|
-
"@soat/sdk": "0.
|
|
25
|
+
"@soat/sdk": "0.68.0",
|
|
26
26
|
"@ttoss/logger": "^0.8.20",
|
|
27
27
|
"commander": "^15.0.0",
|
|
28
28
|
"js-yaml": "^5.4.2"
|