@naturali/cli 0.67.0 → 0.68.0

package/dist/index.mjs

@@ -14,7 +14,7 @@ var __exportAll = (all, no_symbols) => {
 };
 //#endregion
 //#region package.json
-var version = "0.67.0";
+var version = "0.68.0";
 //#endregion
 //#region ../sdk/src/generated/core/bodySerializer.gen.ts
 const jsonBodySerializer = { bodySerializer: (body) => JSON.stringify(body, (_key, value) => typeof value === "bigint" ? value.toString() : value) };
@@ -1676,6 +1676,282 @@ var Conversations = class {
 		});
 	}
 };
+var Evaluations = class {
+	/**
+	* List datasets
+	*
+	* Returns the datasets defined in a project
+	*/
+	static listDatasets(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/datasets",
+			...options
+		});
+	}
+	/**
+	* Create a dataset
+	*
+	* Creates a project-scoped dataset — a named collection of test cases an eval runs an agent against. Names are unique per project.
+	*
+	* Datasets are operator-owned **fixtures**. The platform's content purge never deletes or mutates a dataset item, so erasing a generation cannot silently stop a test suite from being runnable.
+	*/
+	static createDataset(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/datasets",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* Delete a dataset
+	*
+	* Deletes a dataset, its items, and every eval bound to it. Results of runs that already scored those items keep their frozen copies of the input and expected output.
+	*/
+	static deleteDataset(options) {
+		return (options.client ?? client).delete({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}",
+			...options
+		});
+	}
+	/**
+	* Get a dataset
+	*
+	* Returns a specific dataset
+	*/
+	static getDataset(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}",
+			...options
+		});
+	}
+	/**
+	* Update a dataset
+	*
+	* Updates a dataset's name and/or description
+	*/
+	static updateDataset(options) {
+		return (options.client ?? client).put({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* List dataset items
+	*
+	* Returns the test cases in a dataset, oldest first
+	*/
+	static listDatasetItems(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}/items",
+			...options
+		});
+	}
+	/**
+	* Add a dataset item
+	*
+	* Adds one test case. `input` is replayed verbatim as the generation's messages, so it must be a non-empty array of `{ role, content }`.
+	*/
+	static createDatasetItem(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}/items",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* Curate a dataset item from a generation
+	*
+	* Promotes a real, completed generation into a test case: its input messages become the item's `input`, and its own answer becomes `expected_output` unless you supply one. Use it to build an evaluation set out of production traffic rather than hand-authoring fixtures.
+	*
+	* The item is a **copy**, not a view. It keeps working after the source generation's content is purged, and `source_generation_id` goes null if that generation is deleted — a purge can never quietly stop a suite from being runnable.
+	*
+	* Requires both `evaluations:CreateDataset` and `generations:GetGeneration`: the call copies content out of a generation, so a principal that may not read that generation may not curate it either.
+	*
+	* Only a **completed** generation can be promoted (`409 GENERATION_NOT_COMPLETED`), and only while its content is still available: an agent or project running with `trace_content_mode: none` never stored the input, and a purged or expired generation no longer has it (`409 GENERATION_CONTENT_UNAVAILABLE`). Generations that predate input recording answer the same way.
+	*/
+	static createDatasetItemFromGeneration(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}/items/from-generation",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* Delete a dataset item
+	*
+	* Deletes a test case. Results of runs that already scored it stay readable; their `dataset_item_id` becomes null.
+	*/
+	static deleteDatasetItem(options) {
+		return (options.client ?? client).delete({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}/items/{item_id}",
+			...options
+		});
+	}
+	/**
+	* Update a dataset item
+	*
+	* Updates a test case. Runs that already scored it are unaffected — each result carries its own frozen copy of the input and expected output.
+	*/
+	static updateDatasetItem(options) {
+		return (options.client ?? client).put({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}/items/{item_id}",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* List evals
+	*
+	* Returns the evals defined in a project
+	*/
+	static listEvals(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/evals",
+			...options
+		});
+	}
+	/**
+	* Create an eval
+	*
+	* Binds an agent under test to a dataset and a list of scorers. The agent and the dataset must belong to the same project as the eval; a cross-project reference is rejected with 400.
+	*
+	* Scorer config is frozen here rather than read from the agent at run time, so two runs of the same eval are always judged by the same criteria and their comparison measures the agent instead of the config drifting underneath it. Each scorer `type` may appear at most once.
+	*/
+	static createEval(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/evals",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* Delete an eval
+	*
+	* Deletes an eval, its runs, and their results
+	*/
+	static deleteEval(options) {
+		return (options.client ?? client).delete({
+			url: "/v1/projects/{project_id}/evals/{eval_id}",
+			...options
+		});
+	}
+	/**
+	* Get an eval
+	*
+	* Returns a specific eval
+	*/
+	static getEval(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/evals/{eval_id}",
+			...options
+		});
+	}
+	/**
+	* Update an eval
+	*
+	* Updates an eval. Changing `agent_id` re-validates the scorers against the new agent, since an `output_schema` scorer that was legal against the old one may not be.
+	*/
+	static updateEval(options) {
+		return (options.client ?? client).put({
+			url: "/v1/projects/{project_id}/evals/{eval_id}",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* List eval runs
+	*
+	* Returns an eval's runs, newest first
+	*/
+	static listEvalRuns(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/evals/{eval_id}/runs",
+			...options
+		});
+	}
+	/**
+	* Start an eval run
+	*
+	* Runs the eval against its dataset, creating one real agent generation per item and scoring the outputs.
+	*
+	* `wait: true` executes the run synchronously and returns it terminal, with its scores. The dataset is capped at 25 items for a synchronous run; a larger one is rejected with 400 rather than partially scored.
+	*
+	* `wait: false` (the default) enqueues one task per item and returns immediately with `status: "queued"`. A worker executes the items and the run settles itself; poll `GET /evals/{eval_id}/runs/{eval_run_id}` for the terminal status. There is no item cap on a queued run.
+	*
+	* The whole run is pinned to **one** agent version, stamped on `agent_version`: pass one explicitly to evaluate a canary before promoting it, or omit it to use the active release's stable version (or the live draft when no release is in effect). Without the pin, release assignment would bucket each item independently and blend two configs into a single score.
+	*
+	* With `baseline_run_id`, the finished run's `aggregate_scores.baseline` carries per-scorer deltas against that run, computed over the items present and scorable in **both** runs, with the divergence counted. A delta over a shifted dataset is therefore never presented as a clean comparison.
+	*/
+	static startEvalRun(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/evals/{eval_id}/runs",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* Get an eval run
+	*
+	* Returns a run's status, counts, and aggregate scores
+	*/
+	static getEvalRun(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/evals/{eval_id}/runs/{eval_run_id}",
+			...options
+		});
+	}
+	/**
+	* List eval run results
+	*
+	* Returns the per-item results of a run, oldest first
+	*/
+	static listEvalResults(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/evals/{eval_id}/runs/{eval_run_id}/results",
+			...options
+		});
+	}
+	/**
+	* Cancel an eval run
+	*
+	* Cancels a queued or running run: its outstanding item tasks are dropped so it stops consuming provider budget, and the run settles as `canceled`.
+	*
+	* Results already written are kept — they are real measurements of generations that were really paid for — and `completed_count` / `errored_count` report what ran. `aggregate_scores` is deliberately left null: a partial roll-up in the same field a completed run uses would read as a whole-dataset verdict.
+	*
+	* A run that has already finished is rejected with 400.
+	*/
+	static cancelEvalRun(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/evals/{eval_id}/runs/{eval_run_id}/cancel",
+			...options
+		});
+	}
+};
 var Generations = class {
 	/**
 	* List generations
@@ -2253,6 +2529,58 @@ var Tools = class {
 		});
 	}
 };
+var Traces = class {
+	/**
+	* List traces
+	*
+	* Returns a paginated list of execution traces for the project.
+	*/
+	static listTraces(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/traces",
+			...options
+		});
+	}
+	/**
+	* Get a trace
+	*
+	* Returns a single trace by ID.
+	*/
+	static getTrace(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/traces/{trace_id}",
+			...options
+		});
+	}
+	/**
+	* Get trace tree
+	*
+	* Returns the full execution tree rooted at the given trace (or its root if the given trace is a child). Each node represents one agent's execution session. The `children` array contains traces triggered by sub-agent tool calls from that trace.
+	*
+	*/
+	static getTraceTree(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/traces/{trace_id}/tree",
+			...options
+		});
+	}
+	/**
+	* Purge trace content
+	*
+	* Deletes the trace's steps object from storage and clears its content columns (`file_id`, `error`), cascading to every descendant trace and to all of their generations. A descendant holds its own steps object covering the same run, so the cascade is what makes the erasure complete rather than merely partial.
+	*
+	* The rows survive as auditable skeletons with `content_redacted_at` set — ids, timestamps, step counts, and the generations' usage-attribution fields are preserved, because the billing and audit ledger must outlive a tenant's erasure of the content. A purged trace therefore reads back as a skeleton, not a 404: a 404 would prove nothing.
+	*
+	* Idempotent — purging an already-purged trace succeeds and leaves the original `content_redacted_at` in place.
+	*
+	*/
+	static purgeTraceContent(options) {
+		return (options.client ?? client).delete({
+			url: "/v1/projects/{project_id}/traces/{trace_id}/content",
+			...options
+		});
+	}
+};
 var Users = class {
 	/**
 	* Get the current user
@@ -2458,12 +2786,14 @@ var NaturaliClient = class {
 	channels;
 	auth;
 	conversations;
+	evaluations;
 	generations;
 	modelRoutes;
 	projects;
 	secrets;
 	sessions;
 	tools;
+	traces;
 	users;
 	webhooks;
 	/** The underlying HTTP client, for interceptors or one-off requests. */
@@ -2485,12 +2815,14 @@ var NaturaliClient = class {
 		this.channels = bindResource(Channels, this.http);
 		this.auth = bindResource(Auth, this.http);
 		this.conversations = bindResource(Conversations, this.http);
+		this.evaluations = bindResource(Evaluations, this.http);
 		this.generations = bindResource(Generations, this.http);
 		this.modelRoutes = bindResource(ModelRoutes, this.http);
 		this.projects = bindResource(Projects, this.http);
 		this.secrets = bindResource(Secrets, this.http);
 		this.sessions = bindResource(Sessions, this.http);
 		this.tools = bindResource(Tools, this.http);
+		this.traces = bindResource(Traces, this.http);
 		this.users = bindResource(Users, this.http);
 		this.webhooks = bindResource(Webhooks, this.http);
 	}
@@ -2507,6 +2839,7 @@ var src_exports = /* @__PURE__ */ __exportAll({
 	Auth: () => Auth,
 	Channels: () => Channels,
 	Conversations: () => Conversations,
+	Evaluations: () => Evaluations,
 	Generations: () => Generations,
 	ModelRoutes: () => ModelRoutes,
 	NaturaliClient: () => NaturaliClient,
@@ -2514,6 +2847,7 @@ var src_exports = /* @__PURE__ */ __exportAll({
 	Secrets: () => Secrets,
 	Sessions: () => Sessions,
 	Tools: () => Tools,
+	Traces: () => Traces,
 	Users: () => Users,
 	Webhooks: () => Webhooks,
 	createClient: () => createClient,
@@ -6064,21 +6398,14 @@ const routes = {
 			"in": "path"
 		}]
 	},
-	"list-generations": {
-		serviceClass: "Generations",
-		operationId: "listGenerations",
-		description: "Returns generations the caller can access, optionally filtered by agent, trace, and status. Replaces the former per-trace generations endpoint (use the trace_id query filter).",
-		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/generations",
+	"list-datasets": {
+		serviceClass: "Evaluations",
+		operationId: "listDatasets",
+		description: "Returns the datasets defined in a project",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
 		httpMethod: "get",
 		pathParams: ["project_id"],
-		queryParams: [
-			"agent_id",
-			"trace_id",
-			"initiator_generation_id",
-			"status",
-			"limit",
-			"offset"
-		],
+		queryParams: ["limit", "offset"],
 		flags: [
 			{
 				"name": "project_id",
@@ -6088,17 +6415,791 @@ const routes = {
 				"in": "path"
 			},
 			{
-				"name": "agent_id",
-				"description": "Filter by agent public ID",
+				"name": "limit",
+				"description": "Maximum number of results to return",
 				"required": false,
-				"type": "string",
+				"type": "integer",
 				"in": "query"
 			},
 			{
-				"name": "trace_id",
-				"description": "Filter by trace public ID",
+				"name": "offset",
+				"description": "Number of results to skip",
 				"required": false,
-				"type": "string",
+				"type": "integer",
+				"in": "query"
+			}
+		]
+	},
+	"create-dataset": {
+		serviceClass: "Evaluations",
+		operationId: "createDataset",
+		description: "Creates a project-scoped dataset — a named collection of test cases an eval runs an agent against. Names are unique per project. Datasets are operator-owned **fixtures**. The platform's content purge never deletes or mutates a dataset item, so erasing a generation cannot silently stop a test suite from being runnable.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "post",
+		pathParams: ["project_id"],
+		queryParams: [],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "name",
+				"description": "Unique name within the project",
+				"required": true,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "description",
+				"description": "What this suite covers",
+				"required": false,
+				"type": "string",
+				"in": "body"
+			}
+		]
+	},
+	"get-dataset": {
+		serviceClass: "Evaluations",
+		operationId: "getDataset",
+		description: "Returns a specific dataset",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "get",
+		pathParams: ["project_id", "dataset_id"],
+		queryParams: [],
+		flags: [{
+			"name": "project_id",
+			"description": "Project public ID (proj_ prefix).",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}, {
+			"name": "dataset_id",
+			"description": "Dataset ID",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}]
+	},
+	"update-dataset": {
+		serviceClass: "Evaluations",
+		operationId: "updateDataset",
+		description: "Updates a dataset's name and/or description",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "put",
+		pathParams: ["project_id", "dataset_id"],
+		queryParams: [],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "dataset_id",
+				"description": "Dataset ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "name",
+				"description": "",
+				"required": false,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "description",
+				"description": "",
+				"required": false,
+				"type": "string",
+				"in": "body"
+			}
+		]
+	},
+	"delete-dataset": {
+		serviceClass: "Evaluations",
+		operationId: "deleteDataset",
+		description: "Deletes a dataset, its items, and every eval bound to it. Results of runs that already scored those items keep their frozen copies of the input and expected output.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "delete",
+		pathParams: ["project_id", "dataset_id"],
+		queryParams: [],
+		flags: [{
+			"name": "project_id",
+			"description": "Project public ID (proj_ prefix).",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}, {
+			"name": "dataset_id",
+			"description": "Dataset ID",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}]
+	},
+	"list-dataset-items": {
+		serviceClass: "Evaluations",
+		operationId: "listDatasetItems",
+		description: "Returns the test cases in a dataset, oldest first",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "get",
+		pathParams: ["project_id", "dataset_id"],
+		queryParams: ["limit", "offset"],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "dataset_id",
+				"description": "Dataset ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "limit",
+				"description": "Maximum number of results to return",
+				"required": false,
+				"type": "integer",
+				"in": "query"
+			},
+			{
+				"name": "offset",
+				"description": "Number of results to skip",
+				"required": false,
+				"type": "integer",
+				"in": "query"
+			}
+		]
+	},
+	"create-dataset-item": {
+		serviceClass: "Evaluations",
+		operationId: "createDatasetItem",
+		description: "Adds one test case. `input` is replayed verbatim as the generation's messages, so it must be a non-empty array of `{ role, content }`.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "post",
+		pathParams: ["project_id", "dataset_id"],
+		queryParams: [],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "dataset_id",
+				"description": "Dataset ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "input",
+				"description": "Messages replayed verbatim as the generation's input",
+				"required": true,
+				"type": "array",
+				"in": "body"
+			},
+			{
+				"name": "expected_output",
+				"description": "Reference answer for exact_match / llm_judge scorers",
+				"required": false,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "metadata",
+				"description": "Free-form tags, opaque to the platform",
+				"required": false,
+				"type": "object",
+				"in": "body"
+			}
+		]
+	},
+	"create-dataset-item-from-generation": {
+		serviceClass: "Evaluations",
+		operationId: "createDatasetItemFromGeneration",
+		description: "Promotes a real, completed generation into a test case: its input messages become the item's `input`, and its own answer becomes `expected_output` unless you supply one. Use it to build an evaluation set out of production traffic rather than hand-authoring fixtures. The item is a **copy**, not a view. It keeps working after the source generation's content is purged, and `source_generation_id` goes null if that generation is deleted — a purge can never quietly stop a suite from being runnable. Requires both `evaluations:CreateDataset` and `generations:GetGeneration`: the call copies content out of a generation, so a principal that may not read that generation may not curate it either. Only a **completed** generation can be promoted (`409 GENERATION_NOT_COMPLETED`), and only while its content is still available: an agent or project running with `trace_content_mode: none` never stored the input, and a purged or expired generation no longer has it (`409 GENERATION_CONTENT_UNAVAILABLE`). Generations that predate input recording answer the same way.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "post",
+		pathParams: ["project_id", "dataset_id"],
+		queryParams: [],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "dataset_id",
+				"description": "Dataset ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "generation_id",
+				"description": "The completed generation to promote. Must belong to the same project as the dataset.",
+				"required": true,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "expected_output",
+				"description": "Reference answer. Omit to use the generation's own answer; pass `null` to store the item with no reference answer.",
+				"required": false,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "metadata",
+				"description": "Free-form tags, opaque to the platform",
+				"required": false,
+				"type": "object",
+				"in": "body"
+			}
+		]
+	},
+	"update-dataset-item": {
+		serviceClass: "Evaluations",
+		operationId: "updateDatasetItem",
+		description: "Updates a test case. Runs that already scored it are unaffected — each result carries its own frozen copy of the input and expected output.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "put",
+		pathParams: [
+			"project_id",
+			"dataset_id",
+			"item_id"
+		],
+		queryParams: [],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "dataset_id",
+				"description": "Dataset ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "item_id",
+				"description": "Dataset item ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "input",
+				"description": "Messages replayed verbatim as the generation's input",
+				"required": false,
+				"type": "array",
+				"in": "body"
+			},
+			{
+				"name": "expected_output",
+				"description": "",
+				"required": false,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "metadata",
+				"description": "",
+				"required": false,
+				"type": "object",
+				"in": "body"
+			}
+		]
+	},
+	"delete-dataset-item": {
+		serviceClass: "Evaluations",
+		operationId: "deleteDatasetItem",
+		description: "Deletes a test case. Results of runs that already scored it stay readable; their `dataset_item_id` becomes null.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "delete",
+		pathParams: [
+			"project_id",
+			"dataset_id",
+			"item_id"
+		],
+		queryParams: [],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "dataset_id",
+				"description": "Dataset ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "item_id",
+				"description": "Dataset item ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			}
+		]
+	},
+	"list-evals": {
+		serviceClass: "Evaluations",
+		operationId: "listEvals",
+		description: "Returns the evals defined in a project",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "get",
+		pathParams: ["project_id"],
+		queryParams: ["limit", "offset"],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "limit",
+				"description": "Maximum number of results to return",
+				"required": false,
+				"type": "integer",
+				"in": "query"
+			},
+			{
+				"name": "offset",
+				"description": "Number of results to skip",
+				"required": false,
+				"type": "integer",
+				"in": "query"
+			}
+		]
+	},
+	"create-eval": {
+		serviceClass: "Evaluations",
+		operationId: "createEval",
+		description: "Binds an agent under test to a dataset and a list of scorers. The agent and the dataset must belong to the same project as the eval; a cross-project reference is rejected with 400. Scorer config is frozen here rather than read from the agent at run time, so two runs of the same eval are always judged by the same criteria and their comparison measures the agent instead of the config drifting underneath it. Each scorer `type` may appear at most once.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "post",
+		pathParams: ["project_id"],
+		queryParams: [],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "name",
+				"description": "Unique name within the project",
+				"required": true,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "agent_id",
+				"description": "The agent under test",
+				"required": true,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "dataset_id",
+				"description": "The dataset to run it against",
+				"required": true,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "scorers",
+				"description": "Scorer configs, a discriminated union on `type`. Each type may appear at most once. Every scorer produces `{ score: 0–1, passed: boolean }`; binary scorers emit 0 or 1.\n\n`exact_match` compares the trimmed output text to `expected_output`. `contains` looks for `value` in the output text. `json_logic` evaluates `expression` over `{ input, output, object, expected, item.metadata }`, where `object` is the structured output (absent when the agent has no `output_schema`). `output_schema` validates the structured output against the scorer's own `schema`, falling back to the agent's; it requires the agent to carry an `output_schema`, because without one the platform emits no structured output and every item would score 0.\n\n`llm_judge` grades the output with a model completion, returning a continuous score plus its `reasoning`. Its `pass_threshold` is required: a continuous score says nothing about where \"good enough\" is, and a defaulted cutoff would silently decide the gate.\n\n`tool` runs a custom scoring algorithm: a server-callable project tool the engine invokes once per item with the item's context. Unlike the built-in types it may appear several times, each under a distinct `name` — outcomes and aggregates key on the name.",
+				"required": true,
+				"type": "array",
+				"in": "body"
+			},
+			{
+				"name": "pass_threshold",
+				"description": "0–1. The run passes iff its pass rate — passed items over non-errored items — is at least this. Null reports scores without gating on them.",
+				"required": false,
+				"type": "number",
+				"in": "body"
+			}
+		]
+	},
+	"get-eval": {
+		serviceClass: "Evaluations",
+		operationId: "getEval",
+		description: "Returns a specific eval",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "get",
+		pathParams: ["project_id", "eval_id"],
+		queryParams: [],
+		flags: [{
+			"name": "project_id",
+			"description": "Project public ID (proj_ prefix).",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}, {
+			"name": "eval_id",
+			"description": "Eval ID",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}]
+	},
+	"update-eval": {
+		serviceClass: "Evaluations",
+		operationId: "updateEval",
+		description: "Updates an eval. Changing `agent_id` re-validates the scorers against the new agent, since an `output_schema` scorer that was legal against the old one may not be.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "put",
+		pathParams: ["project_id", "eval_id"],
+		queryParams: [],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "eval_id",
+				"description": "Eval ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "name",
+				"description": "",
+				"required": false,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "agent_id",
+				"description": "",
+				"required": false,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "dataset_id",
+				"description": "",
+				"required": false,
+				"type": "string",
+				"in": "body"
+			},
+			{
+				"name": "scorers",
+				"description": "Scorer configs, a discriminated union on `type`. Each type may appear at most once. Every scorer produces `{ score: 0–1, passed: boolean }`; binary scorers emit 0 or 1.\n\n`exact_match` compares the trimmed output text to `expected_output`. `contains` looks for `value` in the output text. `json_logic` evaluates `expression` over `{ input, output, object, expected, item.metadata }`, where `object` is the structured output (absent when the agent has no `output_schema`). `output_schema` validates the structured output against the scorer's own `schema`, falling back to the agent's; it requires the agent to carry an `output_schema`, because without one the platform emits no structured output and every item would score 0.\n\n`llm_judge` grades the output with a model completion, returning a continuous score plus its `reasoning`. Its `pass_threshold` is required: a continuous score says nothing about where \"good enough\" is, and a defaulted cutoff would silently decide the gate.\n\n`tool` runs a custom scoring algorithm: a server-callable project tool the engine invokes once per item with the item's context. Unlike the built-in types it may appear several times, each under a distinct `name` — outcomes and aggregates key on the name.",
+				"required": false,
+				"type": "array",
+				"in": "body"
+			},
+			{
+				"name": "pass_threshold",
+				"description": "",
+				"required": false,
+				"type": "number",
+				"in": "body"
+			}
+		]
+	},
+	"delete-eval": {
+		serviceClass: "Evaluations",
+		operationId: "deleteEval",
+		description: "Deletes an eval, its runs, and their results",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "delete",
+		pathParams: ["project_id", "eval_id"],
+		queryParams: [],
+		flags: [{
+			"name": "project_id",
+			"description": "Project public ID (proj_ prefix).",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}, {
+			"name": "eval_id",
+			"description": "Eval ID",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}]
+	},
+	"list-eval-runs": {
+		serviceClass: "Evaluations",
+		operationId: "listEvalRuns",
+		description: "Returns an eval's runs, newest first",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "get",
+		pathParams: ["project_id", "eval_id"],
+		queryParams: ["limit", "offset"],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "eval_id",
+				"description": "Eval ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "limit",
+				"description": "Maximum number of results to return",
+				"required": false,
+				"type": "integer",
+				"in": "query"
+			},
+			{
+				"name": "offset",
+				"description": "Number of results to skip",
+				"required": false,
+				"type": "integer",
+				"in": "query"
+			}
+		]
+	},
+	"start-eval-run": {
+		serviceClass: "Evaluations",
+		operationId: "startEvalRun",
+		description: "Runs the eval against its dataset, creating one real agent generation per item and scoring the outputs. `wait: true` executes the run synchronously and returns it terminal, with its scores. The dataset is capped at 25 items for a synchronous run; a larger one is rejected with 400 rather than partially scored. `wait: false` (the default) enqueues one task per item and returns immediately with `status: \"queued\"`. A worker executes the items and the run settles itself; poll `GET /evals/{eval_id}/runs/{eval_run_id}` for the terminal status. There is no item cap on a queued run. The whole run is pinned to **one** agent version, stamped on `agent_version`: pass one explicitly to evaluate a canary before promoting it, or omit it to use the active release's stable version (or the live draft when no release is in effect). Without the pin, release assignment would bucket each item independently and blend two configs into a single score. With `baseline_run_id`, the finished run's `aggregate_scores.baseline` carries per-scorer deltas against that run, computed over the items present and scorable in **both** runs, with the divergence counted. A delta over a shifted dataset is therefore never presented as a clean comparison.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "post",
+		pathParams: ["project_id", "eval_id"],
+		queryParams: [],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "eval_id",
+				"description": "Eval ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "wait",
+				"description": "True runs the eval synchronously (25-item cap) and returns a terminal run with its scores. False — the default — enqueues the items and returns a `queued` run immediately.",
+				"required": false,
+				"type": "boolean",
+				"in": "body"
+			},
+			{
+				"name": "agent_version",
+				"description": "An archived agent version to evaluate. Defaults to the active release's stable version, or the live draft version when no release is in effect.",
+				"required": false,
+				"type": "integer",
+				"in": "body"
+			},
+			{
+				"name": "baseline_run_id",
+				"description": "A terminal run of the same eval to compare against. The finished run's `aggregate_scores.baseline` reports per-scorer deltas over the item intersection. A run of a different eval is rejected with 400.",
+				"required": false,
+				"type": "string",
+				"in": "body"
+			}
+		]
+	},
+	"get-eval-run": {
+		serviceClass: "Evaluations",
+		operationId: "getEvalRun",
+		description: "Returns a run's status, counts, and aggregate scores",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "get",
+		pathParams: [
+			"project_id",
+			"eval_id",
+			"eval_run_id"
+		],
+		queryParams: [],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "eval_id",
+				"description": "Eval ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "eval_run_id",
+				"description": "Eval run ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			}
+		]
+	},
+	"list-eval-results": {
+		serviceClass: "Evaluations",
+		operationId: "listEvalResults",
+		description: "Returns the per-item results of a run, oldest first",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "get",
+		pathParams: [
+			"project_id",
+			"eval_id",
+			"eval_run_id"
+		],
+		queryParams: ["limit", "offset"],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "eval_id",
+				"description": "Eval ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "eval_run_id",
+				"description": "Eval run ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "limit",
+				"description": "Maximum number of results to return",
+				"required": false,
+				"type": "integer",
+				"in": "query"
+			},
+			{
+				"name": "offset",
+				"description": "Number of results to skip",
+				"required": false,
+				"type": "integer",
+				"in": "query"
+			}
+		]
+	},
+	"cancel-eval-run": {
+		serviceClass: "Evaluations",
+		operationId: "cancelEvalRun",
+		description: "Cancels a queued or running run: its outstanding item tasks are dropped so it stops consuming provider budget, and the run settles as `canceled`. Results already written are kept — they are real measurements of generations that were really paid for — and `completed_count` / `errored_count` report what ran. `aggregate_scores` is deliberately left null: a partial roll-up in the same field a completed run uses would read as a whole-dataset verdict. A run that has already finished is rejected with 400.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/evaluations",
+		httpMethod: "post",
+		pathParams: [
+			"project_id",
+			"eval_id",
+			"eval_run_id"
+		],
+		queryParams: [],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "eval_id",
+				"description": "Eval ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "eval_run_id",
+				"description": "Eval run ID",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			}
+		]
+	},
+	"list-generations": {
+		serviceClass: "Generations",
+		operationId: "listGenerations",
+		description: "Returns generations the caller can access, optionally filtered by agent, trace, and status. Replaces the former per-trace generations endpoint (use the trace_id query filter).",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/generations",
+		httpMethod: "get",
+		pathParams: ["project_id"],
+		queryParams: [
+			"agent_id",
+			"trace_id",
+			"initiator_generation_id",
+			"status",
+			"limit",
+			"offset"
+		],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "agent_id",
+				"description": "Filter by agent public ID",
+				"required": false,
+				"type": "string",
+				"in": "query"
+			},
+			{
+				"name": "trace_id",
+				"description": "Filter by trace public ID",
+				"required": false,
+				"type": "string",
 				"in": "query"
 			},
 			{
@@ -7634,6 +8735,114 @@ const routes = {
 			}
 		]
 	},
+	"list-traces": {
+		serviceClass: "Traces",
+		operationId: "listTraces",
+		description: "Returns a paginated list of execution traces for the project.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/traces",
+		httpMethod: "get",
+		pathParams: ["project_id"],
+		queryParams: ["limit", "offset"],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "limit",
+				"description": "Maximum number of results to return",
+				"required": false,
+				"type": "integer",
+				"in": "query"
+			},
+			{
+				"name": "offset",
+				"description": "Number of results to skip",
+				"required": false,
+				"type": "integer",
+				"in": "query"
+			}
+		]
+	},
+	"get-trace": {
+		serviceClass: "Traces",
+		operationId: "getTrace",
+		description: "Returns a single trace by ID.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/traces",
+		httpMethod: "get",
+		pathParams: ["project_id", "trace_id"],
+		queryParams: [],
+		flags: [{
+			"name": "project_id",
+			"description": "Project public ID (proj_ prefix).",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}, {
+			"name": "trace_id",
+			"description": "Public ID of the trace",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}]
+	},
+	"get-trace-tree": {
+		serviceClass: "Traces",
+		operationId: "getTraceTree",
+		description: "Returns the full execution tree rooted at the given trace (or its root if the given trace is a child). Each node represents one agent's execution session. The `children` array contains traces triggered by sub-agent tool calls from that trace.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/traces",
+		httpMethod: "get",
+		pathParams: ["project_id", "trace_id"],
+		queryParams: ["include"],
+		flags: [
+			{
+				"name": "project_id",
+				"description": "Project public ID (proj_ prefix).",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "trace_id",
+				"description": "Public ID of any trace in the tree (root or child)",
+				"required": true,
+				"type": "string",
+				"in": "path"
+			},
+			{
+				"name": "include",
+				"description": "Comma-separated list of related resources to embed on each node. Supported value: `generations` — attaches all generations that belong to each trace node (including sub-agent generations linked via `initiator_generation_id`).\n",
+				"required": false,
+				"type": "string",
+				"in": "query"
+			}
+		]
+	},
+	"purge-trace-content": {
+		serviceClass: "Traces",
+		operationId: "purgeTraceContent",
+		description: "Deletes the trace's steps object from storage and clears its content columns (`file_id`, `error`), cascading to every descendant trace and to all of their generations. A descendant holds its own steps object covering the same run, so the cascade is what makes the erasure complete rather than merely partial. The rows survive as auditable skeletons with `content_redacted_at` set — ids, timestamps, step counts, and the generations' usage-attribution fields are preserved, because the billing and audit ledger must outlive a tenant's erasure of the content. A purged trace therefore reads back as a skeleton, not a 404: a 404 would prove nothing. Idempotent — purging an already-purged trace succeeds and leaves the original `content_redacted_at` in place.",
+		moduleDocsUrl: "https://docs.naturali.ai/docs/modules/traces",
+		httpMethod: "delete",
+		pathParams: ["project_id", "trace_id"],
+		queryParams: [],
+		flags: [{
+			"name": "project_id",
+			"description": "Project public ID (proj_ prefix).",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}, {
+			"name": "trace_id",
+			"description": "Public ID of the trace",
+			"required": true,
+			"type": "string",
+			"in": "path"
+		}]
+	},
 	"get-current-user": {
 		serviceClass: "Users",
 		operationId: "getCurrentUser",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturali/cli",
-  "version": "0.67.0",
+  "version": "0.68.0",
   "description": "Command-line interface for the naturali.ai API, generated from its OpenAPI specs",
   "type": "module",
   "bin": {
@@ -30,7 +30,7 @@
     "tsx": "^4.23.1",
     "typescript": "~6.0.3",
     "vitest": "^4.1.10",
-    "@naturali/sdk": "0.67.0"
+    "@naturali/sdk": "0.68.0"
   },
   "scripts": {
     "generate": "tsx scripts/generate.ts",