@naturali/sdk 0.67.0 → 0.68.0

package/dist/index.cjs

@@ -1659,6 +1659,282 @@ var Conversations = class {
 		});
 	}
 };
+var Evaluations = class {
+	/**
+	* List datasets
+	*
+	* Returns the datasets defined in a project
+	*/
+	static listDatasets(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/datasets",
+			...options
+		});
+	}
+	/**
+	* Create a dataset
+	*
+	* Creates a project-scoped dataset — a named collection of test cases an eval runs an agent against. Names are unique per project.
+	*
+	* Datasets are operator-owned **fixtures**. The platform's content purge never deletes or mutates a dataset item, so erasing a generation cannot silently stop a test suite from being runnable.
+	*/
+	static createDataset(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/datasets",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* Delete a dataset
+	*
+	* Deletes a dataset, its items, and every eval bound to it. Results of runs that already scored those items keep their frozen copies of the input and expected output.
+	*/
+	static deleteDataset(options) {
+		return (options.client ?? client).delete({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}",
+			...options
+		});
+	}
+	/**
+	* Get a dataset
+	*
+	* Returns a specific dataset
+	*/
+	static getDataset(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}",
+			...options
+		});
+	}
+	/**
+	* Update a dataset
+	*
+	* Updates a dataset's name and/or description
+	*/
+	static updateDataset(options) {
+		return (options.client ?? client).put({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* List dataset items
+	*
+	* Returns the test cases in a dataset, oldest first
+	*/
+	static listDatasetItems(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}/items",
+			...options
+		});
+	}
+	/**
+	* Add a dataset item
+	*
+	* Adds one test case. `input` is replayed verbatim as the generation's messages, so it must be a non-empty array of `{ role, content }`.
+	*/
+	static createDatasetItem(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}/items",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* Curate a dataset item from a generation
+	*
+	* Promotes a real, completed generation into a test case: its input messages become the item's `input`, and its own answer becomes `expected_output` unless you supply one. Use it to build an evaluation set out of production traffic rather than hand-authoring fixtures.
+	*
+	* The item is a **copy**, not a view. It keeps working after the source generation's content is purged, and `source_generation_id` goes null if that generation is deleted — a purge can never quietly stop a suite from being runnable.
+	*
+	* Requires both `evaluations:CreateDataset` and `generations:GetGeneration`: the call copies content out of a generation, so a principal that may not read that generation may not curate it either.
+	*
+	* Only a **completed** generation can be promoted (`409 GENERATION_NOT_COMPLETED`), and only while its content is still available: an agent or project running with `trace_content_mode: none` never stored the input, and a purged or expired generation no longer has it (`409 GENERATION_CONTENT_UNAVAILABLE`). Generations that predate input recording answer the same way.
+	*/
+	static createDatasetItemFromGeneration(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}/items/from-generation",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* Delete a dataset item
+	*
+	* Deletes a test case. Results of runs that already scored it stay readable; their `dataset_item_id` becomes null.
+	*/
+	static deleteDatasetItem(options) {
+		return (options.client ?? client).delete({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}/items/{item_id}",
+			...options
+		});
+	}
+	/**
+	* Update a dataset item
+	*
+	* Updates a test case. Runs that already scored it are unaffected — each result carries its own frozen copy of the input and expected output.
+	*/
+	static updateDatasetItem(options) {
+		return (options.client ?? client).put({
+			url: "/v1/projects/{project_id}/datasets/{dataset_id}/items/{item_id}",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* List evals
+	*
+	* Returns the evals defined in a project
+	*/
+	static listEvals(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/evals",
+			...options
+		});
+	}
+	/**
+	* Create an eval
+	*
+	* Binds an agent under test to a dataset and a list of scorers. The agent and the dataset must belong to the same project as the eval; a cross-project reference is rejected with 400.
+	*
+	* Scorer config is frozen here rather than read from the agent at run time, so two runs of the same eval are always judged by the same criteria and their comparison measures the agent instead of the config drifting underneath it. Each scorer `type` may appear at most once.
+	*/
+	static createEval(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/evals",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* Delete an eval
+	*
+	* Deletes an eval, its runs, and their results
+	*/
+	static deleteEval(options) {
+		return (options.client ?? client).delete({
+			url: "/v1/projects/{project_id}/evals/{eval_id}",
+			...options
+		});
+	}
+	/**
+	* Get an eval
+	*
+	* Returns a specific eval
+	*/
+	static getEval(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/evals/{eval_id}",
+			...options
+		});
+	}
+	/**
+	* Update an eval
+	*
+	* Updates an eval. Changing `agent_id` re-validates the scorers against the new agent, since an `output_schema` scorer that was legal against the old one may not be.
+	*/
+	static updateEval(options) {
+		return (options.client ?? client).put({
+			url: "/v1/projects/{project_id}/evals/{eval_id}",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* List eval runs
+	*
+	* Returns an eval's runs, newest first
+	*/
+	static listEvalRuns(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/evals/{eval_id}/runs",
+			...options
+		});
+	}
+	/**
+	* Start an eval run
+	*
+	* Runs the eval against its dataset, creating one real agent generation per item and scoring the outputs.
+	*
+	* `wait: true` executes the run synchronously and returns it terminal, with its scores. The dataset is capped at 25 items for a synchronous run; a larger one is rejected with 400 rather than partially scored.
+	*
+	* `wait: false` (the default) enqueues one task per item and returns immediately with `status: "queued"`. A worker executes the items and the run settles itself; poll `GET /evals/{eval_id}/runs/{eval_run_id}` for the terminal status. There is no item cap on a queued run.
+	*
+	* The whole run is pinned to **one** agent version, stamped on `agent_version`: pass one explicitly to evaluate a canary before promoting it, or omit it to use the active release's stable version (or the live draft when no release is in effect). Without the pin, release assignment would bucket each item independently and blend two configs into a single score.
+	*
+	* With `baseline_run_id`, the finished run's `aggregate_scores.baseline` carries per-scorer deltas against that run, computed over the items present and scorable in **both** runs, with the divergence counted. A delta over a shifted dataset is therefore never presented as a clean comparison.
+	*/
+	static startEvalRun(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/evals/{eval_id}/runs",
+			...options,
+			headers: {
+				"Content-Type": "application/json",
+				...options.headers
+			}
+		});
+	}
+	/**
+	* Get an eval run
+	*
+	* Returns a run's status, counts, and aggregate scores
+	*/
+	static getEvalRun(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/evals/{eval_id}/runs/{eval_run_id}",
+			...options
+		});
+	}
+	/**
+	* List eval run results
+	*
+	* Returns the per-item results of a run, oldest first
+	*/
+	static listEvalResults(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/evals/{eval_id}/runs/{eval_run_id}/results",
+			...options
+		});
+	}
+	/**
+	* Cancel an eval run
+	*
+	* Cancels a queued or running run: its outstanding item tasks are dropped so it stops consuming provider budget, and the run settles as `canceled`.
+	*
+	* Results already written are kept — they are real measurements of generations that were really paid for — and `completed_count` / `errored_count` report what ran. `aggregate_scores` is deliberately left null: a partial roll-up in the same field a completed run uses would read as a whole-dataset verdict.
+	*
+	* A run that has already finished is rejected with 400.
+	*/
+	static cancelEvalRun(options) {
+		return (options.client ?? client).post({
+			url: "/v1/projects/{project_id}/evals/{eval_id}/runs/{eval_run_id}/cancel",
+			...options
+		});
+	}
+};
 var Generations = class {
 	/**
 	* List generations
@@ -2236,6 +2512,58 @@ var Tools = class {
 		});
 	}
 };
+var Traces = class {
+	/**
+	* List traces
+	*
+	* Returns a paginated list of execution traces for the project.
+	*/
+	static listTraces(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/traces",
+			...options
+		});
+	}
+	/**
+	* Get a trace
+	*
+	* Returns a single trace by ID.
+	*/
+	static getTrace(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/traces/{trace_id}",
+			...options
+		});
+	}
+	/**
+	* Get trace tree
+	*
+	* Returns the full execution tree rooted at the given trace (or its root if the given trace is a child). Each node represents one agent's execution session. The `children` array contains traces triggered by sub-agent tool calls from that trace.
+	*
+	*/
+	static getTraceTree(options) {
+		return (options.client ?? client).get({
+			url: "/v1/projects/{project_id}/traces/{trace_id}/tree",
+			...options
+		});
+	}
+	/**
+	* Purge trace content
+	*
+	* Deletes the trace's steps object from storage and clears its content columns (`file_id`, `error`), cascading to every descendant trace and to all of their generations. A descendant holds its own steps object covering the same run, so the cascade is what makes the erasure complete rather than merely partial.
+	*
+	* The rows survive as auditable skeletons with `content_redacted_at` set — ids, timestamps, step counts, and the generations' usage-attribution fields are preserved, because the billing and audit ledger must outlive a tenant's erasure of the content. A purged trace therefore reads back as a skeleton, not a 404: a 404 would prove nothing.
+	*
+	* Idempotent — purging an already-purged trace succeeds and leaves the original `content_redacted_at` in place.
+	*
+	*/
+	static purgeTraceContent(options) {
+		return (options.client ?? client).delete({
+			url: "/v1/projects/{project_id}/traces/{trace_id}/content",
+			...options
+		});
+	}
+};
 var Users = class {
 	/**
 	* Get the current user
@@ -2441,12 +2769,14 @@ var NaturaliClient = class {
 	channels;
 	auth;
 	conversations;
+	evaluations;
 	generations;
 	modelRoutes;
 	projects;
 	secrets;
 	sessions;
 	tools;
+	traces;
 	users;
 	webhooks;
 	/** The underlying HTTP client, for interceptors or one-off requests. */
@@ -2468,12 +2798,14 @@ var NaturaliClient = class {
 		this.channels = bindResource(Channels, this.http);
 		this.auth = bindResource(Auth, this.http);
 		this.conversations = bindResource(Conversations, this.http);
+		this.evaluations = bindResource(Evaluations, this.http);
 		this.generations = bindResource(Generations, this.http);
 		this.modelRoutes = bindResource(ModelRoutes, this.http);
 		this.projects = bindResource(Projects, this.http);
 		this.secrets = bindResource(Secrets, this.http);
 		this.sessions = bindResource(Sessions, this.http);
 		this.tools = bindResource(Tools, this.http);
+		this.traces = bindResource(Traces, this.http);
 		this.users = bindResource(Users, this.http);
 		this.webhooks = bindResource(Webhooks, this.http);
 	}
@@ -2488,6 +2820,7 @@ exports.Assistant = Assistant;
 exports.Auth = Auth;
 exports.Channels = Channels;
 exports.Conversations = Conversations;
+exports.Evaluations = Evaluations;
 exports.Generations = Generations;
 exports.ModelRoutes = ModelRoutes;
 exports.NaturaliClient = NaturaliClient;
@@ -2495,6 +2828,7 @@ exports.Projects = Projects;
 exports.Secrets = Secrets;
 exports.Sessions = Sessions;
 exports.Tools = Tools;
+exports.Traces = Traces;
 exports.Users = Users;
 exports.Webhooks = Webhooks;
 exports.createClient = createClient;