@atlanai/sdk 0.1.1 → 0.2.1

package/dist/raw/eval/apis/ExperimentsApi.d.ts

@@ -0,0 +1,371 @@
+/**
+ * Evaluation catalog
+ * The evaluation loop as registry artifacts: curated `dataset`s of examples pinned from production sessions and traces, `experiment` run snapshots with create-only per-case `experiment_result`s, and versioned `scorer` definitions. A verdict is a score span in the run\'s trace; `POST /experiments/{id}/summarize` rolls a run\'s score spans into its durable `summary`. The gateway executes and scores nothing — external harnesses, CI, and judges run the evals, emit score spans, and upload results here.
+ *
+ * The version of the OpenAPI document: 0.1.0
+ *
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+import * as runtime from '../runtime';
+import { type EvalArtifactSearchRequest } from '../models/EvalArtifactSearchRequest';
+import { type EvalBulkExperimentResultResponse } from '../models/EvalBulkExperimentResultResponse';
+import { type EvalExperimentArtifactView } from '../models/EvalExperimentArtifactView';
+import { type EvalExperimentCreate } from '../models/EvalExperimentCreate';
+import { type EvalExperimentPatch } from '../models/EvalExperimentPatch';
+import { type EvalExperimentResultArtifactView } from '../models/EvalExperimentResultArtifactView';
+import { type EvalExperimentResultBulkCreate } from '../models/EvalExperimentResultBulkCreate';
+import { type EvalExperimentResultCreate } from '../models/EvalExperimentResultCreate';
+import { type EvalExperimentStatus } from '../models/EvalExperimentStatus';
+import { type EvalPageExperimentArtifactView } from '../models/EvalPageExperimentArtifactView';
+import { type EvalPageExperimentResultArtifactView } from '../models/EvalPageExperimentResultArtifactView';
+import { type EvalSubjectKind } from '../models/EvalSubjectKind';
+import { type GatewayTraceStatsEnvelope } from '../models/GatewayTraceStatsEnvelope';
+import { type GatewayTraceStatsRequest } from '../models/GatewayTraceStatsRequest';
+import { type SpanListEnvelope } from '../models/SpanListEnvelope';
+import { type TraceDetailView } from '../models/TraceDetailView';
+import { type TraceListEnvelope } from '../models/TraceListEnvelope';
+export interface EvalArchiveExperimentRequest {
+    experimentId: string;
+}
+export interface EvalCreateExperimentRequest {
+    evalExperimentCreate: EvalExperimentCreate;
+    contentEncoding?: EvalCreateExperimentContentEncodingEnum;
+}
+export interface EvalCreateExperimentResultRequest {
+    experimentId: string;
+    evalExperimentResultCreate: EvalExperimentResultCreate;
+    contentEncoding?: EvalCreateExperimentResultContentEncodingEnum;
+}
+export interface EvalCreateExperimentResultsBulkRequest {
+    experimentId: string;
+    evalExperimentResultBulkCreate: EvalExperimentResultBulkCreate;
+    contentEncoding?: EvalCreateExperimentResultsBulkContentEncodingEnum;
+}
+export interface EvalGetExperimentRequest {
+    experimentId: string;
+}
+export interface EvalGetExperimentResultRequest {
+    experimentId: string;
+    resultId: string;
+}
+export interface EvalGetExperimentTraceRequest {
+    experimentId: string;
+    traceId: string;
+}
+export interface EvalListExperimentResultsRequest {
+    experimentId: string;
+    limit?: number;
+    offset?: number;
+    datasetRecordId?: string;
+    sessionId?: string;
+    sort?: string;
+}
+export interface EvalListExperimentTraceSpansRequest {
+    experimentId: string;
+    traceId: string;
+    limit?: number;
+    cursor?: string;
+    fields?: string;
+}
+export interface EvalListExperimentTracesRequest {
+    experimentId: string;
+    startTimeUnixSeconds?: number;
+    endTimeUnixSeconds?: number;
+    limit?: number;
+    cursor?: string;
+    sort?: string;
+}
+export interface EvalListExperimentsRequest {
+    limit?: number;
+    offset?: number;
+    datasetId?: string;
+    subjectKind?: EvalSubjectKind;
+    subjectId?: string;
+    experimentStatus?: EvalExperimentStatus;
+    baselineExperimentId?: string;
+    config?: string;
+    sort?: string;
+}
+export interface EvalPatchExperimentRequest {
+    experimentId: string;
+    evalExperimentPatch: EvalExperimentPatch;
+    contentEncoding?: EvalPatchExperimentContentEncodingEnum;
+}
+export interface EvalQueryExperimentTraceStatsRequest {
+    experimentId: string;
+    gatewayTraceStatsRequest: GatewayTraceStatsRequest;
+    contentEncoding?: EvalQueryExperimentTraceStatsContentEncodingEnum;
+}
+export interface EvalSearchExperimentsRequest {
+    evalArtifactSearchRequest: EvalArtifactSearchRequest;
+    contentEncoding?: EvalSearchExperimentsContentEncodingEnum;
+}
+export interface EvalSummarizeExperimentRequest {
+    experimentId: string;
+}
+/**
+ *
+ */
+export declare class ExperimentsApi extends runtime.BaseAPI {
+    /**
+     * Creates request options for evalArchiveExperiment without sending the request
+     */
+    evalArchiveExperimentRequestOpts(requestParameters: EvalArchiveExperimentRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Soft-delete the experiment and cascade its results. Housekeeping for runs that no longer matter; a completed experiment you still cite should stay.
+     * Archive an experiment
+     */
+    evalArchiveExperimentRaw(requestParameters: EvalArchiveExperimentRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<EvalExperimentArtifactView>>;
+    /**
+     * Soft-delete the experiment and cascade its results. Housekeeping for runs that no longer matter; a completed experiment you still cite should stay.
+     * Archive an experiment
+     */
+    evalArchiveExperiment(requestParameters: EvalArchiveExperimentRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<EvalExperimentArtifactView>;
+    /**
+     * Creates request options for evalCreateExperiment without sending the request
+     */
+    evalCreateExperimentRequestOpts(requestParameters: EvalCreateExperimentRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Record the start of an evaluation run: the subject under test (an agent, harness, or skill), the dataset, the configuration, and optionally the baseline it is compared against. The gateway does not execute anything — your runner or CI executes the cases and uploads results here. Created `running` by default; upload results, then finalize with one PATCH to `completed` carrying the score summary.
+     * Create an experiment
+     */
+    evalCreateExperimentRaw(requestParameters: EvalCreateExperimentRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<EvalExperimentArtifactView>>;
+    /**
+     * Record the start of an evaluation run: the subject under test (an agent, harness, or skill), the dataset, the configuration, and optionally the baseline it is compared against. The gateway does not execute anything — your runner or CI executes the cases and uploads results here. Created `running` by default; upload results, then finalize with one PATCH to `completed` carrying the score summary.
+     * Create an experiment
+     */
+    evalCreateExperiment(requestParameters: EvalCreateExperimentRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<EvalExperimentArtifactView>;
+    /**
+     * Creates request options for evalCreateExperimentResult without sending the request
+     */
+    evalCreateExperimentResultRequestOpts(requestParameters: EvalCreateExperimentResultRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Append one case\'s result to a running experiment. The parent comes from the path, so `experiment_id` is not a body field. **Create-only**: a result is reported, not edited, and a completed or failed experiment accepts no further results — that is what makes a finished experiment an immutable snapshot. `input`/`expected` are snapshot copies taken at run time, so later dataset curation cannot rewrite what this case actually ran.
+     * Report one evaluated case
+     */
+    evalCreateExperimentResultRaw(requestParameters: EvalCreateExperimentResultRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<EvalExperimentResultArtifactView>>;
+    /**
+     * Append one case\'s result to a running experiment. The parent comes from the path, so `experiment_id` is not a body field. **Create-only**: a result is reported, not edited, and a completed or failed experiment accepts no further results — that is what makes a finished experiment an immutable snapshot. `input`/`expected` are snapshot copies taken at run time, so later dataset curation cannot rewrite what this case actually ran.
+     * Report one evaluated case
+     */
+    evalCreateExperimentResult(requestParameters: EvalCreateExperimentResultRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<EvalExperimentResultArtifactView>;
+    /**
+     * Creates request options for evalCreateExperimentResultsBulk without sending the request
+     */
+    evalCreateExperimentResultsBulkRequestOpts(requestParameters: EvalCreateExperimentResultsBulkRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Create up to 100 results in one call — how a runner reports as cases finish. **207 Multi-Status**: one index-aligned slot per submitted item, so a single rejected case does not fail the batch.
+     * Report many evaluated cases
+     */
+    evalCreateExperimentResultsBulkRaw(requestParameters: EvalCreateExperimentResultsBulkRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<EvalBulkExperimentResultResponse>>;
+    /**
+     * Create up to 100 results in one call — how a runner reports as cases finish. **207 Multi-Status**: one index-aligned slot per submitted item, so a single rejected case does not fail the batch.
+     * Report many evaluated cases
+     */
+    evalCreateExperimentResultsBulk(requestParameters: EvalCreateExperimentResultsBulkRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<EvalBulkExperimentResultResponse>;
+    /**
+     * Creates request options for evalGetExperiment without sending the request
+     */
+    evalGetExperimentRequestOpts(requestParameters: EvalGetExperimentRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Read one experiment.
+     * Get an experiment
+     */
+    evalGetExperimentRaw(requestParameters: EvalGetExperimentRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<EvalExperimentArtifactView>>;
+    /**
+     * Read one experiment.
+     * Get an experiment
+     */
+    evalGetExperiment(requestParameters: EvalGetExperimentRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<EvalExperimentArtifactView>;
+    /**
+     * Creates request options for evalGetExperimentResult without sending the request
+     */
+    evalGetExperimentResultRequestOpts(requestParameters: EvalGetExperimentResultRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Read one evaluated case with its full typed metadata — the input/expected snapshot, the output, the scores, and the session/trace join keys. `experiment_id` addresses the result within its experiment; the result\'s own handle is what resolves it, and the path\'s experiment is **not** currently checked against the result\'s own — see the open item in eval.md.
+     * Get one result
+     */
+    evalGetExperimentResultRaw(requestParameters: EvalGetExperimentResultRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<EvalExperimentResultArtifactView>>;
+    /**
+     * Read one evaluated case with its full typed metadata — the input/expected snapshot, the output, the scores, and the session/trace join keys. `experiment_id` addresses the result within its experiment; the result\'s own handle is what resolves it, and the path\'s experiment is **not** currently checked against the result\'s own — see the open item in eval.md.
+     * Get one result
+     */
+    evalGetExperimentResult(requestParameters: EvalGetExperimentResultRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<EvalExperimentResultArtifactView>;
+    /**
+     * Creates request options for evalGetExperimentTrace without sending the request
+     */
+    evalGetExperimentTraceRequestOpts(requestParameters: EvalGetExperimentTraceRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Use this operation to read one of an experiment\'s traces.
+     * Read one of an experiment\'s traces
+     */
+    evalGetExperimentTraceRaw(requestParameters: EvalGetExperimentTraceRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<TraceDetailView>>;
+    /**
+     * Use this operation to read one of an experiment\'s traces.
+     * Read one of an experiment\'s traces
+     */
+    evalGetExperimentTrace(requestParameters: EvalGetExperimentTraceRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<TraceDetailView>;
+    /**
+     * Creates request options for evalListExperimentResults without sending the request
+     */
+    evalListExperimentResultsRequestOpts(requestParameters: EvalListExperimentResultsRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Page the experiment\'s evaluated cases, each with its typed metadata. Filter by `dataset_record_id` to join one case across two experiments over the same dataset.
+     * List an experiment\'s results
+     */
+    evalListExperimentResultsRaw(requestParameters: EvalListExperimentResultsRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<EvalPageExperimentResultArtifactView>>;
+    /**
+     * Page the experiment\'s evaluated cases, each with its typed metadata. Filter by `dataset_record_id` to join one case across two experiments over the same dataset.
+     * List an experiment\'s results
+     */
+    evalListExperimentResults(requestParameters: EvalListExperimentResultsRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<EvalPageExperimentResultArtifactView>;
+    /**
+     * Creates request options for evalListExperimentTraceSpans without sending the request
+     */
+    evalListExperimentTraceSpansRequestOpts(requestParameters: EvalListExperimentTraceSpansRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Use this operation to list spans in one of an experiment\'s traces.
+     * List spans in one of an experiment\'s traces
+     */
+    evalListExperimentTraceSpansRaw(requestParameters: EvalListExperimentTraceSpansRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<SpanListEnvelope>>;
+    /**
+     * Use this operation to list spans in one of an experiment\'s traces.
+     * List spans in one of an experiment\'s traces
+     */
+    evalListExperimentTraceSpans(requestParameters: EvalListExperimentTraceSpansRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<SpanListEnvelope>;
+    /**
+     * Creates request options for evalListExperimentTraces without sending the request
+     */
+    evalListExperimentTracesRequestOpts(requestParameters: EvalListExperimentTracesRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Page the OTel traces stamped with this experiment\'s id — every span of a bench case carries `atlan.eval.experiment_id` via SDK propagation, so a run\'s traces are findable here before any result uploads. Visible in full to every caller who can read the experiment.
+     * List an experiment\'s traces
+     */
+    evalListExperimentTracesRaw(requestParameters: EvalListExperimentTracesRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<TraceListEnvelope>>;
+    /**
+     * Page the OTel traces stamped with this experiment\'s id — every span of a bench case carries `atlan.eval.experiment_id` via SDK propagation, so a run\'s traces are findable here before any result uploads. Visible in full to every caller who can read the experiment.
+     * List an experiment\'s traces
+     */
+    evalListExperimentTraces(requestParameters: EvalListExperimentTracesRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<TraceListEnvelope>;
+    /**
+     * Creates request options for evalListExperiments without sending the request
+     */
+    evalListExperimentsRequestOpts(requestParameters: EvalListExperimentsRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Page the account\'s experiments. Filter by subject pair for one agent\'s, harness\'s, or skill\'s runs; by `dataset_id` for every run over a golden set; by `baseline_experiment_id` for everything compared against one run; or by `config` for the arms of a sweep that share a dimension — `?config=model:claude-sonnet-5,engine:duckdb` answers \"every run of this model on this engine\", which is the slice a multi-arm sweep is actually compared along.
+     * List experiments
+     */
+    evalListExperimentsRaw(requestParameters: EvalListExperimentsRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<EvalPageExperimentArtifactView>>;
+    /**
+     * Page the account\'s experiments. Filter by subject pair for one agent\'s, harness\'s, or skill\'s runs; by `dataset_id` for every run over a golden set; by `baseline_experiment_id` for everything compared against one run; or by `config` for the arms of a sweep that share a dimension — `?config=model:claude-sonnet-5,engine:duckdb` answers \"every run of this model on this engine\", which is the slice a multi-arm sweep is actually compared along.
+     * List experiments
+     */
+    evalListExperiments(requestParameters?: EvalListExperimentsRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<EvalPageExperimentArtifactView>;
+    /**
+     * Creates request options for evalPatchExperiment without sending the request
+     */
+    evalPatchExperimentRequestOpts(requestParameters: EvalPatchExperimentRequest): Promise<runtime.RequestOpts>;
+    /**
+     * The finalize write: move `experiment_status` to `completed` or `failed` and record the score `summary`. The run-defining fields — dataset, subject, config, baseline — are not patchable; they describe a run that already happened. A terminal experiment is frozen entirely.
+     * Finalize or annotate an experiment
+     */
+    evalPatchExperimentRaw(requestParameters: EvalPatchExperimentRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<EvalExperimentArtifactView>>;
+    /**
+     * The finalize write: move `experiment_status` to `completed` or `failed` and record the score `summary`. The run-defining fields — dataset, subject, config, baseline — are not patchable; they describe a run that already happened. A terminal experiment is frozen entirely.
+     * Finalize or annotate an experiment
+     */
+    evalPatchExperiment(requestParameters: EvalPatchExperimentRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<EvalExperimentArtifactView>;
+    /**
+     * Creates request options for evalQueryExperimentTraceStats without sending the request
+     */
+    evalQueryExperimentTraceStatsRequestOpts(requestParameters: EvalQueryExperimentTraceStatsRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Use this operation to query statistics over an experiment\'s traces.
+     * Query statistics over an experiment\'s traces
+     */
+    evalQueryExperimentTraceStatsRaw(requestParameters: EvalQueryExperimentTraceStatsRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<GatewayTraceStatsEnvelope>>;
+    /**
+     * Use this operation to query statistics over an experiment\'s traces.
+     * Query statistics over an experiment\'s traces
+     */
+    evalQueryExperimentTraceStats(requestParameters: EvalQueryExperimentTraceStatsRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<GatewayTraceStatsEnvelope>;
+    /**
+     * Creates request options for evalSearchExperiments without sending the request
+     */
+    evalSearchExperimentsRequestOpts(requestParameters: EvalSearchExperimentsRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Full-text discovery over experiment names and descriptions. Results are summary-only and carry no `metadata`.
+     * Search experiments
+     */
+    evalSearchExperimentsRaw(requestParameters: EvalSearchExperimentsRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<EvalPageExperimentArtifactView>>;
+    /**
+     * Full-text discovery over experiment names and descriptions. Results are summary-only and carry no `metadata`.
+     * Search experiments
+     */
+    evalSearchExperiments(requestParameters: EvalSearchExperimentsRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<EvalPageExperimentArtifactView>;
+    /**
+     * Creates request options for evalSummarizeExperiment without sending the request
+     */
+    evalSummarizeExperimentRequestOpts(requestParameters: EvalSummarizeExperimentRequest): Promise<runtime.RequestOpts>;
+    /**
+     * Use this operation to roll the run\'s score spans up into `summary`.
+     * Roll the run\'s score spans up into `summary`
+     */
+    evalSummarizeExperimentRaw(requestParameters: EvalSummarizeExperimentRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<EvalExperimentArtifactView>>;
+    /**
+     * Use this operation to roll the run\'s score spans up into `summary`.
+     * Roll the run\'s score spans up into `summary`
+     */
+    evalSummarizeExperiment(requestParameters: EvalSummarizeExperimentRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<EvalExperimentArtifactView>;
+}
+/**
+  * @export
+  * @enum {string}
+  */
+export declare enum EvalCreateExperimentContentEncodingEnum {
+    Base64 = "base64",
+    GzipBase64 = "gzip, base64"
+}
+/**
+  * @export
+  * @enum {string}
+  */
+export declare enum EvalCreateExperimentResultContentEncodingEnum {
+    Base64 = "base64",
+    GzipBase64 = "gzip, base64"
+}
+/**
+  * @export
+  * @enum {string}
+  */
+export declare enum EvalCreateExperimentResultsBulkContentEncodingEnum {
+    Base64 = "base64",
+    GzipBase64 = "gzip, base64"
+}
+/**
+  * @export
+  * @enum {string}
+  */
+export declare enum EvalPatchExperimentContentEncodingEnum {
+    Base64 = "base64",
+    GzipBase64 = "gzip, base64"
+}
+/**
+  * @export
+  * @enum {string}
+  */
+export declare enum EvalQueryExperimentTraceStatsContentEncodingEnum {
+    Base64 = "base64",
+    GzipBase64 = "gzip, base64"
+}
+/**
+  * @export
+  * @enum {string}
+  */
+export declare enum EvalSearchExperimentsContentEncodingEnum {
+    Base64 = "base64",
+    GzipBase64 = "gzip, base64"
+}