openlayer 0.0.0 → 0.0.1-alpha.1

package/src/resources/commits/test-results.ts

@@ -0,0 +1,284 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import * as Core from '../../core';
+import { APIResource } from '../../resource';
+import { isRequestOptions } from '../../core';
+import * as TestResultsAPI from './test-results';
+
+export class TestResults extends APIResource {
+  /**
+   * List the test results for a commit (project version).
+   */
+  list(
+    id: string,
+    query?: TestResultListParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<TestResultListResponse>;
+  list(id: string, options?: Core.RequestOptions): Core.APIPromise<TestResultListResponse>;
+  list(
+    id: string,
+    query: TestResultListParams | Core.RequestOptions = {},
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<TestResultListResponse> {
+    if (isRequestOptions(query)) {
+      return this.list(id, {}, query);
+    }
+    return this._client.get(`/versions/${id}/results`, { query, ...options });
+  }
+}
+
+export interface TestResultListResponse {
+  _meta: TestResultListResponse._Meta;
+
+  items: Array<TestResultListResponse.Item>;
+}
+
+export namespace TestResultListResponse {
+  export interface _Meta {
+    /**
+     * The current page.
+     */
+    page: number;
+
+    /**
+     * The number of items per page.
+     */
+    perPage: number;
+
+    /**
+     * The total number of items.
+     */
+    totalItems: number;
+
+    /**
+     * The total number of pages.
+     */
+    totalPages: number;
+  }
+
+  export interface Item {
+    /**
+     * Project version (commit) id.
+     */
+    id: string;
+
+    /**
+     * The creation date.
+     */
+    dateCreated: string;
+
+    /**
+     * The data end date.
+     */
+    dateDataEnds: string | null;
+
+    /**
+     * The data start date.
+     */
+    dateDataStarts: string | null;
+
+    /**
+     * The last updated date.
+     */
+    dateUpdated: string;
+
+    /**
+     * The inference pipeline id.
+     */
+    inferencePipelineId: string | null;
+
+    /**
+     * The project version (commit) id.
+     */
+    projectVersionId: string | null;
+
+    /**
+     * The status of the test.
+     */
+    status: 'running' | 'passing' | 'failing' | 'skipped' | 'error';
+
+    /**
+     * The status message.
+     */
+    statusMessage: string | null;
+
+    goal?: Item.Goal;
+
+    /**
+     * The test id.
+     */
+    goalId?: string | null;
+  }
+
+  export namespace Item {
+    export interface Goal {
+      /**
+       * The test id.
+       */
+      id: string;
+
+      /**
+       * The number of comments on the test.
+       */
+      commentCount: number;
+
+      /**
+       * The test creator id.
+       */
+      creatorId: string | null;
+
+      /**
+       * The date the test was archived.
+       */
+      dateArchived: string | null;
+
+      /**
+       * The creation date.
+       */
+      dateCreated: string;
+
+      /**
+       * The last updated date.
+       */
+      dateUpdated: string;
+
+      /**
+       * The test description.
+       */
+      description: unknown | null;
+
+      /**
+       * The test name.
+       */
+      name: string;
+
+      /**
+       * The test number.
+       */
+      number: number;
+
+      /**
+       * The project version (commit) id where the test was created.
+       */
+      originProjectVersionId: string | null;
+
+      /**
+       * The test subtype.
+       */
+      subtype: string;
+
+      /**
+       * Whether the test is suggested or user-created.
+       */
+      suggested: boolean;
+
+      thresholds: Array<Goal.Threshold>;
+
+      /**
+       * The test type.
+       */
+      type: string;
+
+      /**
+       * Whether the test is archived.
+       */
+      archived?: boolean;
+
+      /**
+       * The delay window in seconds. Only applies to tests that use production data.
+       */
+      delayWindow?: number | null;
+
+      /**
+       * The evaluation window in seconds. Only applies to tests that use production
+       * data.
+       */
+      evaluationWindow?: number | null;
+
+      /**
+       * Whether the test uses an ML model.
+       */
+      usesMlModel?: boolean;
+
+      /**
+       * Whether the test uses production data (monitoring mode only).
+       */
+      usesProductionData?: boolean;
+
+      /**
+       * Whether the test uses a reference dataset (monitoring mode only).
+       */
+      usesReferenceDataset?: boolean;
+
+      /**
+       * Whether the test uses a training dataset.
+       */
+      usesTrainingDataset?: boolean;
+
+      /**
+       * Whether the test uses a validation dataset.
+       */
+      usesValidationDataset?: boolean;
+    }
+
+    export namespace Goal {
+      export interface Threshold {
+        /**
+         * The insight name to be evaluated.
+         */
+        insightName?: string;
+
+        insightParameters?: Array<unknown>;
+
+        /**
+         * The measurement to be evaluated.
+         */
+        measurement?: string;
+
+        /**
+         * The operator to be used for the evaluation.
+         */
+        operator?: string;
+
+        /**
+         * The value to be compared.
+         */
+        value?: number | boolean | string | Array<string>;
+      }
+    }
+  }
+}
+
+export interface TestResultListParams {
+  /**
+   * Include archived goals.
+   */
+  includeArchived?: boolean;
+
+  /**
+   * The page to return in a paginated query.
+   */
+  page?: number;
+
+  /**
+   * Maximum number of items to return per page.
+   */
+  perPage?: number;
+
+  /**
+   * Filter list of test results by status. Available statuses are `running`,
+   * `passing`, `failing`, `skipped`, and `error`.
+   */
+  status?: 'running' | 'passing' | 'failing' | 'skipped' | 'error';
+
+  /**
+   * Filter objects by test type. Available types are `integrity`, `consistency`,
+   * `performance`, `fairness`, and `robustness`.
+   */
+  type?: 'integrity' | 'consistency' | 'performance' | 'fairness' | 'robustness';
+}
+
+export namespace TestResults {
+  export import TestResultListResponse = TestResultsAPI.TestResultListResponse;
+  export import TestResultListParams = TestResultsAPI.TestResultListParams;
+}

package/src/resources/index.ts

@@ -0,0 +1,5 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export { Commits } from './commits/commits';
+export { InferencePipelines } from './inference-pipelines/inference-pipelines';
+export { ProjectListResponse, ProjectListParams, Projects } from './projects/projects';

package/src/resources/inference-pipelines/data.ts

@@ -0,0 +1,285 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import * as Core from '../../core';
+import { APIResource } from '../../resource';
+import * as DataAPI from './data';
+
+export class Data extends APIResource {
+  /**
+   * Stream production data to an inference pipeline in Openlayer.
+   */
+  stream(
+    id: string,
+    body: DataStreamParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<DataStreamResponse> {
+    return this._client.post(`/inference-pipelines/${id}/data-stream`, { body, ...options });
+  }
+}
+
+export interface DataStreamResponse {
+  success: true;
+}
+
+export interface DataStreamParams {
+  /**
+   * Configuration for the data stream. Depends on your **Openlayer project task
+   * type**.
+   */
+  config:
+    | DataStreamParams.LlmData
+    | DataStreamParams.TabularClassificationData
+    | DataStreamParams.TabularRegressionData
+    | DataStreamParams.TextClassificationData;
+
+  /**
+   * A list of entries that represent rows of a csv file
+   */
+  rows: Array<Record<string, unknown>>;
+}
+
+export namespace DataStreamParams {
+  export interface LlmData {
+    /**
+     * Name of the column with the model outputs.
+     */
+    outputColumnName: string;
+
+    /**
+     * Name of the column with the context retrieved. Applies to RAG use cases.
+     * Providing the context enables RAG-specific metrics.
+     */
+    contextColumnName?: string;
+
+    /**
+     * Name of the column with the cost associated with each row.
+     */
+    costColumnName?: string;
+
+    /**
+     * Name of the column with the ground truths.
+     */
+    groundTruthColumnName?: string;
+
+    /**
+     * Name of the column with the inference ids. This is useful if you want to update
+     * rows at a later point in time. If not provided, a unique id is generated by
+     * Openlayer.
+     */
+    inferenceIdColumnName?: string;
+
+    /**
+     * Array of input variable names. Each input variable should be a dataset column.
+     */
+    inputVariableNames?: Array<string>;
+
+    /**
+     * Name of the column with the latencies.
+     */
+    latencyColumnName?: string;
+
+    /**
+     * Object with metadata.
+     */
+    metadata?: unknown;
+
+    /**
+     * Name of the column with the total number of tokens.
+     */
+    numOfTokenColumnName?: string | null;
+
+    /**
+     * Prompt for the LLM.
+     */
+    prompt?: Array<LlmData.Prompt>;
+
+    /**
+     * Name of the column with the questions. Applies to RAG use cases. Providing the
+     * question enables RAG-specific metrics.
+     */
+    questionColumnName?: string;
+
+    /**
+     * Name of the column with the timestamps. Timestamps must be in UNIX sec format.
+     * If not provided, the upload timestamp is used.
+     */
+    timestampColumnName?: string;
+  }
+
+  export namespace LlmData {
+    export interface Prompt {
+      /**
+       * Content of the prompt.
+       */
+      content?: string;
+
+      /**
+       * Role of the prompt.
+       */
+      role?: string;
+    }
+  }
+
+  export interface TabularClassificationData {
+    /**
+     * List of class names indexed by label integer in the dataset. E.g. ["Retained",
+     * "Exited"] when 0, 1 are in your label column.
+     */
+    classNames: Array<string>;
+
+    /**
+     * Array with the names of all categorical features in the dataset. E.g. ["Gender",
+     * "Geography"].
+     */
+    categoricalFeatureNames?: Array<string>;
+
+    /**
+     * Array with all input feature names.
+     */
+    featureNames?: Array<string>;
+
+    /**
+     * Name of the column with the inference ids. This is useful if you want to update
+     * rows at a later point in time. If not provided, a unique id is generated by
+     * Openlayer.
+     */
+    inferenceIdColumnName?: string;
+
+    /**
+     * Name of the column with the labels. The data in this column must be
+     * **zero-indexed integers**, matching the list provided in `classNames`.
+     */
+    labelColumnName?: string;
+
+    /**
+     * Name of the column with the latencies.
+     */
+    latencyColumnName?: string;
+
+    /**
+     * Object with metadata.
+     */
+    metadata?: unknown;
+
+    /**
+     * Name of the column with the model's predictions as **zero-indexed integers**.
+     */
+    predictionsColumnName?: string;
+
+    /**
+     * Name of the column with the model's predictions as **lists of class
+     * probabilities**.
+     */
+    predictionScoresColumnName?: string;
+
+    /**
+     * Name of the column with the timestamps. Timestamps must be in UNIX sec format.
+     * If not provided, the upload timestamp is used.
+     */
+    timestampColumnName?: string;
+  }
+
+  export interface TabularRegressionData {
+    /**
+     * Array with the names of all categorical features in the dataset. E.g. ["Gender",
+     * "Geography"].
+     */
+    categoricalFeatureNames?: Array<string>;
+
+    /**
+     * Array with all input feature names.
+     */
+    featureNames?: Array<string>;
+
+    /**
+     * Name of the column with the inference ids. This is useful if you want to update
+     * rows at a later point in time. If not provided, a unique id is generated by
+     * Openlayer.
+     */
+    inferenceIdColumnName?: string;
+
+    /**
+     * Name of the column with the latencies.
+     */
+    latencyColumnName?: string;
+
+    /**
+     * Object with metadata.
+     */
+    metadata?: unknown;
+
+    /**
+     * Name of the column with the model's predictions.
+     */
+    predictionsColumnName?: string;
+
+    /**
+     * Name of the column with the targets (ground truth values).
+     */
+    targetColumnName?: string;
+
+    /**
+     * Name of the column with the timestamps. Timestamps must be in UNIX sec format.
+     * If not provided, the upload timestamp is used.
+     */
+    timestampColumnName?: string;
+  }
+
+  export interface TextClassificationData {
+    /**
+     * List of class names indexed by label integer in the dataset. E.g. ["Retained",
+     * "Exited"] when 0, 1 are in your label column.
+     */
+    classNames: Array<string>;
+
+    /**
+     * Name of the column with the inference ids. This is useful if you want to update
+     * rows at a later point in time. If not provided, a unique id is generated by
+     * Openlayer.
+     */
+    inferenceIdColumnName?: string;
+
+    /**
+     * Name of the column with the labels. The data in this column must be
+     * **zero-indexed integers**, matching the list provided in `classNames`.
+     */
+    labelColumnName?: string;
+
+    /**
+     * Name of the column with the latencies.
+     */
+    latencyColumnName?: string;
+
+    /**
+     * Object with metadata.
+     */
+    metadata?: unknown;
+
+    /**
+     * Name of the column with the model's predictions as **zero-indexed integers**.
+     */
+    predictionsColumnName?: string;
+
+    /**
+     * Name of the column with the model's predictions as **lists of class
+     * probabilities**.
+     */
+    predictionScoresColumnName?: string;
+
+    /**
+     * Name of the column with the text data.
+     */
+    textColumnName?: string;
+
+    /**
+     * Name of the column with the timestamps. Timestamps must be in UNIX sec format.
+     * If not provided, the upload timestamp is used.
+     */
+    timestampColumnName?: string;
+  }
+}
+
+export namespace Data {
+  export import DataStreamResponse = DataAPI.DataStreamResponse;
+  export import DataStreamParams = DataAPI.DataStreamParams;
+}

package/src/resources/inference-pipelines/index.ts

@@ -0,0 +1,5 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export { DataStreamResponse, DataStreamParams, Data } from './data';
+export { InferencePipelines } from './inference-pipelines';
+export { TestResultListResponse, TestResultListParams, TestResults } from './test-results';

package/src/resources/inference-pipelines/inference-pipelines.ts

@@ -0,0 +1,19 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../../resource';
+import * as DataAPI from './data';
+import * as TestResultsAPI from './test-results';
+
+export class InferencePipelines extends APIResource {
+  data: DataAPI.Data = new DataAPI.Data(this._client);
+  testResults: TestResultsAPI.TestResults = new TestResultsAPI.TestResults(this._client);
+}
+
+export namespace InferencePipelines {
+  export import Data = DataAPI.Data;
+  export import DataStreamResponse = DataAPI.DataStreamResponse;
+  export import DataStreamParams = DataAPI.DataStreamParams;
+  export import TestResults = TestResultsAPI.TestResults;
+  export import TestResultListResponse = TestResultsAPI.TestResultListResponse;
+  export import TestResultListParams = TestResultsAPI.TestResultListParams;
+}