@iblai/data-layer 1.9.2 → 1.9.4

package/dist/src/features/evaluations/constants.d.ts

@@ -0,0 +1,102 @@
+import { SERVICES } from '@data-layer/constants';
+export declare const EVALUATIONS_REDUCER_PATH = "evaluationsApiSlice";
+export declare const EVALUATIONS_ENDPOINTS: {
+    readonly GET_TASK: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, taskId: string) => string;
+    };
+    readonly LIST_DATASETS: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string) => string;
+    };
+    readonly CREATE_DATASET: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string) => string;
+    };
+    readonly GET_DATASET: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string) => string;
+    };
+    readonly LIST_DATASET_ITEMS: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string) => string;
+    };
+    readonly ADD_DATASET_ITEMS: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string) => string;
+    };
+    readonly UPLOAD_DATASET_CSV: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string) => string;
+    };
+    readonly UPDATE_DATASET_ITEM: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string, itemId: string) => string;
+    };
+    readonly DELETE_DATASET_ITEM: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string, itemId: string) => string;
+    };
+    readonly LIST_RUNS: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string) => string;
+    };
+    readonly START_RUN: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string) => string;
+    };
+    readonly GET_RUN: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string, runName: string) => string;
+    };
+    readonly EXPORT_RUN: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string, runName: string) => string;
+    };
+    readonly DELETE_RUN: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string, runName: string) => string;
+    };
+    readonly EVALUATE_RUN: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string, runName: string) => string;
+    };
+    readonly LIST_RUN_JUDGES: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, datasetName: string, runName: string) => string;
+    };
+    readonly LIST_JUDGES: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string) => string;
+    };
+    readonly LIST_SCORES: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string) => string;
+    };
+    readonly CREATE_SCORE: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string) => string;
+    };
+    readonly DELETE_SCORE: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string, scoreId: string) => string;
+    };
+    readonly LIST_SCORE_CONFIGS: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string) => string;
+    };
+    readonly CREATE_SCORE_CONFIG: {
+        readonly service: SERVICES.AXD;
+        readonly path: (org: string, userId: string) => string;
+    };
+};
+export declare const EVALUATIONS_TAGS: {
+    readonly DATASETS: "EVAL_DATASETS";
+    readonly DATASET: "EVAL_DATASET";
+    readonly DATASET_ITEMS: "EVAL_DATASET_ITEMS";
+    readonly RUNS: "EVAL_RUNS";
+    readonly RUN: "EVAL_RUN";
+    readonly SCORES: "EVAL_SCORES";
+    readonly SCORE_CONFIGS: "EVAL_SCORE_CONFIGS";
+    readonly JUDGES: "EVAL_JUDGES";
+};

package/dist/src/features/evaluations/types.d.ts

@@ -0,0 +1,285 @@
+export type EvalScoreDataType = 'NUMERIC' | 'BOOLEAN' | 'CATEGORICAL';
+export type EvalDatasetItemStatus = 'ACTIVE' | 'ARCHIVED';
+/** Lifecycle status surfaced on runs and judge task records. */
+export type EvalTaskStatusValue = 'pending' | 'in_progress' | 'completed' | 'failed' | (string & {});
+/**
+ * Standard DRF page-number pagination envelope.
+ * `next` / `previous` are absolute URLs (or null at the boundaries).
+ */
+export interface EvalPaginatedResponse<T> {
+    count: number;
+    next: string | null;
+    previous: string | null;
+    results: T[];
+}
+/** Total page count derived from `count` and the requested page size. */
+export declare function evalTotalPages(count: number | undefined, limit: number): number;
+export interface EvalUserStamp {
+    username?: string;
+    user_email?: string;
+}
+export interface EvalDataset extends EvalUserStamp {
+    name: string;
+    description: string;
+    metadata: Record<string, unknown>;
+    created_at: string;
+    updated_at: string;
+}
+/** Dataset item `input` may be a bare string or a structured object. */
+export type EvalItemInput = string | Record<string, unknown>;
+export interface EvalDatasetItem {
+    id: string;
+    input: EvalItemInput;
+    expected_output: string | null;
+    metadata: Record<string, unknown>;
+    status: EvalDatasetItemStatus;
+    source_trace_id: string | null;
+    source_observation_id: string | null;
+    created_at: string;
+    updated_at: string;
+    /** Present when `include_trace=true` and the item has a source trace. */
+    trace_input?: EvalItemInput | null;
+    trace_output?: string | null;
+}
+export interface EvalScore extends EvalUserStamp {
+    id: string;
+    name: string;
+    value: number;
+    string_value: string | null;
+    data_type: EvalScoreDataType;
+    comment: string | null;
+    trace_id: string;
+    observation_id: string | null;
+    dataset_run_id: string | null;
+    created_at: string;
+}
+/** One (run, dataset_item) row, fully expanded on the run-detail endpoint. */
+export interface EvalDatasetRunItem {
+    id: string;
+    dataset_item_id: string;
+    trace_id: string;
+    observation_id: string | null;
+    created_at: string;
+    input: EvalItemInput;
+    expected_output: EvalItemInput | null;
+    actual_output: EvalItemInput | null;
+    error: EvalItemInput | null;
+    scores: EvalScore[];
+}
+/** Judge task record (LLM-as-Judge job lifecycle). */
+export interface EvalJudgeTask extends EvalUserStamp {
+    id: number;
+    dataset_name: string;
+    run_name: string;
+    score_name: string;
+    status: EvalTaskStatusValue;
+    task_id: string;
+    request_metadata: {
+        criteria?: string;
+        llm_provider?: string;
+        llm_name?: string;
+        max_concurrency?: number;
+        [key: string]: unknown;
+    };
+    error_message: string;
+    created_at: string;
+    started_at: string | null;
+    completed_at: string | null;
+}
+/** Run summary as returned by the runs list (merges pending + persisted). */
+export interface EvalDatasetRun extends EvalUserStamp {
+    /** Empty string for a not-yet-persisted (pending/in_progress) run. */
+    id: string;
+    name: string;
+    metadata: Record<string, unknown>;
+    created_at: string;
+    updated_at: string;
+    status: EvalTaskStatusValue;
+    /** Present on pending/in-flight entries. */
+    task_id?: string;
+    task_record_id?: number;
+}
+/** Run detail: every item fully expanded plus in-flight judge records. */
+export interface EvalDatasetRunDetail extends EvalDatasetRun {
+    dataset_run_items: EvalDatasetRunItem[];
+    pending_judges: EvalJudgeTask[];
+}
+export interface EvalScoreConfigCategory {
+    value: number;
+    label: string;
+}
+export interface EvalScoreConfig {
+    id: string;
+    name: string;
+    data_type: EvalScoreDataType;
+    min_value: number | null;
+    max_value: number | null;
+    categories: EvalScoreConfigCategory[] | null;
+    description: string;
+    is_archived: boolean;
+    created_at: string;
+}
+export interface EvalPaginationParams {
+    page?: number;
+    limit?: number;
+}
+export interface EvalTenantUserArgs {
+    org: string;
+    userId: string;
+}
+export interface ListDatasetsArgs extends EvalTenantUserArgs, EvalPaginationParams {
+    /** Case-insensitive substring match on the dataset name. */
+    name?: string;
+    /** Exact email match on the creator. */
+    user_email?: string;
+}
+export interface CreateDatasetArgs extends EvalTenantUserArgs {
+    body: {
+        name: string;
+        description?: string;
+        metadata?: Record<string, unknown>;
+    };
+}
+export interface DatasetNameArgs extends EvalTenantUserArgs {
+    datasetName: string;
+}
+export interface ListDatasetItemsArgs extends DatasetNameArgs, EvalPaginationParams {
+    /** Attach the linked trace's input/output to items with a source trace. */
+    include_trace?: boolean;
+}
+export interface AddDatasetItemsArgs extends DatasetNameArgs {
+    body: {
+        items: Array<{
+            input: string;
+            expected_output?: string | null;
+        }>;
+    } | {
+        trace_ids: string[];
+    };
+}
+export interface UploadDatasetCsvArgs extends DatasetNameArgs {
+    formData: FormData;
+}
+export interface UpdateDatasetItemArgs extends DatasetNameArgs {
+    itemId: string;
+    body: Partial<{
+        input: string;
+        expected_output: string | null;
+        metadata: Record<string, unknown>;
+        status: EvalDatasetItemStatus;
+    }>;
+}
+export interface DeleteDatasetItemArgs extends DatasetNameArgs {
+    itemId: string;
+}
+export interface ListRunsArgs extends DatasetNameArgs, EvalPaginationParams {
+    /** Filter by the initiator's email. */
+    user_email?: string;
+}
+export interface StartRunArgs extends DatasetNameArgs {
+    body: {
+        mentor_unique_id: string;
+        run_name?: string;
+        metadata?: Record<string, unknown>;
+    };
+}
+export interface RunNameArgs extends DatasetNameArgs {
+    runName: string;
+}
+export interface EvaluateRunArgs extends RunNameArgs {
+    body: {
+        criteria: string;
+        score_name: string;
+        llm_provider?: string;
+        llm_name?: string;
+        max_concurrency?: number;
+    };
+}
+export interface ListRunJudgesArgs extends RunNameArgs, EvalPaginationParams {
+    status?: EvalTaskStatusValue;
+    user_email?: string;
+}
+export interface ListJudgesArgs extends EvalTenantUserArgs, EvalPaginationParams {
+    status?: EvalTaskStatusValue;
+    user_email?: string;
+    dataset_name?: string;
+}
+export interface ListScoresArgs extends EvalTenantUserArgs, EvalPaginationParams {
+    dataset_run_id?: string;
+    trace_id?: string;
+    name?: string;
+}
+export interface CreateScoreArgs extends EvalTenantUserArgs {
+    /**
+     * Cache hint, not sent to the API: when set (with `runName`), the cached
+     * run detail is patched in place on success so the new score renders
+     * immediately — the eval store ingests scores asynchronously, so an
+     * immediate refetch would return the run without it.
+     */
+    datasetName?: string;
+    /** Cache hint, see `datasetName`. */
+    runName?: string;
+    body: {
+        trace_id: string;
+        name: string;
+        value: number;
+        data_type?: EvalScoreDataType;
+        comment?: string;
+        observation_id?: string;
+        config_id?: string;
+        dataset_run_id?: string;
+    };
+}
+export interface DeleteScoreArgs extends EvalTenantUserArgs {
+    scoreId: string;
+    /** Cache hint, see `CreateScoreArgs.datasetName`. */
+    datasetName?: string;
+    /** Cache hint, see `CreateScoreArgs.datasetName`. */
+    runName?: string;
+}
+export interface ListScoreConfigsArgs extends EvalTenantUserArgs, EvalPaginationParams {
+}
+export interface CreateScoreConfigArgs extends EvalTenantUserArgs {
+    body: {
+        name: string;
+        data_type: EvalScoreDataType;
+        min_value?: number;
+        max_value?: number;
+        categories?: EvalScoreConfigCategory[];
+        description?: string;
+    };
+}
+/** 202 response when dispatching a run — a pending task record. */
+export interface StartRunResponse extends EvalUserStamp {
+    id: string;
+    name: string;
+    metadata: Record<string, unknown>;
+    created_at: string;
+    updated_at: string;
+    status: EvalTaskStatusValue;
+    task_id: string;
+    task_record_id: number;
+}
+/** 202 response when dispatching LLM-as-Judge — a pending judge record. */
+export type EvaluateRunResponse = EvalJudgeTask;
+export interface AddDatasetItemsResponse {
+    created: number;
+    items: EvalDatasetItem[];
+}
+export interface CreateScoreResponse extends EvalUserStamp {
+    status: string;
+    name: string;
+    value: number;
+}
+export interface GetTaskArgs extends EvalTenantUserArgs {
+    taskId: string;
+}
+export interface EvalTaskStatus {
+    task_id: string;
+    status: string;
+    result?: unknown;
+    error?: string | null;
+    date_done?: string | null;
+    traceback?: string | null;
+    [key: string]: unknown;
+}

package/dist/src/features/mentor/privacy.d.ts

@@ -14,7 +14,7 @@
  * `useGetMentorPublicSettingsQuery`) — this file just adds the typed
  * surface for the new fields.
  */
-export declare const PRIVACY_ACTIONS: readonly ["redact", "mask", "block"];
+export declare const PRIVACY_ACTIONS: readonly ["allow", "redact", "mask", "block"];
 export type PrivacyAction = (typeof PRIVACY_ACTIONS)[number];
 export declare const PRIVACY_ENTITY_TYPES: readonly ["PERSON", "EMAIL_ADDRESS", "PHONE_NUMBER", "US_SSN", "CREDIT_CARD", "LOCATION", "DATE_TIME", "US_PASSPORT", "US_DRIVER_LICENSE", "IP_ADDRESS", "IBAN_CODE", "MEDICAL_LICENSE", "US_BANK_NUMBER"];
 export type PrivacyEntityType = (typeof PRIVACY_ENTITY_TYPES)[number];

package/dist/src/index.d.ts

@@ -112,6 +112,9 @@ export * from './features/workflows/constants';
 export * from './features/audit-logs/api-slice';
 export * from './features/audit-logs/constants';
 export * from './features/audit-logs/types';
+export * from './features/evaluations/api-slice';
+export * from './features/evaluations/constants';
+export * from './features/evaluations/types';
 export * from './features/watched-groups/api-slice';
 export * from './features/watched-groups/constants';
 export * from './features/watched-groups/types';