notdiamond 2.0.0-rc2 → 2.0.0-rc5

package/src/resources/prompt/prompt.ts

@@ -0,0 +1,398 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../../core/resource';
+import * as PromptAPI from './prompt';
+import * as AdaptAPI from './adapt';
+import {
+  Adapt,
+  AdaptCreateParams,
+  AdaptCreateResponse,
+  AdaptGetCostsResponse,
+  GoldenRecord,
+  RequestProvider,
+} from './adapt';
+import { APIPromise } from '../../core/api-promise';
+import { RequestOptions } from '../../internal/request-options';
+import { path } from '../../internal/utils/path';
+
+export class Prompt extends APIResource {
+  adapt: AdaptAPI.Adapt = new AdaptAPI.Adapt(this._client);
+
+  /**
+   * Retrieve the complete results of a prompt adaptation run, including optimized
+   * prompts for all target models.
+   *
+   * This endpoint returns the adapted prompts and evaluation metrics for each target
+   * model in your adaptation request. Call this endpoint after the adaptation status
+   * is 'completed' to get your optimized prompts.
+   *
+   * **Response Structure:**
+   *
+   * - **origin_model**: Baseline performance of your original prompt on the origin
+   *   model
+   *   - Includes: system_prompt, user_message_template, score, evaluation metrics,
+   *     cost
+   * - **target_models**: Array of results for each target model
+   *   - Includes: optimized system_prompt, user_message_template, template_fields
+   *   - pre_optimization_score: Performance before adaptation
+   *   - post_optimization_score: Performance after adaptation
+   *   - Evaluation metrics and cost information
+   *
+   * **Using Adapted Prompts:**
+   *
+   * 1. Extract the `system_prompt` and `user_message_template` from each target
+   *    model result
+   * 2. Use `user_message_template_fields` to know which fields to substitute
+   * 3. Apply the optimized prompts when calling the respective target models
+   * 4. Compare pre/post optimization scores to see improvement
+   *
+   * **Evaluation Scores:**
+   *
+   * - Scores range from 0-10 (higher is better)
+   * - Compare origin_model score with target_models pre_optimization_score for
+   *   baseline
+   * - Compare pre_optimization_score with post_optimization_score to see improvement
+   *   from adaptation
+   * - Typical improvements range from 5-30% on evaluation metrics
+   *
+   * **Status Handling:**
+   *
+   * - If adaptation is still processing, target model results will have
+   *   `result_status: "processing"`
+   * - Only completed target models will have system_prompt and template values
+   * - Failed target models will have `result_status: "failed"` with null values
+   *
+   * **Cost Information:**
+   *
+   * - Each model result includes cost in USD for the adaptation process
+   * - Costs vary based on model pricing and number of evaluation examples
+   * - Typical range: $0.10 - $2.00 per target model
+   *
+   * **Best Practices:**
+   *
+   * 1. Wait for status 'completed' before calling this endpoint
+   * 2. Check result_status for each target model
+   * 3. Validate that post_optimization_score > pre_optimization_score
+   * 4. Save optimized prompts for production use
+   * 5. A/B test adapted prompts against originals in production
+   *
+   * @example
+   * ```ts
+   * const response = await client.prompt.getAdaptResults(
+   *   'adaptation_run_id',
+   * );
+   * ```
+   */
+  getAdaptResults(
+    adaptationRunID: string,
+    options?: RequestOptions,
+  ): APIPromise<PromptGetAdaptResultsResponse> {
+    return this._client.get(path`/v2/prompt/adaptResults/${adaptationRunID}`, options);
+  }
+
+  /**
+   * Check the status of a prompt adaptation run.
+   *
+   * Use this endpoint to poll the status of your adaptation request. Processing is
+   * asynchronous, so you'll need to check periodically until the status indicates
+   * completion.
+   *
+   * **Status Values:**
+   *
+   * - `created`: Initial state, not yet processing
+   * - `queued`: Waiting for processing capacity (check queue_position)
+   * - `processing`: Currently optimizing prompts
+   * - `completed`: All target models have been processed successfully
+   * - `failed`: One or more target models failed to process
+   *
+   * **Polling Recommendations:**
+   *
+   * - Poll every 30-60 seconds during processing
+   * - Check queue_position if status is 'queued' to estimate wait time
+   * - Stop polling once status is 'completed' or 'failed'
+   * - Use GET /v2/prompt/adaptResults to retrieve results after completion
+   *
+   * **Queue Position:**
+   *
+   * - Only present when status is 'queued'
+   * - Lower numbers mean earlier processing (position 1 is next)
+   * - Typical wait time: 1-5 minutes per position
+   *
+   * **Note:** This endpoint only returns status information. To get the actual
+   * adapted prompts and evaluation results, use GET /v2/prompt/adaptResults once
+   * status is 'completed'.
+   *
+   * @example
+   * ```ts
+   * const response = await client.prompt.getAdaptStatus(
+   *   'adaptation_run_id',
+   * );
+   * ```
+   */
+  getAdaptStatus(
+    adaptationRunID: string,
+    options?: RequestOptions,
+  ): APIPromise<PromptGetAdaptStatusResponse> {
+    return this._client.get(path`/v2/prompt/adaptStatus/${adaptationRunID}`, options);
+  }
+}
+
+/**
+ * Status enum for asynchronous jobs (prompt adaptation, custom router training,
+ * etc.).
+ *
+ * Represents the current state of a long-running operation:
+ *
+ * - **created**: Job has been initialized but not yet queued
+ * - **queued**: Job is waiting in the queue to be processed
+ * - **processing**: Job is currently being executed
+ * - **completed**: Job finished successfully and results are available
+ * - **failed**: Job encountered an error and did not complete
+ */
+export type JobStatus = 'created' | 'queued' | 'processing' | 'completed' | 'failed';
+
+/**
+ * Response model for GET /v2/prompt/adaptResults/{adaptation_run_id} endpoint.
+ *
+ * Contains the complete results of a prompt adaptation run, including optimized
+ * prompts and evaluation metrics for all target models. Use this to retrieve your
+ * adapted prompts after the adaptation status is 'completed'.
+ *
+ * The response includes:
+ *
+ * - Baseline performance of your original prompt on the origin model
+ * - Optimized prompts for each target model with pre/post optimization scores
+ * - Evaluation metrics and cost information for each model
+ */
+export interface PromptGetAdaptResultsResponse {
+  /**
+   * Unique ID for this adaptation run
+   */
+  id: string;
+
+  /**
+   * Timestamp when this adaptation run was created
+   */
+  created_at: string;
+
+  /**
+   * Overall status of the adaptation run (queued, running, completed, failed)
+   */
+  job_status: JobStatus;
+
+  /**
+   * Results for each target model with optimized prompts and improvement scores
+   */
+  target_models: Array<PromptGetAdaptResultsResponse.TargetModel>;
+
+  /**
+   * Timestamp of last update to this adaptation run
+   */
+  updated_at: string | null;
+
+  evaluation_config?: string | null;
+
+  evaluation_metric?: string | null;
+
+  /**
+   * Metrics for the LLM requests made during the adaptation run (e.g.,
+   * total_requests, avg_latency)
+   */
+  llm_request_metrics?: { [key: string]: number };
+
+  /**
+   * Baseline results for the origin model in prompt adaptation.
+   *
+   * Part of AdaptationRunResultsResponse. Contains the performance metrics and
+   * prompt configuration for your original prompt on the origin model. This serves
+   * as the baseline to compare against optimized prompts for target models.
+   *
+   * **Fields include:**
+   *
+   * - Original system prompt and user message template
+   * - Baseline performance score and evaluation metrics
+   * - Cost of running the baseline evaluation
+   * - Job status for the origin model evaluation
+   */
+  origin_model?: PromptGetAdaptResultsResponse.OriginModel | null;
+}
+
+export namespace PromptGetAdaptResultsResponse {
+  /**
+   * Optimized prompt results for a single target model in prompt adaptation.
+   *
+   * Part of AdaptationRunResultsResponse. Contains the optimized system prompt and
+   * user message template for a specific target model, along with performance scores
+   * before and after optimization. Use these optimized prompts with the target model
+   * to achieve better performance than the original prompt.
+   *
+   * **Key metrics:**
+   *
+   * - **pre_optimization_score**: Performance with original prompt on this target
+   *   model
+   * - **post_optimization_score**: Performance with optimized prompt on this target
+   *   model
+   * - **Score improvement**: post - pre shows how much optimization helped
+   *
+   * **Usage:**
+   *
+   * 1. Extract the optimized system_prompt and user_message_template
+   * 2. Replace placeholders in user_message_template using fields from your data
+   * 3. Use these prompts when calling this target model
+   * 4. Compare pre/post scores to see improvement gained
+   */
+  export interface TargetModel {
+    cost: number | null;
+
+    model_name: string;
+
+    post_optimization_evals: { [key: string]: unknown } | null;
+
+    post_optimization_score: number | null;
+
+    pre_optimization_evals: { [key: string]: unknown } | null;
+
+    pre_optimization_score: number | null;
+
+    task_type: string | null;
+
+    /**
+     * Status enum for asynchronous jobs (prompt adaptation, custom router training,
+     * etc.).
+     *
+     * Represents the current state of a long-running operation:
+     *
+     * - **created**: Job has been initialized but not yet queued
+     * - **queued**: Job is waiting in the queue to be processed
+     * - **processing**: Job is currently being executed
+     * - **completed**: Job finished successfully and results are available
+     * - **failed**: Job encountered an error and did not complete
+     */
+    result_status?: PromptAPI.JobStatus | null;
+
+    /**
+     * Optimized system prompt for this target model. Use this as the system message in
+     * your LLM calls
+     */
+    system_prompt?: string | null;
+
+    /**
+     * Optimized user message template with placeholders. Substitute fields using your
+     * data before calling the LLM
+     */
+    user_message_template?: string | null;
+
+    /**
+     * List of field names to substitute in the template (e.g., ['question',
+     * 'context']). These match the curly-brace placeholders in user_message_template
+     */
+    user_message_template_fields?: Array<string> | null;
+  }
+
+  /**
+   * Baseline results for the origin model in prompt adaptation.
+   *
+   * Part of AdaptationRunResultsResponse. Contains the performance metrics and
+   * prompt configuration for your original prompt on the origin model. This serves
+   * as the baseline to compare against optimized prompts for target models.
+   *
+   * **Fields include:**
+   *
+   * - Original system prompt and user message template
+   * - Baseline performance score and evaluation metrics
+   * - Cost of running the baseline evaluation
+   * - Job status for the origin model evaluation
+   */
+  export interface OriginModel {
+    cost: number | null;
+
+    evals: { [key: string]: unknown } | null;
+
+    model_name: string | null;
+
+    score: number | null;
+
+    /**
+     * Status enum for asynchronous jobs (prompt adaptation, custom router training,
+     * etc.).
+     *
+     * Represents the current state of a long-running operation:
+     *
+     * - **created**: Job has been initialized but not yet queued
+     * - **queued**: Job is waiting in the queue to be processed
+     * - **processing**: Job is currently being executed
+     * - **completed**: Job finished successfully and results are available
+     * - **failed**: Job encountered an error and did not complete
+     */
+    result_status?: PromptAPI.JobStatus | null;
+
+    /**
+     * Original system prompt used for the origin model
+     */
+    system_prompt?: string | null;
+
+    /**
+     * Original user message template used for the origin model
+     */
+    user_message_template?: string | null;
+  }
+}
+
+/**
+ * Response model for GET /v2/prompt/adaptStatus/{adaptation_run_id} endpoint.
+ *
+ * Returns the current status of an asynchronous prompt adaptation job. Poll this
+ * endpoint periodically to track progress. When status is 'completed', you can
+ * retrieve the optimized prompts using the /adaptResults endpoint.
+ *
+ * **Status values:**
+ *
+ * - **created**: Job has been initialized
+ * - **queued**: Waiting in queue (check queue_position for your place in line)
+ * - **processing**: Currently running optimization
+ * - **completed**: Finished successfully, results available via /adaptResults
+ * - **failed**: Encountered an error during processing
+ *
+ * **Polling recommendations:**
+ *
+ * - Poll every 30-60 seconds while status is incomplete
+ * - Stop polling once status is 'completed' or 'failed'
+ * - Adaptation typically takes 10-30 minutes total
+ */
+export interface PromptGetAdaptStatusResponse {
+  /**
+   * Unique identifier for this adaptation run. Use this to poll status and retrieve
+   * optimized prompts when complete
+   */
+  adaptation_run_id: string;
+
+  /**
+   * Current status of the adaptation run. Poll until this is 'completed' or 'failed'
+   */
+  status: JobStatus;
+
+  /**
+   * Position in queue when status is 'queued'. Lower numbers process sooner. Null
+   * when not queued
+   */
+  queue_position?: number | null;
+}
+
+Prompt.Adapt = Adapt;
+
+export declare namespace Prompt {
+  export {
+    type JobStatus as JobStatus,
+    type PromptGetAdaptResultsResponse as PromptGetAdaptResultsResponse,
+    type PromptGetAdaptStatusResponse as PromptGetAdaptStatusResponse,
+  };
+
+  export {
+    Adapt as Adapt,
+    type GoldenRecord as GoldenRecord,
+    type RequestProvider as RequestProvider,
+    type AdaptCreateResponse as AdaptCreateResponse,
+    type AdaptGetCostsResponse as AdaptGetCostsResponse,
+    type AdaptCreateParams as AdaptCreateParams,
+  };
+}

package/src/resources/prompt.ts

@@ -0,0 +1,3 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export * from './prompt/index';

package/src/resources/pzn.ts

@@ -0,0 +1,273 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { type Uploadable } from '../core/uploads';
+import { buildHeaders } from '../internal/headers';
+import { RequestOptions } from '../internal/request-options';
+import { multipartFormRequestOptions } from '../internal/uploads';
+
+export class Pzn extends APIResource {
+  /**
+   * Submit a survey response for personalized routing setup.
+   *
+   * This admin endpoint processes survey responses to set up personalized routing
+   * configurations for users based on their use case, LLM preferences, and
+   * constraints.
+   *
+   * **Survey Data:**
+   *
+   * - User information and use case description
+   * - Preferred LLM providers and models
+   * - Constraint priorities (quality, cost, latency)
+   * - Optional prompts and evaluation datasets
+   *
+   * **File Uploads:**
+   *
+   * - `prompt_file`: Optional CSV file with prompts
+   * - `dataset_file`: Optional CSV file with evaluation dataset
+   *
+   * **Note:** This is an admin-only endpoint for internal use.
+   *
+   * @example
+   * ```ts
+   * const response = await client.pzn.submitSurveyResponse({
+   *   constraint_priorities: 'constraint_priorities',
+   *   email: 'email',
+   *   llm_providers: 'llm_providers',
+   *   use_case_desc: 'use_case_desc',
+   *   user_id: 'user_id',
+   *   'x-token': 'x-token',
+   * });
+   * ```
+   */
+  submitSurveyResponse(params: PznSubmitSurveyResponseParams, options?: RequestOptions): APIPromise<unknown> {
+    const { 'x-token': xToken, ...body } = params;
+    return this._client.post(
+      '/v2/pzn/surveyResponse',
+      multipartFormRequestOptions(
+        { body, ...options, headers: buildHeaders([{ 'x-token': xToken }, options?.headers]) },
+        this._client,
+      ),
+    );
+  }
+
+  /**
+   * Train a custom router on your evaluation data to optimize routing for your
+   * specific use case.
+   *
+   * This endpoint allows you to train a domain-specific router that learns which
+   * models perform best for different types of queries in your application. The
+   * router analyzes your evaluation dataset, clusters similar queries, and learns
+   * model performance patterns.
+   *
+   * **Training Process:**
+   *
+   * 1. Upload a CSV file with your evaluation data
+   * 2. Specify which models to route between
+   * 3. Define the evaluation metric (score column)
+   * 4. The system trains asynchronously and returns a preference_id
+   * 5. Use the preference_id in model_select() calls once training completes
+   *
+   * **Dataset Requirements:**
+   *
+   * - Format: CSV file
+   * - Minimum samples: 25 (more is better for accuracy)
+   * - Required columns:
+   *   - Prompt column (specified in prompt_column parameter)
+   *   - For each model: `{provider}/{model}/score` and `{provider}/{model}/response`
+   *
+   * **Example CSV structure:**
+   *
+   * ```
+   * prompt,openai/gpt-4o/score,openai/gpt-4o/response,anthropic/claude-sonnet-4-5-20250929/score,anthropic/claude-sonnet-4-5-20250929/response
+   * "Explain quantum computing",0.95,"Quantum computing uses...",0.87,"Quantum computers leverage..."
+   * "Write a Python function",0.82,"def my_function()...",0.91,"Here's a Python function..."
+   * ```
+   *
+   * **Model Selection:**
+   *
+   * - Specify standard models: `{"provider": "openai", "model": "gpt-4o"}`
+   * - Or custom models with pricing:
+   *   `{"provider": "custom", "model": "my-model", "is_custom": true, "input_price": 10.0, "output_price": 30.0, "context_length": 8192, "latency": 1.5}`
+   *
+   * **Training Time:**
+   *
+   * - Training is asynchronous and typically takes 5-15 minutes
+   * - Larger datasets or more models take longer
+   * - You'll receive a preference_id immediately
+   * - Check training status by attempting to use the preference_id in model_select()
+   *
+   * **Best Practices:**
+   *
+   * 1. Use diverse, representative examples from your production workload
+   * 2. Include at least 50-100 samples for best results
+   * 3. Ensure consistent evaluation metrics across all models
+   * 4. Use the same models you plan to route between in production
+   *
+   * **Related Documentation:** See
+   * https://docs.notdiamond.ai/docs/adapting-prompts-to-new-models for detailed
+   * guide.
+   *
+   * @example
+   * ```ts
+   * const response = await client.pzn.trainCustomRouter({
+   *   dataset_file: fs.createReadStream('path/to/file'),
+   *   language: 'english',
+   *   llm_providers:
+   *     '[{"provider": "openai", "model": "gpt-4o"}, {"provider": "anthropic", "model": "claude-sonnet-4-5-20250929"}]',
+   *   maximize: true,
+   *   prompt_column: 'prompt',
+   * });
+   * ```
+   */
+  trainCustomRouter(
+    body: PznTrainCustomRouterParams,
+    options?: RequestOptions,
+  ): APIPromise<PznTrainCustomRouterResponse> {
+    return this._client.post(
+      '/v2/pzn/trainCustomRouter',
+      multipartFormRequestOptions({ body, ...options }, this._client),
+    );
+  }
+}
+
+export type PznSubmitSurveyResponseResponse = unknown;
+
+/**
+ * Response model for POST /v2/pzn/trainCustomRouter endpoint.
+ *
+ * Returned immediately after submitting a custom router training request. The
+ * training process runs asynchronously (typically 5-15 minutes), so use the
+ * returned preference_id to make routing calls once training completes.
+ *
+ * **Next steps:**
+ *
+ * 1. Store the preference_id
+ * 2. Wait for training to complete (typically 5-15 minutes)
+ * 3. Use this preference_id in POST /v2/modelRouter/modelSelect requests
+ * 4. The router will use your custom-trained model to make routing decisions
+ *
+ * **How to use the preference_id:**
+ *
+ * - Include it in the 'preference_id' field of model_select() calls
+ * - The system automatically uses your custom router once training is complete
+ * - No need to poll status - you can start using it immediately (will use default
+ *   until ready)
+ */
+export interface PznTrainCustomRouterResponse {
+  /**
+   * Unique identifier for the custom router. Use this in model_select() calls to
+   * enable routing with your custom-trained router
+   */
+  preference_id: string;
+}
+
+export interface PznSubmitSurveyResponseParams {
+  /**
+   * Body param: JSON string of constraint priorities object
+   */
+  constraint_priorities: string;
+
+  /**
+   * Body param: User email address
+   */
+  email: string;
+
+  /**
+   * Body param: JSON string of LLM providers array
+   */
+  llm_providers: string;
+
+  /**
+   * Body param: Description of the user's use case
+   */
+  use_case_desc: string;
+
+  /**
+   * Body param: User ID from Supabase
+   */
+  user_id: string;
+
+  /**
+   * Header param:
+   */
+  'x-token': string;
+
+  /**
+   * Body param: Optional additional preferences text
+   */
+  additional_preferences?: string | null;
+
+  /**
+   * Body param: Optional CSV file with evaluation dataset
+   */
+  dataset_file?: Uploadable | null;
+
+  /**
+   * Body param: Optional preference name
+   */
+  name?: string | null;
+
+  /**
+   * Body param: Optional CSV file with prompts
+   */
+  prompt_file?: Uploadable | null;
+
+  /**
+   * Body param: Optional JSON string of prompts array
+   */
+  prompts?: string | null;
+}
+
+export interface PznTrainCustomRouterParams {
+  /**
+   * CSV file containing evaluation data with prompt column and score/response
+   * columns for each model
+   */
+  dataset_file: Uploadable;
+
+  /**
+   * Language of the evaluation data. Use 'english' for English-only data or
+   * 'multilingual' for multi-language support
+   */
+  language: string;
+
+  /**
+   * JSON string array of LLM providers to train the router on. Format:
+   * '[{"provider": "openai", "model": "gpt-4o"}, {"provider": "anthropic", "model":
+   * "claude-sonnet-4-5-20250929"}]'
+   */
+  llm_providers: string;
+
+  /**
+   * Whether higher scores are better. Set to true if higher scores indicate better
+   * performance, false otherwise
+   */
+  maximize: boolean;
+
+  /**
+   * Name of the column in the CSV file that contains the prompts
+   */
+  prompt_column: string;
+
+  /**
+   * Whether to override an existing custom router for this preference_id
+   */
+  override?: boolean | null;
+
+  /**
+   * Optional preference ID to update an existing router. If not provided, a new
+   * preference will be created
+   */
+  preference_id?: string | null;
+}
+
+export declare namespace Pzn {
+  export {
+    type PznSubmitSurveyResponseResponse as PznSubmitSurveyResponseResponse,
+    type PznTrainCustomRouterResponse as PznTrainCustomRouterResponse,
+    type PznSubmitSurveyResponseParams as PznSubmitSurveyResponseParams,
+    type PznTrainCustomRouterParams as PznTrainCustomRouterParams,
+  };
+}

package/src/resources/report/index.ts

@@ -0,0 +1,4 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export { Metrics, type MetricSubmitFeedbackResponse, type MetricSubmitFeedbackParams } from './metrics';
+export { Report } from './report';