@llmops/gateway 0.1.0-beta.12 → 0.1.0-beta.13

package/dist/index.d.cts

@@ -0,0 +1,434 @@
+import * as hono_types0 from "hono/types";
+import { Context, Hono } from "hono";
+
+//#region src/globals.d.ts
+
+declare enum BatchEndpoints {
+  CHAT_COMPLETIONS = "/v1/chat/completions",
+  COMPLETIONS = "/v1/completions",
+  EMBEDDINGS = "/v1/embeddings",
+}
+//#endregion
+//#region src/middlewares/hooks/types.d.ts
+interface Check {
+  id: string;
+  parameters: object;
+  is_enabled?: boolean;
+}
+interface HookOnFailObject {
+  feedback?: HookFeedback;
+}
+interface HookOnSuccessObject {
+  feedback?: HookFeedback;
+}
+interface HookObject {
+  type: HookType;
+  id: string;
+  checks?: Check[];
+  async?: boolean;
+  sequential?: boolean;
+  onFail?: HookOnFailObject;
+  onSuccess?: HookOnSuccessObject;
+  deny?: boolean;
+  eventType: 'beforeRequestHook' | 'afterRequestHook';
+}
+interface GuardrailFeedbackMetadata {
+  successfulChecks: string;
+  failedChecks: string;
+  erroredChecks: string;
+}
+interface GuardrailFeedback {
+  value?: number;
+  weight?: number;
+  metadata?: GuardrailFeedbackMetadata;
+}
+type HookFeedback = GuardrailFeedback;
+declare enum HookType {
+  GUARDRAIL = "guardrail",
+  MUTATOR = "mutator",
+}
+//#endregion
+//#region src/types/requestBody.d.ts
+/**
+ * Settings for retrying requests.
+ * @interface
+ */
+interface RetrySettings {
+  /** The maximum number of retry attempts. */
+  attempts: number;
+  /** The HTTP status codes on which to retry. */
+  onStatusCodes: number[];
+  /** Whether to use the provider's retry wait. */
+  useRetryAfterHeader?: boolean;
+}
+interface CacheSettings {
+  mode: string;
+  maxAge?: number;
+}
+/**
+ * Configuration for an AI provider.
+ * @interface
+ */
+interface Options {
+  /** The name of the provider. */
+  provider: string;
+  /** The name of the API key for the provider. */
+  virtualKey?: string;
+  /** The API key for the provider. */
+  apiKey?: string;
+  /** The weight of the provider, used for load balancing. */
+  weight?: number;
+  /** The retry settings for the provider. */
+  retry?: RetrySettings;
+  /** The parameters to override in the request. */
+  overrideParams?: Params;
+  /** The actual url used to make llm calls */
+  urlToFetch?: string;
+  /** Azure specific */
+  resourceName?: string;
+  deploymentId?: string;
+  apiVersion?: string;
+  adAuth?: string;
+  azureAuthMode?: string;
+  azureManagedClientId?: string;
+  azureWorkloadClientId?: string;
+  azureEntraClientId?: string;
+  azureEntraClientSecret?: string;
+  azureEntraTenantId?: string;
+  azureAdToken?: string;
+  azureModelName?: string;
+  /** Workers AI specific */
+  workersAiAccountId?: string;
+  /** The parameter to set custom base url */
+  customHost?: string;
+  /** The parameter to set list of headers to be forwarded as-is to the provider */
+  forwardHeaders?: string[];
+  /** provider option index picked based on weight in loadbalance mode */
+  index?: number;
+  cache?: CacheSettings | string;
+  metadata?: Record<string, string>;
+  requestTimeout?: number;
+  /** This is used to determine if the request should be transformed to formData Example: Stability V2 */
+  transformToFormData?: boolean;
+  /** AWS specific (used for Bedrock and Sagemaker) */
+  awsSecretAccessKey?: string;
+  awsAccessKeyId?: string;
+  awsSessionToken?: string;
+  awsRegion?: string;
+  awsAuthType?: string;
+  awsRoleArn?: string;
+  awsExternalId?: string;
+  awsS3Bucket?: string;
+  awsS3ObjectKey?: string;
+  awsBedrockModel?: string;
+  awsServerSideEncryption?: string;
+  awsServerSideEncryptionKMSKeyId?: string;
+  awsService?: string;
+  foundationModel?: string;
+  /** Sagemaker specific */
+  amznSagemakerCustomAttributes?: string;
+  amznSagemakerTargetModel?: string;
+  amznSagemakerTargetVariant?: string;
+  amznSagemakerTargetContainerHostname?: string;
+  amznSagemakerInferenceId?: string;
+  amznSagemakerEnableExplanations?: string;
+  amznSagemakerInferenceComponent?: string;
+  amznSagemakerSessionId?: string;
+  amznSagemakerModelName?: string;
+  /** Stability AI specific */
+  stabilityClientId?: string;
+  stabilityClientUserId?: string;
+  stabilityClientVersion?: string;
+  /** Hugging Face specific */
+  huggingfaceBaseUrl?: string;
+  /** Google Vertex AI specific */
+  vertexRegion?: string;
+  vertexProjectId?: string;
+  vertexServiceAccountJson?: Record<string, any>;
+  vertexStorageBucketName?: string;
+  vertexModelName?: string;
+  vertexBatchEndpoint?: BatchEndpoints;
+  filename?: string;
+  afterRequestHooks?: HookObject[];
+  beforeRequestHooks?: HookObject[];
+  defaultInputGuardrails?: HookObject[];
+  defaultOutputGuardrails?: HookObject[];
+  /** OpenAI specific */
+  openaiProject?: string;
+  openaiOrganization?: string;
+  openaiBeta?: string;
+  /** Azure Inference Specific */
+  azureApiVersion?: string;
+  azureFoundryUrl?: string;
+  azureExtraParameters?: string;
+  azureDeploymentName?: string;
+  /** The parameter to determine if extra non-openai compliant fields should be returned in response */
+  strictOpenAiCompliance?: boolean;
+  /** Parameter to determine if fim/completions endpoint is to be used */
+  mistralFimCompletion?: string;
+  /** Anthropic specific headers */
+  anthropicBeta?: string;
+  anthropicVersion?: string;
+  anthropicApiKey?: string;
+  /** Fireworks finetune required fields */
+  fireworksAccountId?: string;
+  fireworksFileLength?: string;
+  /** Cortex specific fields */
+  snowflakeAccount?: string;
+  /** Azure entra scope */
+  azureEntraScope?: string;
+  oracleApiVersion?: string;
+  oracleRegion?: string;
+  oracleCompartmentId?: string;
+  oracleServingMode?: string;
+  oracleTenancy?: string;
+  oracleUser?: string;
+  oracleFingerprint?: string;
+  oraclePrivateKey?: string;
+  oracleKeyPassphrase?: string;
+  /** Model pricing config */
+  modelPricingConfig?: Record<string, any>;
+}
+/**
+ * TODO: make this a union type
+ * A message content type.
+ * @interface
+ */
+interface ContentType extends PromptCache {
+  type: string;
+  text?: string;
+  thinking?: string;
+  signature?: string;
+  image_url?: {
+    url: string;
+    detail?: string;
+    mime_type?: string;
+  };
+  data?: string;
+  file?: {
+    file_data?: string;
+    file_id?: string;
+    file_name?: string;
+    file_url?: string;
+    mime_type?: string;
+  };
+  input_audio?: {
+    data: string;
+    format: 'mp3' | 'wav' | string;
+  };
+}
+type OpenAIMessageRole = 'system' | 'user' | 'assistant' | 'function' | 'tool' | 'developer';
+/**
+ * A message in the conversation.
+ * @interface
+ */
+interface Message {
+  /** The role of the message sender. It can be 'system', 'user', 'assistant', or 'function'. */
+  role: OpenAIMessageRole;
+  /** The content of the message. */
+  content?: string | ContentType[];
+  /** The content blocks of the message. */
+  content_blocks?: ContentType[];
+  /** The name of the function to call, if any. */
+  name?: string;
+  /** The function call to make, if any. */
+  function_call?: any;
+  tool_calls?: any;
+  tool_call_id?: string;
+  citationMetadata?: CitationMetadata;
+  /** Reasoning details for models that support extended thinking/reasoning. (Gemini) */
+  reasoning_details?: any[];
+}
+interface PromptCache {
+  cache_control?: {
+    type: 'ephemeral';
+  };
+}
+interface CitationMetadata {
+  citationSources?: CitationSource[];
+}
+interface CitationSource {
+  startIndex?: number;
+  endIndex?: number;
+  uri?: string;
+  license?: string;
+}
+/**
+ * A JSON schema.
+ * @interface
+ */
+interface JsonSchema {
+  /** The schema definition, indexed by key. */
+  [key: string]: any;
+}
+/**
+ * A function in the conversation.
+ * @interface
+ */
+interface Function {
+  /** The name of the function. */
+  name: string;
+  /** A description of the function. */
+  description?: string;
+  /** The parameters for the function. */
+  parameters?: JsonSchema;
+  /** Whether to enable strict schema adherence when generating the function call. If set to true, the model will follow the exact schema defined in the parameters field. Only a subset of JSON Schema is supported when strict is true */
+  strict?: boolean;
+  /**
+   * When true, this tool is not loaded into context initially.
+   * Claude discovers it via Tool Search Tool on-demand.
+   * Part of Anthropic's advanced tool use beta features.
+   */
+  defer_loading?: boolean;
+  /**
+   * List of tool types that can call this tool programmatically.
+   * E.g., ["code_execution_20250825"] enables Programmatic Tool Calling.
+   * Part of Anthropic's advanced tool use beta features.
+   */
+  allowed_callers?: string[];
+  /**
+   * Example inputs demonstrating how to use this tool.
+   * Helps Claude understand usage patterns beyond JSON schema.
+   * Part of Anthropic's advanced tool use beta features.
+   */
+  input_examples?: Record<string, any>[];
+}
+interface ToolChoiceObject {
+  type: string;
+  function: {
+    name: string;
+  };
+}
+interface CustomToolChoice {
+  type: 'custom';
+  custom: {
+    name?: string;
+  };
+}
+type ToolChoice = ToolChoiceObject | CustomToolChoice | 'none' | 'auto' | 'required';
+/**
+ * A tool in the conversation.
+ *
+ * `cache_control` is extended to support for prompt-cache
+ *
+ * @interface
+ */
+interface Tool extends PromptCache {
+  /** The name of the function. */
+  type: string;
+  /** A description of the function. */
+  function?: Function;
+  [key: string]: any;
+}
+/**
+ * The parameters for the request.
+ * @interface
+ */
+interface Params {
+  model?: string;
+  prompt?: string | string[];
+  messages?: Message[];
+  functions?: Function[];
+  function_call?: 'none' | 'auto' | {
+    name: string;
+  };
+  max_tokens?: number;
+  max_completion_tokens?: number;
+  temperature?: number;
+  top_p?: number;
+  n?: number;
+  stream?: boolean;
+  logprobs?: number;
+  top_logprobs?: boolean;
+  echo?: boolean;
+  stop?: string | string[];
+  presence_penalty?: number;
+  frequency_penalty?: number;
+  best_of?: number;
+  logit_bias?: {
+    [key: string]: number;
+  };
+  user?: string;
+  context?: string;
+  examples?: Examples[];
+  top_k?: number;
+  tools?: Tool[];
+  tool_choice?: ToolChoice;
+  reasoning_effort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | string;
+  response_format?: {
+    type: 'json_object' | 'text' | 'json_schema';
+    json_schema?: any;
+  };
+  seed?: number;
+  store?: boolean;
+  metadata?: object;
+  modalities?: string[];
+  audio?: {
+    voice: string;
+    format: string;
+  };
+  service_tier?: string;
+  prediction?: {
+    type: string;
+    content: {
+      type: string;
+      text: string;
+    }[] | string;
+  };
+  safety_settings?: any;
+  anthropic_beta?: string;
+  anthropic_version?: string;
+  thinking?: {
+    type?: string;
+    budget_tokens: number;
+  };
+  dimensions?: number;
+  parameters?: any;
+  version?: number;
+}
+interface Examples {
+  input?: Message;
+  output?: Message;
+}
+/**
+ * The full structure of the request body.
+ * @interface
+ */
+//#endregion
+//#region src/providers/types.d.ts
+
+type endpointStrings = 'complete' | 'chatComplete' | 'embed' | 'rerank' | 'moderate' | 'stream-complete' | 'stream-chatComplete' | 'stream-messages' | 'proxy' | 'imageGenerate' | 'imageEdit' | 'createSpeech' | 'createTranscription' | 'createTranslation' | 'realtime' | 'uploadFile' | 'listFiles' | 'retrieveFile' | 'deleteFile' | 'retrieveFileContent' | 'createBatch' | 'retrieveBatch' | 'cancelBatch' | 'listBatches' | 'getBatchOutput' | 'listFinetunes' | 'createFinetune' | 'retrieveFinetune' | 'cancelFinetune' | 'createModelResponse' | 'getModelResponse' | 'deleteModelResponse' | 'listResponseInputItems' | 'messages' | 'messagesCountTokens';
+type RequestHandler<T = Params | FormData | ArrayBuffer | ReadableStream> = (Params: {
+  c: Context;
+  providerOptions: Options;
+  requestURL: string;
+  requestHeaders: Record<string, string>;
+  requestBody: T;
+}) => Promise<Response>;
+type RequestHandlers = Partial<Record<endpointStrings, RequestHandler<any>>>;
+/**
+ * A collection of configurations for multiple AI providers.
+ * @interface
+ */
+interface ProviderConfigs {
+  /** The configuration for each provider, indexed by provider name. */
+  [key: string]: any;
+  requestHandlers?: RequestHandlers;
+  getConfig?: ({
+    params,
+    providerOptions
+  }: {
+    params: Params;
+    providerOptions: Options;
+  }) => any;
+}
+//#endregion
+//#region src/providers/index.d.ts
+declare const Providers: {
+  [key: string]: ProviderConfigs;
+};
+//#endregion
+//#region src/index.d.ts
+declare const app: Hono<hono_types0.BlankEnv, hono_types0.BlankSchema, "/">;
+//#endregion
+export { Providers, app as default };

package/dist/index.mjs

@@ -4,6 +4,7 @@ import { prettyJSON } from "hono/pretty-json";
 import { HTTPException } from "hono/http-exception";
 import { compress } from "hono/compress";
 import { env, getRuntimeKey } from "hono/adapter";
+import * as path from "path";
 import { Validator } from "@cfworker/json-schema";
 import { Sha256 } from "@aws-crypto/sha256-js";
 import { SignatureV4 } from "@smithy/signature-v4";
@@ -11,7 +12,6 @@ import { Agent } from "https";
 import { importJWK, jwtVerify } from "jose";
 import retry from "async-retry";
 import * as fs from "fs/promises";
-import * as path from "path";
 import Redis from "ioredis";
 
 //#region rolldown:runtime
@@ -4160,8 +4160,8 @@ const isNodeInstance = getRuntimeKey() == "node";
 let path$1;
 let fs$1;
 if (isNodeInstance) {
-	path$1 = await import("path");
-	fs$1 = await import("fs");
+	path$1 = __require("path");
+	fs$1 = __require("fs");
 }
 function getValueOrFileContents(value, ignore) {
 	if (!value || ignore) return value;
@@ -9953,7 +9953,7 @@ const retryRequest = async (url, options, retryCount, statusCodesToRetry, timeou
 
 //#endregion
 //#region package.json
-var version = "0.1.0-beta.12";
+var version = "0.1.0-beta.13";
 
 //#endregion
 //#region src/providers/bytez/api.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llmops/gateway",
-  "version": "0.1.0-beta.12",
+  "version": "0.1.0-beta.13",
   "description": "AI gateway for LLMOps (forked from Portkey)",
   "type": "module",
   "license": "Apache-2.0",
@@ -23,11 +23,20 @@
   "files": [
     "dist"
   ],
-  "main": "./dist/index.mjs",
+  "main": "./dist/index.cjs",
   "module": "./dist/index.mjs",
   "types": "./dist/index.d.mts",
   "exports": {
-    ".": "./dist/index.mjs",
+    ".": {
+      "import": {
+        "types": "./dist/index.d.mts",
+        "default": "./dist/index.mjs"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    },
     "./package.json": "./package.json"
   },
   "publishConfig": {