@friendliai/ai-provider 0.3.0-beta.6 → 0.3.0

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @friendliai/ai-provider
 
+## 0.3.0
+
+### Patch Changes
+
+- 10ab7aa: add chat_template_kwargs option
+- 10ab7aa: Added supported type model to friendli-provider
+- 10ab7aa: bump v5 package latest
+- 10ab7aa: dump deps (alpha.6)
+- 10ab7aa: Initial build for ai sdk v5 support (broken Friendli compatibility)
+- 10ab7aa: As of 25.08.19, the latest serverless list and model list are synchronized.
+- 10ab7aa: enable regex on v5 (beta)
+- 10ab7aa: Add getAvailableModels() for dynamic model discovery
+  ref: https://vercel.com/docs/ai-gateway/models-and-providers#dynamic-model-discovery
+- 10ab7aa: enable support Tool Assisted API
+
+## 0.3.0-beta.7
+
+### Patch Changes
+
+- ac374ab: Add getAvailableModels() for dynamic model discovery
+  ref: https://vercel.com/docs/ai-gateway/models-and-providers#dynamic-model-discovery
+
 ## 0.3.0-beta.6
 
 ### Patch Changes

package/README.md

@@ -1,115 +1,34 @@
+<!-- header start -->
+<p align="center">
+  <img src="https://huggingface.co/datasets/FriendliAI/documentation-images/resolve/main/model-card-assets/friendliai.png" width="100%" alt="FriendliAI Logo">
+</p>
+<!-- header end -->
+
 # @friendliai/ai-provider
 
-Learn how to use the FriendliAI provider for the Vercel AI SDK.
+[![npm](https://img.shields.io/npm/v/@friendliai/ai-provider)](https://www.npmjs.com/package/@friendliai/ai-provider)
+[![npm](https://img.shields.io/npm/dt/@friendliai/ai-provider)](https://www.npmjs.com/package/@friendliai/ai-provider)
 
-## Installation
+A provider to use FriendliAI models with the Vercel AI SDK and OpenAI-compatible APIs.
 
-You can install the package via npm:
+## Installation
 
 ```bash
 npm i @friendliai/ai-provider
 ```
 
-## Credentials
-
-The tokens required for model usage can be obtained from the [Friendli suite](https://friendli.ai/suite/).
-
-To use the provider, you need to set the `FRIENDLI_TOKEN` environment variable with your personal access token.
-
-```bash
-export FRIENDLI_TOKEN="YOUR_FRIENDLI_TOKEN"
-```
-
-Check the [FriendliAI documentation](https://friendli.ai/docs/guides/personal_access_tokens) for more information.
-
-## Provider Instance
+## Example
 
 ```ts
-import { friendli } from '@friendliai/ai-provider'
-```
-
-## Language Models
-
-You can create [FriendliAI models](https://friendli.ai/docs/guides/serverless_endpoints/text_generation#model-supports) using a provider instance.
-The first argument is the model id, e.g. `meta-llama-3.1-8b-instruct`.
-
-```ts
-const model = friendli('meta-llama-3.1-8b-instruct')
-```
-
-### Example: Generating text
-
-You can use FriendliAI language models to generate text with the `generateText` function:
-
-```ts
-import { friendli } from "@friendliai/ai-provider";
-import { generateText } from 'ai'
+import { friendli } from '@friendliai/ai-provider';
+import { generateText } from 'ai';
 
 const { text } = await generateText({
-  model: friendli('meta-llama-3.1-8b-instruct')
+  model: friendli('meta-llama-3.3-70b-instruct'),
   prompt: 'What is the meaning of life?',
-})
-```
-
-### Example: Using Enforcing Patterns (Regex)
-
-Specify a specific pattern (e.g., CSV), character sets, or specific language characters (e.g., Korean Hangul characters) for your LLM's output.
-
-```ts
-import { friendli } from '@friendliai/ai-provider'
-import { generateText } from 'ai'
-
-const { text } = await generateText({
-  model: friendli('meta-llama-3.1-8b-instruct', {
-    regex: '[\n ,.?!0-9\uac00-\ud7af]*',
-  }),
-  maxTokens: 40,
-  prompt: 'who is the first king of the Joseon Dynasty?',
-})
-
-console.log(text)
+  maxTokens: 20,
+});
+console.log(text);
 ```
 
-### Example: Using built-in tools (Beta)
-
-If you use `@friendliai/ai-provider`, you can use the [built-in tools](https://friendli.ai/docs/guides/serverless_endpoints/tools/built_in_tools) via the `tools` option.
-
-Built-in tools allow models to use tools to generate better answers. For example, a `web:search` tool can provide up-to-date answers to current questions.
-
-```ts highlight="1,8,9,10,11,12,13,14,15"
-import { friendli } from '@friendliai/ai-provider'
-import { convertToCoreMessages, streamText } from 'ai'
-
-export async function POST(req: Request) {
-  const { messages } = await req.json()
-
-  const result = await streamText({
-    model: friendli('meta-llama-3.1-8b-instruct', {
-      tools: [
-        { type: 'web:search' },
-        { type: 'math:calculator' },
-        { type: 'code:python-interpreter' }, // and more tools..!!
-      ],
-    }),
-    messages: convertToCoreMessages(messages),
-  })
-
-  return result.toDataStreamResponse()
-}
-```
-
-FriendliAI language models can also be used in the `streamText`, `generateObject`, `streamObject`, and `streamUI` functions.
-(see [AI SDK Core](/docs/ai-sdk-core) and [AI SDK RSC](/docs/ai-sdk-rsc)).
-
-## OpenAI Compatibility
-
-We can also use `@ai-sdk/openai` with OpenAI compatibility.
-
-```ts
-import { createOpenAI } from '@ai-sdk/openai'
-
-const friendli = createOpenAI({
-  baseURL: 'https://api.friendli.ai/serverless/v1',
-  apiKey: process.env.FRIENDLI_TOKEN,
-})
-```
+For more details, see the [official documentation](https://friendli.ai/docs/).

package/dist/index.d.mts

@@ -6,6 +6,27 @@ declare const FriendliAIServerlessModelIds: readonly ["meta-llama-3.1-8b-instruc
 type FriendliAIServerlessModelId = (typeof FriendliAIServerlessModelIds)[number];
 type FriendliAILanguageModelId = FriendliAIServerlessModelId | (string & {});
 
+type Pricing = {
+    inputToken?: number;
+    outputToken?: number;
+    responseTime?: number;
+    unitType?: 'TOKEN' | 'SECOND';
+    currency?: string;
+    unit?: string;
+};
+type FriendliAvailableModel = {
+    id: string;
+    name?: string | null;
+    description?: string | null;
+    pricing?: Pricing;
+    warm?: boolean;
+    cold?: boolean;
+    contextLength?: number | null;
+};
+type FriendliAvailableModelsResponse = {
+    models: FriendliAvailableModel[];
+};
+
 declare function webUrlBetaTool(): LanguageModelV2ProviderDefinedTool;
 declare function webSearchBetaTool(): LanguageModelV2ProviderDefinedTool;
 declare function mathCalendarBetaTool(): LanguageModelV2ProviderDefinedTool;
@@ -65,6 +86,14 @@ interface FriendliAIProvider extends ProviderV2 {
      * Creates a text embedding model for text generation.
      * TODO: Implement for Dedicated users
      */
+    embedding(modelId: string & {}): LanguageModelV2;
+    textEmbeddingModel(modelId: string & {}): LanguageModelV2;
+    /**
+     * Returns the available models and their metadata.
+     */
+    getAvailableModels(options?: {
+        graphqlURL?: string;
+    }): Promise<FriendliAvailableModelsResponse>;
     embedding(modelId: string & {}): EmbeddingModelV2<string>;
     textEmbeddingModel(modelId: string & {}): EmbeddingModelV2<string>;
     /**
@@ -102,4 +131,4 @@ declare const friendliaiErrorSchema: z.ZodObject<{
 }, z.core.$strip>;
 type FriendliAIErrorData = z.infer<typeof friendliaiErrorSchema>;
 
-export { type FriendliAIErrorData, type FriendliAIProvider, type FriendliAIProviderSettings, createFriendli, friendli };
+export { type FriendliAIErrorData, type FriendliAIProvider, type FriendliAIProviderSettings, type FriendliAvailableModel, type FriendliAvailableModelsResponse, createFriendli, friendli };

package/dist/index.d.ts

@@ -6,6 +6,27 @@ declare const FriendliAIServerlessModelIds: readonly ["meta-llama-3.1-8b-instruc
 type FriendliAIServerlessModelId = (typeof FriendliAIServerlessModelIds)[number];
 type FriendliAILanguageModelId = FriendliAIServerlessModelId | (string & {});
 
+type Pricing = {
+    inputToken?: number;
+    outputToken?: number;
+    responseTime?: number;
+    unitType?: 'TOKEN' | 'SECOND';
+    currency?: string;
+    unit?: string;
+};
+type FriendliAvailableModel = {
+    id: string;
+    name?: string | null;
+    description?: string | null;
+    pricing?: Pricing;
+    warm?: boolean;
+    cold?: boolean;
+    contextLength?: number | null;
+};
+type FriendliAvailableModelsResponse = {
+    models: FriendliAvailableModel[];
+};
+
 declare function webUrlBetaTool(): LanguageModelV2ProviderDefinedTool;
 declare function webSearchBetaTool(): LanguageModelV2ProviderDefinedTool;
 declare function mathCalendarBetaTool(): LanguageModelV2ProviderDefinedTool;
@@ -65,6 +86,14 @@ interface FriendliAIProvider extends ProviderV2 {
      * Creates a text embedding model for text generation.
      * TODO: Implement for Dedicated users
      */
+    embedding(modelId: string & {}): LanguageModelV2;
+    textEmbeddingModel(modelId: string & {}): LanguageModelV2;
+    /**
+     * Returns the available models and their metadata.
+     */
+    getAvailableModels(options?: {
+        graphqlURL?: string;
+    }): Promise<FriendliAvailableModelsResponse>;
     embedding(modelId: string & {}): EmbeddingModelV2<string>;
     textEmbeddingModel(modelId: string & {}): EmbeddingModelV2<string>;
     /**
@@ -102,4 +131,4 @@ declare const friendliaiErrorSchema: z.ZodObject<{
 }, z.core.$strip>;
 type FriendliAIErrorData = z.infer<typeof friendliaiErrorSchema>;
 
-export { type FriendliAIErrorData, type FriendliAIProvider, type FriendliAIProviderSettings, createFriendli, friendli };
+export { type FriendliAIErrorData, type FriendliAIProvider, type FriendliAIProviderSettings, type FriendliAvailableModel, type FriendliAvailableModelsResponse, createFriendli, friendli };

package/dist/index.js

@@ -27,7 +27,7 @@ module.exports = __toCommonJS(index_exports);
 
 // src/friendli-provider.ts
 var import_provider3 = require("@ai-sdk/provider");
-var import_provider_utils3 = require("@ai-sdk/provider-utils");
+var import_provider_utils4 = require("@ai-sdk/provider-utils");
 var import_openai_compatible = require("@ai-sdk/openai-compatible");
 
 // src/friendli-settings.ts
@@ -72,9 +72,7 @@ var friendliaiErrorStructure = {
   errorSchema: friendliaiErrorSchema,
   errorToMessage: (data) => data.message
 };
-var friendliaiFailedResponseHandler = (0, import_provider_utils.createJsonErrorResponseHandler)(
-  friendliaiErrorStructure
-);
+var friendliaiFailedResponseHandler = (0, import_provider_utils.createJsonErrorResponseHandler)(friendliaiErrorStructure);
 
 // src/friendli-prepare-tools.ts
 var import_provider = require("@ai-sdk/provider");
@@ -142,12 +140,8 @@ var FriendliAIChatLanguageModel = class {
     this.modelId = modelId;
     this.config = config;
     const errorStructure = friendliaiErrorStructure;
-    this.chunkSchema = createOpenAICompatibleChatChunkSchema(
-      errorStructure.errorSchema
-    );
-    this.failedResponseHandler = (0, import_provider_utils2.createJsonErrorResponseHandler)(
-      friendliaiErrorStructure
-    );
+    this.chunkSchema = createOpenAICompatibleChatChunkSchema(errorStructure.errorSchema);
+    this.failedResponseHandler = (0, import_provider_utils2.createJsonErrorResponseHandler)(friendliaiErrorStructure);
     this.supportsStructuredOutputs = (_a = config.supportsStructuredOutputs) != null ? _a : true;
   }
   get provider() {
@@ -253,9 +247,7 @@ var FriendliAIChatLanguageModel = class {
       headers: (0, import_provider_utils2.combineHeaders)(this.config.headers(), options.headers),
       body: args,
       failedResponseHandler: this.failedResponseHandler,
-      successfulResponseHandler: (0, import_provider_utils2.createJsonResponseHandler)(
-        OpenAICompatibleChatResponseSchema
-      ),
+      successfulResponseHandler: (0, import_provider_utils2.createJsonResponseHandler)(OpenAICompatibleChatResponseSchema),
       abortSignal: options.abortSignal,
       fetch: this.config.fetch
     });
@@ -320,9 +312,7 @@ var FriendliAIChatLanguageModel = class {
       headers: (0, import_provider_utils2.combineHeaders)(this.config.headers(), options.headers),
       body,
       failedResponseHandler: this.failedResponseHandler,
-      successfulResponseHandler: (0, import_provider_utils2.createEventSourceResponseHandler)(
-        this.chunkSchema
-      ),
+      successfulResponseHandler: (0, import_provider_utils2.createEventSourceResponseHandler)(this.chunkSchema),
       abortSignal: options.abortSignal,
       fetch: this.config.fetch
     });
@@ -397,9 +387,7 @@ var FriendliAIChatLanguageModel = class {
             }
             const choice = value.choices[0];
             if ((choice == null ? void 0 : choice.finish_reason) != null) {
-              finishReason = (0, import_internal.mapOpenAICompatibleFinishReason)(
-                choice.finish_reason
-              );
+              finishReason = (0, import_internal.mapOpenAICompatibleFinishReason)(choice.finish_reason);
             }
             if ((choice == null ? void 0 : choice.delta) == null) {
               return;
@@ -689,6 +677,113 @@ var friendliProviderOptionsSchema = import_zod2.z.object({
   chat_template_kwargs: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.any()).nullish()
 });
 
+// src/get-available-models.ts
+var import_provider_utils3 = require("@ai-sdk/provider-utils");
+var DEFAULT_GRAPHQL_URL = "https://api-internal.friendli.ai/api/graphql";
+async function postGraphQL(url, body, headers) {
+  const res = await fetch(url, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      ...headers
+    },
+    body: JSON.stringify(body)
+  });
+  let json;
+  try {
+    json = await res.json();
+  } catch (err) {
+    console.error(
+      "Failed to parse JSON response from Friendli API:",
+      err,
+      "Status:",
+      res.status,
+      res.statusText
+    );
+    throw new Error(
+      `Failed to parse JSON response from Friendli API: ${err instanceof Error ? err.message : String(err)}`
+    );
+  }
+  return json;
+}
+function normalizePriceUnit(unit) {
+  if (!unit) return void 0;
+  return unit;
+}
+async function getAvailableModelsImpl(options) {
+  var _a, _b, _c, _d, _e, _f;
+  let token;
+  try {
+    token = (_a = options.apiKey) != null ? _a : (0, import_provider_utils3.loadApiKey)({
+      apiKey: void 0,
+      environmentVariableName: "FRIENDLI_TOKEN",
+      description: "FRIENDLI_TOKEN"
+    });
+  } catch (e) {
+    token = void 0;
+  }
+  const headers = {
+    ...token ? { Authorization: `Bearer ${token}` } : {},
+    ...options.teamId ? { "X-Friendli-Team": options.teamId } : {},
+    ...(_b = options.headers) != null ? _b : {}
+  };
+  const url = (_c = options.graphqlURL) != null ? _c : DEFAULT_GRAPHQL_URL;
+  const query = `
+  query Edges {
+    serverlessEndpoints {
+      edges {
+        ... on ServerlessChatEndpointCatalog {
+          id
+          name
+          status
+          price {
+            inputPrice
+            outputPrice
+            unit
+            responseTimePrice
+            priceUnitType
+          }
+          contextLength
+        }
+      }
+    }
+  }
+  `;
+  const resp = await postGraphQL(
+    url,
+    { query, variables: {}, operationName: "Edges" },
+    headers
+  );
+  if (resp.errors && resp.errors.length > 0) {
+    throw new Error(
+      `getAvailableModels: GraphQL error: ${resp.errors.map((e) => e.message).join("; ")}`
+    );
+  }
+  const edges = (_f = (_e = (_d = resp.data) == null ? void 0 : _d.serverlessEndpoints) == null ? void 0 : _e.edges) != null ? _f : [];
+  const models = edges.map((e) => {
+    var _a2, _b2, _c2, _d2, _e2, _f2;
+    const warm = e.status === "WARM";
+    const pricing = e.price ? {
+      inputToken: (_a2 = e.price.inputPrice) != null ? _a2 : void 0,
+      outputToken: (_b2 = e.price.outputPrice) != null ? _b2 : void 0,
+      responseTime: (_c2 = e.price.responseTimePrice) != null ? _c2 : void 0,
+      unitType: (_d2 = e.price.priceUnitType) != null ? _d2 : void 0,
+      unit: normalizePriceUnit(e.price.unit),
+      currency: "USD"
+    } : void 0;
+    return {
+      id: e.id,
+      name: (_e2 = e.name) != null ? _e2 : void 0,
+      description: void 0,
+      pricing,
+      warm,
+      cold: warm === false,
+      contextLength: (_f2 = e.contextLength) != null ? _f2 : void 0
+    };
+  });
+  return { models };
+}
+
 // src/friendli-tools.ts
 function webUrlBetaTool() {
   return {
@@ -750,7 +845,7 @@ var friendliTools = {
 // src/friendli-provider.ts
 function createFriendli(options = {}) {
   const getHeaders = () => ({
-    Authorization: `Bearer ${(0, import_provider_utils3.loadApiKey)({
+    Authorization: `Bearer ${(0, import_provider_utils4.loadApiKey)({
       apiKey: options.apiKey,
       environmentVariableName: "FRIENDLI_TOKEN",
       description: "FRIENDLI_TOKEN"
@@ -764,7 +859,7 @@ function createFriendli(options = {}) {
       serverless_tools: "https://api.friendli.ai/serverless/tools/v1",
       dedicated: "https://api.friendli.ai/dedicated/v1"
     };
-    const customBaseURL = (0, import_provider_utils3.withoutTrailingSlash)(baseURL);
+    const customBaseURL = (0, import_provider_utils4.withoutTrailingSlash)(baseURL);
     if (typeof customBaseURL === "string" && customBaseURL !== "dedicated" && customBaseURL !== "serverless" && customBaseURL !== "serverless-tools") {
       return { baseURL: customBaseURL, type: "custom" };
     }
@@ -785,9 +880,7 @@ function createFriendli(options = {}) {
           type: "serverless-tools"
         };
       default:
-        if (FriendliAIServerlessModelIds.includes(
-          modelId
-        )) {
+        if (FriendliAIServerlessModelIds.includes(modelId)) {
           return {
             baseURL: FriendliBaseURL.serverless,
             type: "serverless"
@@ -837,6 +930,15 @@ function createFriendli(options = {}) {
   provider.completion = createCompletionModel;
   provider.embedding = createTextEmbeddingModel;
   provider.textEmbeddingModel = createTextEmbeddingModel;
+  provider.getAvailableModels = async (opts) => {
+    var _a;
+    const defaultURL = "https://api-internal.friendli.ai/api/graphql";
+    const graphqlURL = (_a = opts == null ? void 0 : opts.graphqlURL) != null ? _a : defaultURL;
+    const apiKey = options.apiKey;
+    const teamId = options.teamId;
+    const headers = options.headers;
+    return getAvailableModelsImpl({ apiKey, teamId, headers, graphqlURL });
+  };
   provider.imageModel = createImageModel;
   provider.transcription = createTranscriptionModel;
   provider.speech = createSpeechModel;