@browser-ai/core 1.0.0

package/README.md

@@ -0,0 +1,379 @@
+# Built-in AI provider for Vercel AI SDK
+
+<div align="center">
+<img src="./hero.png">
+</div>
+
+<div align="center">
+
+[![NPM Version](https://img.shields.io/npm/v/%40built-in-ai%2Fcore)](https://www.npmjs.com/package/@built-in-ai/core)
+[![NPM Downloads](https://img.shields.io/npm/dm/%40built-in-ai%2Fcore)](https://www.npmjs.com/package/@built-in-ai/core)
+
+</div>
+
+A TypeScript library that provides access to browser-based AI capabilities with seamless fallback to using server-side models using the [Vercel AI SDK](https://ai-sdk.dev/). This library enables you to leverage **Chrome** and **Edge's** built-in AI features ([Prompt API](https://github.com/webmachinelearning/prompt-api)) with the AI SDK.
+
+> [!IMPORTANT]
+> This package is under constant development as the Prompt API matures, and may contain errors and incompatible changes.
+
+## Installation
+
+```bash
+npm i @built-in-ai/core
+```
+
+The `@built-in-ai/core` package is the AI SDK provider for your Chrome and Edge browser's built-in AI models. It provides seamless access to both language models and text embeddings through browser-native APIs.
+
+## Browser Requirements
+
+> [!IMPORTANT]
+> The Prompt API is currently experimental and might change as it matures. The below enablement guide of the API might also change in the future.
+
+1. You need Chrome (v. 128 or higher) or Edge Dev/Canary (v. 138.0.3309.2 or higher)
+
+2. Enable these experimental flags:
+   - If you're using Chrome:
+     1. Go to `chrome://flags/`, search for _'Prompt API for Gemini Nano with Multimodal Input'_ and set it to Enabled
+     2. Go to `chrome://components` and click Check for Update on Optimization Guide On Device Model
+   - If you're using Edge:
+     1. Go to `edge://flags/#prompt-api-for-phi-mini` and set it to Enabled
+
+For more information, check out [this guide](https://developer.chrome.com/docs/extensions/ai/prompt-api)
+
+## Usage
+
+### Basic Usage (chat)
+
+```typescript
+import { streamText } from "ai";
+import { builtInAI } from "@built-in-ai/core";
+
+const result = streamText({
+  // or generateText
+  model: builtInAI(),
+  messages: [{ role: "user", content: "Hello, how are you?" }],
+});
+
+for await (const chunk of result.textStream) {
+  console.log(chunk);
+}
+```
+
+### Language Models
+
+```typescript
+import { generateText } from "ai";
+import { builtInAI } from "@built-in-ai/core";
+
+const model = builtInAI();
+
+const result = await generateText({
+  model,
+  messages: [{ role: "user", content: "Write a short poem about AI" }],
+});
+```
+
+### Text Embeddings
+
+```typescript
+import { embed, embedMany } from "ai";
+import { builtInAI } from "@built-in-ai/core";
+
+// Single embedding
+const result = await embed({
+  model: builtInAI.textEmbedding("embedding"),
+  value: "Hello, world!",
+});
+
+console.log(result.embedding); // [0.1, 0.2, 0.3, ...]
+
+// Multiple embeddings
+const results = await embedMany({
+  model: builtInAI.textEmbedding("embedding"),
+  values: ["Hello", "World", "AI"],
+});
+
+console.log(results.embeddings); // [[...], [...], [...]]
+```
+
+## Download Progress Tracking
+
+When using the built-in AI models in Chrome & Edge for the first time, the model needs to be downloaded first.
+
+You'll probably want to show download progress in your applications to improve UX.
+
+### Basic Progress Monitoring
+
+```typescript
+import { streamText } from "ai";
+import { builtInAI } from "@built-in-ai/core";
+
+const model = builtInAI();
+const availability = await model.availability();
+
+if (availability === "unavailable") {
+  console.log("Browser doesn't support built-in AI");
+  return;
+}
+
+if (availability === "downloadable") {
+  await model.createSessionWithProgress((progress) => {
+    console.log(`Download progress: ${Math.round(progress * 100)}%`);
+  });
+}
+
+// Model is ready
+const result = streamText({
+  model,
+  messages: [{ role: "user", content: "Hello!" }],
+});
+```
+
+## Integration with useChat Hook
+
+When using this library with the `useChat` hook, you'll need to create a [custom transport](https://v5.ai-sdk.dev/docs/ai-sdk-ui/transport#transport) implementation to handle client-side AI with download progress. You can do this by importing `BuiltInAIUIMessage` from `@built-in-ai/core` that extends `UIMessage` to include [data parts](https://v5.ai-sdk.dev/docs/ai-sdk-ui/streaming-data) such as download progress.
+
+See the complete working example: **[`/examples/next-hybrid/app/(core)/util/client-side-chat-transport.ts`](<../../examples/next-hybrid/app/(core)/util/client-side-chat-transport.ts>)** and the **[`/examples/next-hybrid/app/page.tsx`](<../../examples/next-hybrid/app/(core)/page.tsx>)** components.
+
+This example includes:
+
+- Download progress with UI progress bar and status message updates
+- Hybrid client/server architecture with fallback
+- Error handling and notifications
+- Full integration with `useChat` hook
+
+## Multimodal Support
+
+The Prompt API supports both images and audio files:
+
+```typescript
+import { streamText } from "ai";
+import { builtInAI } from "@built-in-ai/core";
+
+const result = streamText({
+  model: builtInAI(),
+  messages: [
+    {
+      role: "user",
+      content: [
+        { type: "text", text: "What's in this image?" },
+        { type: "file", mediaType: "image/png", data: base64ImageData },
+      ],
+    },
+    {
+      role: "user",
+      content: [{ type: "file", mediaType: "audio/mp3", data: audioData }],
+    },
+  ],
+});
+
+for await (const chunk of result.textStream) {
+  console.log(chunk);
+}
+```
+
+## Tool Calling (with support for multiple steps)
+
+The `builtInAI` model supports tool calling, allowing the AI to use external functions and APIs. This is particularly useful for building AI agents that can perform actions or retrieve data:
+
+```typescript
+import { streamText, stepCountIs } from "ai";
+import { builtInAI } from "@built-in-ai/core";
+import { z } from "zod";
+
+const result = await streamText({
+  model: builtInAI(),
+  messages: [{ role: "user", content: "What's the weather in San Francisco?" }],
+  tools: {
+    search: {
+      description: "Search the web for information",
+      parameters: z.object({
+        query: z.string(),
+      }),
+      execute: async ({ query }) => {
+        // Search implementation
+        return { results: [{ title: "...", url: "..." }] };
+      },
+    },
+    fetchContent: {
+      description: "Fetch the content of a URL",
+      parameters: z.object({
+        url: z.string(),
+      }),
+      execute: async ({ url }) => {
+        // Fetch implementation
+        return { content: "Article content..." };
+      },
+    },
+  },
+  stopWhen: stepCountIs(5), // allow multiple steps
+});
+```
+
+## Generating Structured Data
+
+The `builtInAI` model also allows using the AI SDK `generateObject` and `streamObject`:
+
+### streamObject
+
+```typescript
+import { streamObject } from "ai";
+import { builtInAI } from "@built-in-ai/core";
+
+const { object } = await streamObject({
+  model: builtInAI(),
+  schema: z.object({
+    recipe: z.object({
+      name: z.string(),
+      ingredients: z.array(z.object({ name: z.string(), amount: z.string() })),
+      steps: z.array(z.string()),
+    }),
+  }),
+  prompt: "Generate a lasagna recipe.",
+});
+```
+
+### generateObject
+
+```typescript
+const { object } = await generateObject({
+  model: builtInAI(),
+  schema: z.object({
+    recipe: z.object({
+      name: z.string(),
+      ingredients: z.array(z.object({ name: z.string(), amount: z.string() })),
+      steps: z.array(z.string()),
+    }),
+  }),
+  prompt: "Generate a lasagna recipe.",
+});
+```
+
+## Features
+
+### Supported
+
+- [x] **Text generation** (`generateText()`)
+- [x] **Streaming responses** (`streamText()`)
+- [x] **Download progress streaming** - Real-time progress updates during model downloads
+- [x] **Multimodal functionality** (image and audio support)\*
+- [x] **Temperature control**
+- [x] **Response format constraints** (JSON `generateObject()/streamObject()`)
+- [x] **Tool calling** - Full support for function calling with JSON format
+- [x] **Abort signals**
+
+### Planned (when implemented in the Prompt API)
+
+- [ ] **Token counting**
+- [ ] **Custom stop sequences**
+- [ ] **Presence/frequency penalties**
+
+> \*Multimodal functionality is currently only available in Chrome's Prompt API implementation
+
+## API Reference
+
+### `builtInAI(modelId?, settings?)`
+
+Creates a browser AI model instance for chat or embeddings.
+
+**For Chat Models:**
+
+- `modelId` (optional): The model identifier, defaults to 'text'
+- `settings` (optional): Configuration options for the chat model
+  - `temperature?: number` - Controls randomness (0-1)
+  - `topK?: number` - Limits vocabulary selection
+
+**Returns:** `BuiltInAIChatLanguageModel` instance
+
+**For Embedding Models:**
+
+- `modelId`: Must be 'embedding'
+- `settings` (optional): Configuration options for the embedding model
+  - `wasmLoaderPath?: string` - Path to WASM loader (default: CDN hosted)
+  - `wasmBinaryPath?: string` - Path to WASM binary (default: CDN hosted)
+  - `modelAssetPath?: string` - Path to model asset file (default: CDN hosted)
+  - `l2Normalize?: boolean` - Whether to normalize with L2 norm (default: false)
+  - `quantize?: boolean` - Whether to quantize embeddings to bytes (default: false)
+  - `delegate?: 'CPU' | 'GPU'` - Backend to use for inference
+
+**Returns:** `BuiltInAIEmbeddingModel` instance
+
+### `doesBrowserSupportBuiltInAI(): boolean`
+
+Quick check if the browser supports the built-in AI API. Useful for component-level decisions and feature flags.
+
+**Returns:** `boolean` - `true` if browser supports the Prompt API, `false` otherwise
+
+**Example:**
+
+```typescript
+import { doesBrowserSupportBuiltInAI } from "@built-in-ai/core";
+
+if (doesBrowserSupportBuiltInAI()) {
+  // Show built-in AI option in UI
+} else {
+  // Show server-side option only
+}
+```
+
+### `BuiltInAIUIMessage`
+
+Extended UI message type for use with the `useChat` hook that includes custom data parts for built-in AI functionality.
+
+**Type Definition:**
+
+```typescript
+type BuiltInAIUIMessage = UIMessage<
+  never,
+  {
+    modelDownloadProgress: {
+      status: "downloading" | "complete" | "error";
+      progress?: number;
+      message: string;
+    };
+    notification: {
+      message: string;
+      level: "info" | "warning" | "error";
+    };
+  }
+>;
+```
+
+**Data Parts:**
+
+- `modelDownloadProgress` - Tracks browser AI model download status and progress
+- `notification` - Displays temporary messages and alerts to users
+
+### `BuiltInAIChatLanguageModel.createSessionWithProgress(onDownloadProgress?)`
+
+Creates a language model session with optional download progress monitoring.
+
+**Parameters:**
+
+- `onDownloadProgress?: (progress: number) => void` - Optional callback that receives progress values from 0 to 1 during model download
+
+**Returns:** `Promise<LanguageModel>` - The configured language model session
+
+**Example:**
+
+```typescript
+const model = builtInAI();
+await model.createSessionWithProgress((progress) => {
+  console.log(`Download: ${Math.round(progress * 100)}%`);
+});
+```
+
+### `BuiltInAIChatLanguageModel.availability()`
+
+Checks the current availability status of the built-in AI model.
+
+**Returns:** `Promise<"unavailable" | "downloadable" | "downloading" | "available">`
+
+- `"unavailable"` - Model is not supported in the browser
+- `"downloadable"` - Model is supported but needs to be downloaded first
+- `"downloading"` - Model is currently being downloaded
+- `"available"` - Model is ready to use
+
+## Author
+
+2025 © Jakob Hoeg Mørk

package/dist/index.d.mts

@@ -0,0 +1,243 @@
+import { LanguageModelV2, LanguageModelV2CallOptions, LanguageModelV2Content, LanguageModelV2FinishReason, LanguageModelV2CallWarning, LanguageModelV2StreamPart, EmbeddingModelV2, EmbeddingModelV2Embedding, ProviderV2 } from '@ai-sdk/provider';
+import { TextEmbedder } from '@mediapipe/tasks-text';
+import { UIMessage } from 'ai';
+
+type BuiltInAIChatModelId = "text";
+interface BuiltInAIChatSettings extends LanguageModelCreateOptions {
+    /**
+     * Expected input types for the session, for multimodal inputs.
+     */
+    expectedInputs?: Array<{
+        type: "text" | "image" | "audio";
+        languages?: string[];
+    }>;
+}
+/**
+ * Check if the browser supports the built-in AI API
+ * @returns true if the browser supports the built-in AI API, false otherwise
+ */
+declare function doesBrowserSupportBuiltInAI(): boolean;
+/**
+ * Check if the Prompt API is available
+ * @deprecated Use `doesBrowserSupportBuiltInAI()` instead for clearer naming
+ * @returns true if the browser supports the built-in AI API, false otherwise
+ */
+declare function isBuiltInAIModelAvailable(): boolean;
+declare class BuiltInAIChatLanguageModel implements LanguageModelV2 {
+    readonly specificationVersion = "v2";
+    readonly modelId: BuiltInAIChatModelId;
+    readonly provider = "browser-ai";
+    private readonly config;
+    private readonly sessionManager;
+    constructor(modelId: BuiltInAIChatModelId, options?: BuiltInAIChatSettings);
+    readonly supportedUrls: Record<string, RegExp[]>;
+    /**
+     * Gets a session with the specified options
+     * Delegates to SessionManager for all session lifecycle management
+     * @private
+     */
+    private getSession;
+    private getArgs;
+    /**
+     * Generates a complete text response using the browser's built-in Prompt API
+     * @param options
+     * @returns Promise resolving to the generated content with finish reason, usage stats, and any warnings
+     * @throws {LoadSettingError} When the Prompt API is not available or model needs to be downloaded
+     * @throws {UnsupportedFunctionalityError} When unsupported features like file input are used
+     */
+    doGenerate(options: LanguageModelV2CallOptions): Promise<{
+        content: LanguageModelV2Content[];
+        finishReason: LanguageModelV2FinishReason;
+        usage: {
+            inputTokens: undefined;
+            outputTokens: undefined;
+            totalTokens: undefined;
+        };
+        request: {
+            body: {
+                messages: LanguageModelMessage[];
+                options: LanguageModelPromptOptions & LanguageModelCreateCoreOptions;
+            };
+        };
+        warnings: LanguageModelV2CallWarning[];
+    }>;
+    /**
+     * Check the availability of the built-in AI model
+     * @returns Promise resolving to "unavailable", "available", or "available-after-download"
+     */
+    availability(): Promise<Availability>;
+    /**
+     * Creates a session with download progress monitoring.
+     *
+     * @example
+     * ```typescript
+     * const session = await model.createSessionWithProgress(
+     *   (progress) => {
+     *     console.log(`Download progress: ${Math.round(progress * 100)}%`);
+     *   }
+     * );
+     * ```
+     *
+     * @param onDownloadProgress Optional callback receiving progress values 0-1 during model download
+     * @returns Promise resolving to a configured LanguageModel session
+     * @throws {LoadSettingError} When the Prompt API is not available or model is unavailable
+     */
+    createSessionWithProgress(onDownloadProgress?: (progress: number) => void): Promise<LanguageModel>;
+    /**
+     * Generates a streaming text response using the browser's built-in Prompt API
+     * @param options
+     * @returns Promise resolving to a readable stream of text chunks and request metadata
+     * @throws {LoadSettingError} When the Prompt API is not available or model needs to be downloaded
+     * @throws {UnsupportedFunctionalityError} When unsupported features like file input are used
+     */
+    doStream(options: LanguageModelV2CallOptions): Promise<{
+        stream: ReadableStream<LanguageModelV2StreamPart>;
+        request: {
+            body: {
+                messages: LanguageModelMessage[];
+                options: LanguageModelPromptOptions & LanguageModelCreateCoreOptions;
+            };
+        };
+    }>;
+}
+
+interface BuiltInAIEmbeddingModelSettings {
+    /**
+     * An optional base path to specify the directory the Wasm files should be loaded from.
+     * @default 'https://pub-ddcfe353995744e89b8002f16bf98575.r2.dev/text_wasm_internal.js'
+     */
+    wasmLoaderPath?: string;
+    /**
+     * It's about 6mb before gzip.
+     * @default 'https://pub-ddcfe353995744e89b8002f16bf98575.r2.dev/text_wasm_internal.wasm'
+     */
+    wasmBinaryPath?: string;
+    /**
+     * The model path to the model asset file.
+     * It's about 6.1mb before gzip.
+     * @default 'https://pub-ddcfe353995744e89b8002f16bf98575.r2.dev/universal_sentence_encoder.tflite'
+     */
+    modelAssetPath?: string;
+    /**
+     * Whether to normalize the returned feature vector with L2 norm. Use this
+     * option only if the model does not already contain a native L2_NORMALIZATION
+     * TF Lite Op. In most cases, this is already the case and L2 norm is thus
+     * achieved through TF Lite inference.
+     * @default false
+     */
+    l2Normalize?: boolean;
+    /**
+     * Whether the returned embedding should be quantized to bytes via scalar
+     * quantization. Embeddings are implicitly assumed to be unit-norm and
+     * therefore any dimension is guaranteed to have a value in [-1.0, 1.0]. Use
+     * the l2_normalize option if this is not the case.
+     * @default false
+     */
+    quantize?: boolean;
+    /**
+     * Overrides the default backend to use for the provided model.
+     */
+    delegate?: "CPU" | "GPU";
+}
+declare class BuiltInAIEmbeddingModel implements EmbeddingModelV2<string> {
+    readonly specificationVersion = "v2";
+    readonly provider = "google-mediapipe";
+    readonly modelId: string;
+    readonly supportsParallelCalls = true;
+    readonly maxEmbeddingsPerCall: undefined;
+    private settings;
+    private modelAssetBuffer;
+    private textEmbedder;
+    constructor(settings?: BuiltInAIEmbeddingModelSettings);
+    protected getTextEmbedder: () => Promise<TextEmbedder>;
+    doEmbed: (options: {
+        values: string[];
+        abortSignal?: AbortSignal;
+    }) => Promise<{
+        embeddings: Array<EmbeddingModelV2Embedding>;
+        rawResponse?: Record<PropertyKey, any>;
+    }>;
+}
+
+interface BuiltInAIProvider extends ProviderV2 {
+    (modelId?: BuiltInAIChatModelId, settings?: BuiltInAIChatSettings): BuiltInAIChatLanguageModel;
+    /**
+     * Creates a model for text generation.
+     */
+    languageModel(modelId: BuiltInAIChatModelId, settings?: BuiltInAIChatSettings): BuiltInAIChatLanguageModel;
+    /**
+     * Creates a model for text generation.
+     */
+    chat(modelId: BuiltInAIChatModelId, settings?: BuiltInAIChatSettings): BuiltInAIChatLanguageModel;
+    textEmbedding(modelId: "embedding", settings?: BuiltInAIEmbeddingModelSettings): EmbeddingModelV2<string>;
+    textEmbeddingModel: (modelId: "embedding", settings?: BuiltInAIEmbeddingModelSettings) => EmbeddingModelV2<string>;
+    imageModel(modelId: string): never;
+    speechModel(modelId: string): never;
+    transcriptionModel(modelId: string): never;
+}
+interface BuiltInAIProviderSettings {
+}
+/**
+ * Create a BuiltInAI provider instance.
+ */
+declare function createBuiltInAI(options?: BuiltInAIProviderSettings): BuiltInAIProvider;
+/**
+ * Default BuiltInAI provider instance.
+ */
+declare const builtInAI: BuiltInAIProvider;
+
+/**
+ * UI message type for built-in AI features with custom data parts.
+ *
+ * Extends base UIMessage to include specific data part schemas
+ * for built-in AI functionality such as model download progress tracking
+ *
+ * @example
+ * // Import and use with useChat hook from @ai-sdk/react
+ * ```typescript
+ * import { useChat } from "@ai-sdk/react";
+ * import { BuiltInAIUIMessage } from "@built-in-ai/core";
+ *
+ * const { messages, sendMessage } = useChat<BuiltInAIUIMessage>({
+ *   onData: (dataPart) => {
+ *     if (dataPart.type === 'data-modelDownloadProgress') {
+ *       console.log(`Download: ${dataPart.data.progress}%`);
+ *     }
+ *     if (dataPart.type === 'data-notification') {
+ *       console.log(`${dataPart.data.level}: ${dataPart.data.message}`);
+ *     }
+ *   }
+ * });
+ * ```
+ *
+ * @see {@link https://v5.ai-sdk.dev/docs/reference/ai-sdk-ui/use-chat | useChat hook documentation}
+ */
+type BuiltInAIUIMessage = UIMessage<
+  never, // No custom metadata type
+  {
+    /**
+     * Model download progress data part for tracking browser AI model download status.
+     * Used to display download progress bars and status messages to users.
+     */
+    modelDownloadProgress: {
+      /** Current download/initialization status */
+      status: "downloading" | "complete" | "error";
+      /** Download progress percentage (0-100), undefined for non-downloading states */
+      progress?: number;
+      /** Human-readable status message to display to users */
+      message: string;
+    };
+    /**
+     * User notification data part for displaying temporary messages and alerts.
+     * These are typically transient and not persisted in message history.
+     */
+    notification: {
+      /** The notification message text */
+      message: string;
+      /** Notification severity level for styling and priority */
+      level: "info" | "warning" | "error";
+    };
+  }
+>;
+
+export { BuiltInAIChatLanguageModel, type BuiltInAIChatSettings, BuiltInAIEmbeddingModel, type BuiltInAIEmbeddingModelSettings, type BuiltInAIProvider, type BuiltInAIProviderSettings, type BuiltInAIUIMessage, builtInAI, createBuiltInAI, doesBrowserSupportBuiltInAI, isBuiltInAIModelAvailable };