@fondation-io/ai 7.0.0-beta.45

package/src/util/create-stitchable-stream.ts

@@ -0,0 +1,112 @@
+import { createResolvablePromise } from './create-resolvable-promise';
+
+/**
+ * Creates a stitchable stream that can pipe one stream at a time.
+ *
+ * @template T - The type of values emitted by the streams.
+ * @returns {Object} An object containing the stitchable stream and control methods.
+ */
+export function createStitchableStream<T>(): {
+  stream: ReadableStream<T>;
+  addStream: (innerStream: ReadableStream<T>) => void;
+  close: () => void;
+  terminate: () => void;
+} {
+  let innerStreamReaders: ReadableStreamDefaultReader<T>[] = [];
+  let controller: ReadableStreamDefaultController<T> | null = null;
+  let isClosed = false;
+  let waitForNewStream = createResolvablePromise<void>();
+
+  const terminate = () => {
+    isClosed = true;
+    waitForNewStream.resolve();
+
+    innerStreamReaders.forEach(reader => reader.cancel());
+    innerStreamReaders = [];
+    controller?.close();
+  };
+
+  const processPull = async () => {
+    // Case 1: Outer stream is closed and no more inner streams
+    if (isClosed && innerStreamReaders.length === 0) {
+      controller?.close();
+      return;
+    }
+
+    // Case 2: No inner streams available, but outer stream is open
+    // wait for a new inner stream to be added or the outer stream to close
+    if (innerStreamReaders.length === 0) {
+      waitForNewStream = createResolvablePromise<void>();
+      await waitForNewStream.promise;
+      return processPull();
+    }
+
+    try {
+      const { value, done } = await innerStreamReaders[0].read();
+
+      if (done) {
+        // Case 3: Current inner stream is done
+        innerStreamReaders.shift(); // Remove the finished stream
+
+        if (innerStreamReaders.length === 0 && isClosed) {
+          // when closed and no more inner streams, stop pulling
+          controller?.close();
+        } else {
+          // continue pulling from the next stream
+          await processPull();
+        }
+      } else {
+        // Case 4: Current inner stream returns an item
+        controller?.enqueue(value);
+      }
+    } catch (error) {
+      // Case 5: Current inner stream throws an error
+      controller?.error(error);
+      innerStreamReaders.shift(); // Remove the errored stream
+      terminate(); // we have errored, terminate all streams
+    }
+  };
+
+  return {
+    stream: new ReadableStream<T>({
+      start(controllerParam) {
+        controller = controllerParam;
+      },
+      pull: processPull,
+      async cancel() {
+        for (const reader of innerStreamReaders) {
+          await reader.cancel();
+        }
+        innerStreamReaders = [];
+        isClosed = true;
+      },
+    }),
+    addStream: (innerStream: ReadableStream<T>) => {
+      if (isClosed) {
+        throw new Error('Cannot add inner stream: outer stream is closed');
+      }
+
+      innerStreamReaders.push(innerStream.getReader());
+      waitForNewStream.resolve();
+    },
+
+    /**
+     * Gracefully close the outer stream. This will let the inner streams
+     * finish processing and then close the outer stream.
+     */
+    close: () => {
+      isClosed = true;
+      waitForNewStream.resolve();
+
+      if (innerStreamReaders.length === 0) {
+        controller?.close();
+      }
+    },
+
+    /**
+     * Immediately close the outer stream. This will cancel all inner streams
+     * and close the outer stream.
+     */
+    terminate,
+  };
+}

package/src/util/data-url.ts

@@ -0,0 +1,17 @@
+/**
+ * Converts a data URL of type text/* to a text string.
+ */
+export function getTextFromDataUrl(dataUrl: string): string {
+  const [header, base64Content] = dataUrl.split(',');
+  const mediaType = header.split(';')[0].split(':')[1];
+
+  if (mediaType == null || base64Content == null) {
+    throw new Error('Invalid data URL format');
+  }
+
+  try {
+    return window.atob(base64Content);
+  } catch (error) {
+    throw new Error(`Error decoding data URL`);
+  }
+}

package/src/util/deep-partial.ts

@@ -0,0 +1,84 @@
+// License for this File only:
+//
+// MIT License
+//
+// Copyright (c) Sindre Sorhus <sindresorhus@gmail.com> (https://sindresorhus.com)
+// Copyright (c) Vercel, Inc. (https://vercel.com)
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+// documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+// to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions
+// of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+// CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+
+import { FlexibleSchema, InferSchema } from '@ai-sdk/provider-utils';
+
+/**
+ * Create a type from an object with all keys and nested keys set to optional.
+ * The helper supports normal objects and schemas (which are resolved automatically).
+ * It always recurses into arrays.
+ *
+ * Adopted from [type-fest](https://github.com/sindresorhus/type-fest/tree/main) PartialDeep.
+ */
+
+export type DeepPartial<T> = T extends FlexibleSchema
+  ? DeepPartialInternal<InferSchema<T>> // resolve schemas first to prevent infinite recursion
+  : DeepPartialInternal<T>;
+
+type DeepPartialInternal<T> = T extends
+  | null
+  | undefined
+  | string
+  | number
+  | boolean
+  | symbol
+  | bigint
+  | void
+  | Date
+  | RegExp
+  | ((...arguments_: any[]) => unknown)
+  | (new (...arguments_: any[]) => unknown)
+  ? T
+  : T extends Map<infer KeyType, infer ValueType>
+    ? PartialMap<KeyType, ValueType>
+    : T extends Set<infer ItemType>
+      ? PartialSet<ItemType>
+      : T extends ReadonlyMap<infer KeyType, infer ValueType>
+        ? PartialReadonlyMap<KeyType, ValueType>
+        : T extends ReadonlySet<infer ItemType>
+          ? PartialReadonlySet<ItemType>
+          : T extends object
+            ? T extends ReadonlyArray<infer ItemType> // Test for arrays/tuples, per https://github.com/microsoft/TypeScript/issues/35156
+              ? ItemType[] extends T // Test for arrays (non-tuples) specifically
+                ? readonly ItemType[] extends T // Differentiate readonly and mutable arrays
+                  ? ReadonlyArray<DeepPartialInternal<ItemType | undefined>>
+                  : Array<DeepPartialInternal<ItemType | undefined>>
+                : PartialObject<T> // Tuples behave properly
+              : PartialObject<T>
+            : unknown;
+
+type PartialMap<KeyType, ValueType> = {} & Map<
+  DeepPartialInternal<KeyType>,
+  DeepPartialInternal<ValueType>
+>;
+
+type PartialSet<T> = {} & Set<DeepPartialInternal<T>>;
+
+type PartialReadonlyMap<KeyType, ValueType> = {} & ReadonlyMap<
+  DeepPartialInternal<KeyType>,
+  DeepPartialInternal<ValueType>
+>;
+
+type PartialReadonlySet<T> = {} & ReadonlySet<DeepPartialInternal<T>>;
+
+type PartialObject<ObjectType extends object> = {
+  [KeyType in keyof ObjectType]?: DeepPartialInternal<ObjectType[KeyType]>;
+};

package/src/util/detect-media-type.ts

@@ -0,0 +1,226 @@
+import { convertBase64ToUint8Array } from '@ai-sdk/provider-utils';
+
+export const imageMediaTypeSignatures = [
+  {
+    mediaType: 'image/gif' as const,
+    bytesPrefix: [0x47, 0x49, 0x46], // GIF
+  },
+  {
+    mediaType: 'image/png' as const,
+    bytesPrefix: [0x89, 0x50, 0x4e, 0x47], // PNG
+  },
+  {
+    mediaType: 'image/jpeg' as const,
+    bytesPrefix: [0xff, 0xd8], // JPEG
+  },
+  {
+    mediaType: 'image/webp' as const,
+    bytesPrefix: [
+      0x52,
+      0x49,
+      0x46,
+      0x46, // "RIFF"
+      null,
+      null,
+      null,
+      null, // file size (variable)
+      0x57,
+      0x45,
+      0x42,
+      0x50, // "WEBP"
+    ],
+  },
+  {
+    mediaType: 'image/bmp' as const,
+    bytesPrefix: [0x42, 0x4d],
+  },
+  {
+    mediaType: 'image/tiff' as const,
+    bytesPrefix: [0x49, 0x49, 0x2a, 0x00],
+  },
+  {
+    mediaType: 'image/tiff' as const,
+    bytesPrefix: [0x4d, 0x4d, 0x00, 0x2a],
+  },
+  {
+    mediaType: 'image/avif' as const,
+    bytesPrefix: [
+      0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70, 0x61, 0x76, 0x69, 0x66,
+    ],
+  },
+  {
+    mediaType: 'image/heic' as const,
+    bytesPrefix: [
+      0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63,
+    ],
+  },
+] as const;
+
+export const audioMediaTypeSignatures = [
+  {
+    mediaType: 'audio/mpeg' as const,
+    bytesPrefix: [0xff, 0xfb],
+  },
+  {
+    mediaType: 'audio/mpeg' as const,
+    bytesPrefix: [0xff, 0xfa],
+  },
+  {
+    mediaType: 'audio/mpeg' as const,
+    bytesPrefix: [0xff, 0xf3],
+  },
+  {
+    mediaType: 'audio/mpeg' as const,
+    bytesPrefix: [0xff, 0xf2],
+  },
+  {
+    mediaType: 'audio/mpeg' as const,
+    bytesPrefix: [0xff, 0xe3],
+  },
+  {
+    mediaType: 'audio/mpeg' as const,
+    bytesPrefix: [0xff, 0xe2],
+  },
+  {
+    mediaType: 'audio/wav' as const,
+    bytesPrefix: [
+      0x52, // R
+      0x49, // I
+      0x46, // F
+      0x46, // F
+      null,
+      null,
+      null,
+      null,
+      0x57, // W
+      0x41, // A
+      0x56, // V
+      0x45, // E
+    ],
+  },
+  {
+    mediaType: 'audio/ogg' as const,
+    bytesPrefix: [0x4f, 0x67, 0x67, 0x53],
+  },
+  {
+    mediaType: 'audio/flac' as const,
+    bytesPrefix: [0x66, 0x4c, 0x61, 0x43],
+  },
+  {
+    mediaType: 'audio/aac' as const,
+    bytesPrefix: [0x40, 0x15, 0x00, 0x00],
+  },
+  {
+    mediaType: 'audio/mp4' as const,
+    bytesPrefix: [0x66, 0x74, 0x79, 0x70],
+  },
+  {
+    mediaType: 'audio/webm',
+    bytesPrefix: [0x1a, 0x45, 0xdf, 0xa3],
+  },
+] as const;
+
+export const videoMediaTypeSignatures = [
+  {
+    mediaType: 'video/mp4' as const,
+    bytesPrefix: [
+      0x00,
+      0x00,
+      0x00,
+      null,
+      0x66,
+      0x74,
+      0x79,
+      0x70, // ftyp
+    ],
+  },
+  {
+    mediaType: 'video/webm' as const,
+    bytesPrefix: [0x1a, 0x45, 0xdf, 0xa3], // EBML
+  },
+  {
+    mediaType: 'video/quicktime' as const,
+    bytesPrefix: [
+      0x00,
+      0x00,
+      0x00,
+      0x14,
+      0x66,
+      0x74,
+      0x79,
+      0x70,
+      0x71,
+      0x74, // ftypqt
+    ],
+  },
+  {
+    mediaType: 'video/x-msvideo' as const,
+    bytesPrefix: [0x52, 0x49, 0x46, 0x46], // RIFF (AVI)
+  },
+] as const;
+
+const stripID3 = (data: Uint8Array | string) => {
+  const bytes =
+    typeof data === 'string' ? convertBase64ToUint8Array(data) : data;
+  const id3Size =
+    ((bytes[6] & 0x7f) << 21) |
+    ((bytes[7] & 0x7f) << 14) |
+    ((bytes[8] & 0x7f) << 7) |
+    (bytes[9] & 0x7f);
+
+  // The raw MP3 starts here
+  return bytes.slice(id3Size + 10);
+};
+
+function stripID3TagsIfPresent(data: Uint8Array | string): Uint8Array | string {
+  const hasId3 =
+    (typeof data === 'string' && data.startsWith('SUQz')) ||
+    (typeof data !== 'string' &&
+      data.length > 10 &&
+      data[0] === 0x49 && // 'I'
+      data[1] === 0x44 && // 'D'
+      data[2] === 0x33); // '3'
+
+  return hasId3 ? stripID3(data) : data;
+}
+
+/**
+ * Detect the media IANA media type of a file using a list of signatures.
+ *
+ * @param data - The file data.
+ * @param signatures - The signatures to use for detection.
+ * @returns The media type of the file.
+ */
+export function detectMediaType({
+  data,
+  signatures,
+}: {
+  data: Uint8Array | string;
+  signatures:
+    | typeof audioMediaTypeSignatures
+    | typeof imageMediaTypeSignatures
+    | typeof videoMediaTypeSignatures;
+}): (typeof signatures)[number]['mediaType'] | undefined {
+  const processedData = stripID3TagsIfPresent(data);
+
+  // Convert the first ~18 bytes (24 base64 chars) for consistent detection logic:
+  const bytes =
+    typeof processedData === 'string'
+      ? convertBase64ToUint8Array(
+          processedData.substring(0, Math.min(processedData.length, 24)),
+        )
+      : processedData;
+
+  for (const signature of signatures) {
+    if (
+      bytes.length >= signature.bytesPrefix.length &&
+      signature.bytesPrefix.every(
+        (byte, index) => byte === null || bytes[index] === byte,
+      )
+    ) {
+      return signature.mediaType;
+    }
+  }
+
+  return undefined;
+}

package/src/util/download/create-download.ts

@@ -0,0 +1,13 @@
+import { download as internalDownload } from './download';
+
+/**
+ * Creates a download function with configurable options.
+ *
+ * @param options - Configuration options for the download function.
+ * @param options.maxBytes - Maximum allowed download size in bytes. Default: 2 GiB.
+ * @returns A download function that can be passed to `transcribe()` or `experimental_generateVideo()`.
+ */
+export function createDownload(options?: { maxBytes?: number }) {
+  return ({ url, abortSignal }: { url: URL; abortSignal?: AbortSignal }) =>
+    internalDownload({ url, maxBytes: options?.maxBytes, abortSignal });
+}

package/src/util/download/download-function.ts

@@ -0,0 +1,45 @@
+import { download as originalDownload } from './download';
+
+/**
+ * Experimental. Can change in patch versions without warning.
+ *
+ * Download function. Called with the array of URLs and a boolean indicating
+ * whether the URL is supported by the model.
+ *
+ * The download function can decide for each URL:
+ * - to return null (which means that the URL should be passed to the model)
+ * - to download the asset and return the data (incl. retries, authentication, etc.)
+ *
+ * Should throw DownloadError if the download fails.
+ *
+ * Should return an array of objects sorted by the order of the requested downloads.
+ * For each object, the data should be a Uint8Array if the URL was downloaded.
+ * For each object, the mediaType should be the media type of the downloaded asset.
+ * For each object, the data should be null if the URL should be passed through as is.
+ */
+export type DownloadFunction = (
+  options: Array<{
+    url: URL;
+    isUrlSupportedByModel: boolean;
+  }>,
+) => PromiseLike<
+  Array<{
+    data: Uint8Array;
+    mediaType: string | undefined;
+  } | null>
+>;
+
+/**
+ * Default download function.
+ * Downloads the file if it is not supported by the model.
+ */
+export const createDefaultDownloadFunction =
+  (download: typeof originalDownload = originalDownload): DownloadFunction =>
+  requestedDownloads =>
+    Promise.all(
+      requestedDownloads.map(async requestedDownload =>
+        requestedDownload.isUrlSupportedByModel
+          ? null
+          : download(requestedDownload),
+      ),
+    );

package/src/util/download/download.ts

@@ -0,0 +1,74 @@
+import {
+  DownloadError,
+  readResponseWithSizeLimit,
+  DEFAULT_MAX_DOWNLOAD_SIZE,
+  validateDownloadUrl,
+} from '@ai-sdk/provider-utils';
+import {
+  withUserAgentSuffix,
+  getRuntimeEnvironmentUserAgent,
+} from '@ai-sdk/provider-utils';
+import { VERSION } from '../../version';
+
+/**
+ * Download a file from a URL.
+ *
+ * @param url - The URL to download from.
+ * @param maxBytes - Maximum allowed download size in bytes. Defaults to 100 MiB.
+ * @param abortSignal - An optional abort signal to cancel the download.
+ * @returns The downloaded data and media type.
+ *
+ * @throws DownloadError if the download fails or exceeds maxBytes.
+ */
+export const download = async ({
+  url,
+  maxBytes,
+  abortSignal,
+}: {
+  url: URL;
+  maxBytes?: number;
+  abortSignal?: AbortSignal;
+}) => {
+  const urlText = url.toString();
+  validateDownloadUrl(urlText);
+  try {
+    const response = await fetch(urlText, {
+      headers: withUserAgentSuffix(
+        {},
+        `ai-sdk/${VERSION}`,
+        getRuntimeEnvironmentUserAgent(),
+      ),
+      signal: abortSignal,
+    });
+
+    // Validate final URL after redirects to prevent SSRF via open redirect
+    if (response.redirected) {
+      validateDownloadUrl(response.url);
+    }
+
+    if (!response.ok) {
+      throw new DownloadError({
+        url: urlText,
+        statusCode: response.status,
+        statusText: response.statusText,
+      });
+    }
+
+    const data = await readResponseWithSizeLimit({
+      response,
+      url: urlText,
+      maxBytes: maxBytes ?? DEFAULT_MAX_DOWNLOAD_SIZE,
+    });
+
+    return {
+      data,
+      mediaType: response.headers.get('content-type') ?? undefined,
+    };
+  } catch (error) {
+    if (DownloadError.isInstance(error)) {
+      throw error;
+    }
+
+    throw new DownloadError({ url: urlText, cause: error });
+  }
+};

package/src/util/error-handler.ts

@@ -0,0 +1 @@
+export type ErrorHandler = (error: unknown) => void;