@agorapete/wllama 3.5.1-q2.0

package/src/utils.test.ts

@@ -0,0 +1,231 @@
+import { expect, test, describe } from 'vitest';
+import {
+  joinBuffers,
+  bufToText,
+  padDigits,
+  sumArr,
+  isString,
+  delay,
+  absoluteUrl,
+  parseShardNumber,
+  parseModelUrl,
+  sortFileByShard,
+  isValidGgufFile,
+} from './utils';
+
+describe('joinBuffers', () => {
+  test('joins multiple buffers correctly', () => {
+    const buf1 = new Uint8Array([1, 2, 3]);
+    const buf2 = new Uint8Array([4, 5]);
+
+    const result = joinBuffers([buf1, buf2]);
+
+    expect(result).toEqual(new Uint8Array([1, 2, 3, 4, 5]));
+  });
+});
+
+describe('bufToText', () => {
+  test('converts buffer to string', () => {
+    const buffer = new TextEncoder().encode('hello world');
+    expect(bufToText(buffer)).toBe('hello world');
+  });
+});
+
+describe('padDigits', () => {
+  test('pads number with zeros', () => {
+    expect(padDigits(5, 3)).toBe('005');
+    expect(padDigits(42, 4)).toBe('0042');
+    expect(padDigits(1234, 2)).toBe('1234');
+  });
+});
+
+describe('sumArr', () => {
+  test('sums array of numbers', () => {
+    expect(sumArr([1, 2, 3])).toBe(6);
+    expect(sumArr([])).toBe(0);
+    expect(sumArr([5])).toBe(5);
+  });
+});
+
+describe('isString', () => {
+  test('checks if value is string', () => {
+    expect(isString('test')).toBe(true);
+    expect(isString('')).toBe(true);
+    expect(isString(null)).toBe(false);
+    expect(isString(undefined)).toBe(false);
+    expect(isString(123)).toBe(false);
+  });
+});
+
+describe('delay', () => {
+  test('delays execution', async () => {
+    const start = Date.now();
+    await delay(100);
+    const elapsed = Date.now() - start;
+    expect(elapsed).toBeGreaterThanOrEqual(100);
+  });
+});
+
+describe('absoluteUrl', () => {
+  test('converts relative path to absolute url', () => {
+    // Mock document.baseURI
+    Object.defineProperty(document, 'baseURI', {
+      value: 'http://example.com/app/',
+      writable: true,
+    });
+
+    expect(absoluteUrl('test.html')).toBe('http://example.com/app/test.html');
+    expect(absoluteUrl('/test.html')).toBe('http://example.com/test.html');
+  });
+});
+
+describe('shard processing', () => {
+  test('parseShardNumber extracts correct info', () => {
+    expect(parseShardNumber('abcdef-123456-00001-of-00005.gguf')).toEqual({
+      baseURL: 'abcdef-123456',
+      current: 1,
+      total: 5,
+    });
+
+    expect(parseShardNumber('abcdef-123456.9090-q8_0.gguf')).toEqual({
+      baseURL: 'abcdef-123456.9090-q8_0.gguf',
+      current: 1,
+      total: 1,
+    });
+
+    expect(
+      parseShardNumber('abcdef-123456-00001-of-00005.gguf?param=value')
+    ).toEqual({
+      baseURL: 'abcdef-123456',
+      current: 1,
+      total: 5,
+    });
+
+    expect(
+      parseShardNumber('abcdef-123456-00001-of-00005.gguf?no-inline')
+    ).toEqual({
+      baseURL: 'abcdef-123456',
+      current: 1,
+      total: 5,
+    });
+
+    expect(
+      parseShardNumber(
+        'abcdef-123456-00001-of-00005.gguf?param1=value1&param2=value2'
+      )
+    ).toEqual({
+      baseURL: 'abcdef-123456',
+      current: 1,
+      total: 5,
+    });
+  });
+
+  test('parseModelUrl generates correct shard URLs', () => {
+    const singleFile = 'model.gguf';
+    expect(parseModelUrl(singleFile)).toEqual(['model.gguf']);
+
+    const shardedFile = 'model-00001-of-00003.gguf';
+    expect(parseModelUrl(shardedFile)).toEqual([
+      'model-00001-of-00003.gguf',
+      'model-00002-of-00003.gguf',
+      'model-00003-of-00003.gguf',
+    ]);
+
+    const complexPath = 'https://example.com/models/llama-00001-of-00002.gguf';
+    expect(parseModelUrl(complexPath)).toEqual([
+      'https://example.com/models/llama-00001-of-00002.gguf',
+      'https://example.com/models/llama-00002-of-00002.gguf',
+    ]);
+
+    const shardedFileWithQuery = 'model-00001-of-00003.gguf?param=value';
+    expect(parseModelUrl(shardedFileWithQuery)).toEqual([
+      'model-00001-of-00003.gguf?param=value',
+      'model-00002-of-00003.gguf?param=value',
+      'model-00003-of-00003.gguf?param=value',
+    ]);
+
+    const complexPathWithQuery =
+      'https://example.com/models/llama-00001-of-00002.gguf?no-inline';
+    expect(parseModelUrl(complexPathWithQuery)).toEqual([
+      'https://example.com/models/llama-00001-of-00002.gguf?no-inline',
+      'https://example.com/models/llama-00002-of-00002.gguf?no-inline',
+    ]);
+  });
+
+  test('sortFileByShard sorts files by shard number', () => {
+    const files = [
+      new File(
+        [],
+        'e2fc714c4727ee9395f324cd2e7f331f-model-00003-of-00005.gguf'
+      ),
+      new File(
+        [],
+        '187ef4436122d1cc2f40dc2b92f0eba0-model-00001-of-00005.gguf'
+      ),
+      new File(
+        [],
+        'c4357687ea2b461cb07cf0a0a3de939f-model-00002-of-00005.gguf'
+      ),
+      new File(
+        [],
+        '6a4d40512eabd63221cbdf3df4636cd7-model-00005-of-00005.gguf'
+      ),
+      new File(
+        [],
+        '0952e4c6ba320f5278605eb5333eec0f-model-00004-of-00005.gguf'
+      ),
+    ];
+
+    sortFileByShard(files);
+
+    expect(files.map((f) => parseShardNumber(f.name).current)).toEqual([
+      1, 2, 3, 4, 5,
+    ]);
+
+    // Single file should not be affected
+    const singleFile = [new File([], 'model.gguf')];
+    sortFileByShard(singleFile);
+    expect(singleFile[0].name).toBe('model.gguf');
+
+    // Regular blobs should not be affected
+    const blobs = [new Blob(), new Blob()];
+    sortFileByShard(blobs);
+    expect(blobs.length).toBe(2);
+  });
+});
+
+describe('GGUF file validation', () => {
+  test('isValidGgufFile should correctly validate GGUF files', () => {
+    // Basic valid cases
+    expect(isValidGgufFile('model.gguf')).toBe(true);
+    expect(isValidGgufFile('path/to/model.gguf')).toBe(true);
+
+    // With Vite query parameters
+    expect(isValidGgufFile('model.gguf?no-inline')).toBe(true);
+    expect(isValidGgufFile('foo.gguf?no-inline')).toBe(true);
+
+    // With query parameters
+    expect(isValidGgufFile('model.gguf?param=value')).toBe(true);
+    expect(isValidGgufFile('model.gguf?param1&param2')).toBe(true);
+    expect(isValidGgufFile('model.gguf?no-inline&v=123')).toBe(true);
+    expect(isValidGgufFile('model.gguf?param1=value1&param2=value2')).toBe(
+      true
+    );
+    expect(isValidGgufFile('model.gguf?param=value&special=!@#$%^&*()')).toBe(
+      true
+    );
+    expect(isValidGgufFile('model.gguf?')).toBe(true);
+
+    // With fragments
+    expect(isValidGgufFile('model.gguf?param=value#fragment')).toBe(true);
+
+    // Invalid cases
+    expect(isValidGgufFile('model.bin')).toBe(false);
+    expect(isValidGgufFile('model.gguf.bin')).toBe(false);
+    expect(isValidGgufFile('modelgguf')).toBe(false);
+    expect(isValidGgufFile('path/to/model.txt?ext=gguf')).toBe(false);
+
+    // Case sensitivity
+    expect(isValidGgufFile('model.GGUF?param=value')).toBe(false);
+  });
+});

package/src/utils.ts

@@ -0,0 +1,403 @@
+export const joinBuffers = (buffers: Uint8Array[]): Uint8Array => {
+  const totalSize = buffers.reduce((acc, buf) => acc + buf.length, 0);
+  const output = new Uint8Array(totalSize);
+  output.set(buffers[0], 0);
+  for (let i = 1; i < buffers.length; i++) {
+    output.set(buffers[i], buffers[i - 1].length);
+  }
+  return output;
+};
+
+const textDecoder = new TextDecoder();
+
+/**
+ * Convert list of bytes (number) to text
+ * @param buffer
+ * @returns a string
+ */
+export const bufToText = (buffer: ArrayBuffer | Uint8Array): string => {
+  return textDecoder.decode(buffer);
+};
+
+/**
+ * Get default stdout/stderr config for wasm module
+ */
+export const getWModuleConfig = (pathConfig: {
+  [filename: string]: string;
+}) => {
+  return {
+    noInitialRun: true,
+    print: function (text: any) {
+      if (arguments.length > 1)
+        text = Array.prototype.slice.call(arguments).join(' ');
+      console.log(text);
+    },
+    printErr: function (text: any) {
+      if (arguments.length > 1)
+        text = Array.prototype.slice.call(arguments).join(' ');
+      console.warn(text);
+    },
+    // @ts-ignore
+    locateFile: function (filename: string, basePath: string) {
+      const p = pathConfig[filename];
+      console.log(`Loading "${filename}" from "${p}"`);
+      return p;
+    },
+  };
+};
+
+export interface ShardInfo {
+  baseURL: string;
+  current: number;
+  total: number;
+}
+
+const URL_PARTS_REGEX = /-(\d{5})-of-(\d{5})\.gguf(?:\?.*)?$/;
+
+/**
+ * Parse shard number and total from a file name or URL
+ */
+export const parseShardNumber = (fnameOrUrl: string): ShardInfo => {
+  const matches = fnameOrUrl.match(URL_PARTS_REGEX);
+  if (!matches) {
+    return {
+      baseURL: fnameOrUrl,
+      current: 1,
+      total: 1,
+    };
+  } else {
+    return {
+      baseURL: fnameOrUrl.replace(URL_PARTS_REGEX, ''),
+      current: parseInt(matches[1]),
+      total: parseInt(matches[2]),
+    };
+  }
+};
+
+/**
+ * Parses a model URL and returns an array of URLs based on the following patterns:
+ * - If the input URL is an array, it returns the array itself.
+ * - If the input URL is a string in the `gguf-split` format, it returns an array containing the URL of each shard in ascending order.
+ * - Otherwise, it returns an array containing the input URL as a single element array.
+ * @param modelUrl URL or list of URLs
+ */
+export const parseModelUrl = (modelUrl: string): string[] => {
+  const { baseURL, current, total } = parseShardNumber(modelUrl);
+  if (current == total && total == 1) {
+    return [modelUrl];
+  } else {
+    const queryMatch = modelUrl.match(/\.gguf(\?.*)?$/);
+    const queryParams = queryMatch?.[1] ?? '';
+    const paddedShardIds = Array.from({ length: total }, (_, index) =>
+      (index + 1).toString().padStart(5, '0')
+    );
+    return paddedShardIds.map(
+      (current) =>
+        `${baseURL}-${current}-of-${total.toString().padStart(5, '0')}.gguf${queryParams}`
+    );
+  }
+};
+
+/**
+ * Check if the given blobs are files or not, then sort them by shard number
+ */
+export const sortFileByShard = (blobs: Blob[]): void => {
+  const isFiles = blobs.every((b) => !!(b as File).name);
+  if (isFiles && blobs.length > 1) {
+    const files = blobs as File[];
+    files.sort((a, b) => {
+      const infoA = parseShardNumber(a.name);
+      const infoB = parseShardNumber(b.name);
+      return infoA.current - infoB.current;
+    });
+  }
+};
+
+export const isMmproj = async (blob: Blob): Promise<boolean> => {
+  const META_NAME = 'general.architecture';
+  const META_VAL = 'clip';
+  const tmp = blob.slice(0, 128 * 1024);
+  const header = await tmp.arrayBuffer();
+
+  const buf = new Uint8Array(header);
+  const nameBytes = new TextEncoder().encode(META_NAME);
+  const valBytes = new TextEncoder().encode(META_VAL);
+
+  // Find offset of META_NAME in buffer
+  let offset = -1;
+  outer: for (let i = 0; i <= buf.length - nameBytes.length; i++) {
+    for (let j = 0; j < nameBytes.length; j++) {
+      if (buf[i + j] !== nameBytes[j]) continue outer;
+    }
+    offset = i;
+    break;
+  }
+  if (offset === -1) return false;
+
+  // Read valLen as uint64 at offset+8*3 (little-endian, read low 32 bits)
+  if (offset + 8 * 4 + 4 > buf.length) return false;
+  const view = new DataView(header);
+  const valLen = view.getBigUint64(offset + 8 * 3, true);
+  if (valLen !== 4n) return false;
+
+  // Read 4 bytes at offset+8*4, compare with META_VAL bytes
+  for (let i = 0; i < valBytes.length; i++) {
+    if (buf[offset + 8 * 4 + i] !== valBytes[i]) return false;
+  }
+  return true;
+};
+
+export const delay = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+export const absoluteUrl = (relativePath: string) =>
+  new URL(relativePath, document.baseURI).href;
+
+export const padDigits = (number: number, digits: number) => {
+  return (
+    Array(Math.max(digits - String(number).length + 1, 0)).join('0') + number
+  );
+};
+
+export const sumArr = (arr: number[]) =>
+  arr.reduce((prev, curr) => prev + curr, 0);
+
+export const isString = (value: any): boolean => !!value?.startsWith;
+
+export const MMPROJ_FILE_NAME = 'mmproj.gguf';
+
+type ModelShard = { blob: Blob; name: string };
+export const prepareBlobs = async (
+  blobsInp: Blob[]
+): Promise<{
+  llm: ModelShard[];
+  mmproj: ModelShard | null;
+  all: ModelShard[];
+}> => {
+  const blobs: Blob[] = [];
+  let blobMmproj: Blob | null = null;
+
+  for (const blob of blobsInp) {
+    if (await isMmproj(blob)) {
+      blobMmproj = blob;
+    } else {
+      blobs.push(blob);
+    }
+  }
+
+  // prepare model-XXXXX-of-XXXXX.gguf blobs
+  sortFileByShard(blobs);
+  const result = blobs.map((blob, i) => ({
+    blob,
+    name: `model-${padDigits(i + 1, 5)}-of-${padDigits(blobs.length, 5)}.gguf`,
+  }));
+
+  // prepare mmproj.gguf blob
+  if (blobMmproj) {
+    result.push({
+      blob: blobMmproj,
+      name: MMPROJ_FILE_NAME,
+    });
+  }
+
+  return {
+    llm: result.filter((f) => f.name !== MMPROJ_FILE_NAME),
+    mmproj: blobMmproj ? { blob: blobMmproj, name: MMPROJ_FILE_NAME } : null,
+    all: result,
+  };
+};
+
+/**
+ * Browser feature detection
+ * Copied from https://unpkg.com/wasm-feature-detect?module (Apache License)
+ */
+
+/**
+ * @returns true if browser support multi-threads
+ */
+export const isSupportMultiThread = () =>
+  (async (e) => {
+    try {
+      return (
+        'undefined' != typeof MessageChannel &&
+          new MessageChannel().port1.postMessage(new SharedArrayBuffer(1)),
+        WebAssembly.validate(e)
+      );
+    } catch (e) {
+      return !1;
+    }
+  })(
+    new Uint8Array([
+      0, 97, 115, 109, 1, 0, 0, 0, 1, 4, 1, 96, 0, 0, 3, 2, 1, 0, 5, 4, 1, 3, 1,
+      1, 10, 11, 1, 9, 0, 65, 0, 254, 16, 2, 0, 26, 11,
+    ])
+  );
+
+/**
+ * @returns true if browser support wasm "native" exception handler
+ */
+const isSupportExceptions = async () =>
+  WebAssembly.validate(
+    new Uint8Array([
+      0, 97, 115, 109, 1, 0, 0, 0, 1, 4, 1, 96, 0, 0, 3, 2, 1, 0, 10, 8, 1, 6,
+      0, 6, 64, 25, 11, 11,
+    ])
+  );
+
+/**
+ * @returns true if browser support wasm SIMD
+ */
+const isSupportSIMD = async () =>
+  WebAssembly.validate(
+    new Uint8Array([
+      0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10,
+      1, 8, 0, 65, 0, 253, 15, 253, 98, 11,
+    ])
+  );
+
+/**
+ * @returns true if browser support JSPI
+ */
+export const isSupportJSPI = () => {
+  return !!(WebAssembly as any).Suspending;
+};
+
+/**
+ * @returns true if brower support WebGPU. Note: for browser without JSPI support, compat mode will be used.
+ */
+export const isSupportWebGPU = () => {
+  return !!(navigator as any).gpu;
+};
+
+/**
+ * @returns true if browser support WASM Memory64
+ */
+export const isSupportMem64 = (): boolean => {
+  try {
+    new WebAssembly.Memory({
+      address: 'i64',
+      initial: 1n, // 1 page (64 KiB)
+    } as any);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * Throws an error if the environment is not compatible
+ */
+export const checkEnvironmentCompatible = async (): Promise<void> => {
+  if (!(await isSupportExceptions())) {
+    throw new Error('WebAssembly runtime does not support exception handling');
+  }
+  if (!(await isSupportSIMD())) {
+    throw new Error('WebAssembly runtime does not support SIMD');
+  }
+};
+
+/**
+ * Check if browser is Safari
+ * Source: https://github.com/DamonOehlman/detect-browser/blob/master/src/index.ts
+ */
+export const isSafari = (): boolean => {
+  return (
+    isSafariMobile() ||
+    !!navigator.userAgent.match(/Version\/([0-9\._]+).*Safari/)
+  ); // safari
+};
+
+/**
+ * Check if browser is Firefox
+ */
+export const isFirefox = (): boolean => {
+  return !!navigator.userAgent.match(/Firefox\/([0-9\.]+)(?:\s|$)/);
+};
+
+/**
+ * Regular expression to validate GGUF file paths/URLs
+ * Matches paths ending with .gguf and optional query parameters
+ */
+export const GGUF_FILE_REGEX = /^.*\.gguf(?:\?.*)?$/;
+
+/**
+ * Validates if a given string is a valid GGUF file path/URL
+ * @param path The file path or URL to validate
+ * @returns true if the path is a valid GGUF file path/URL
+ */
+export const isValidGgufFile = (path: string): boolean => {
+  return GGUF_FILE_REGEX.test(path);
+};
+
+/**
+ * Check if browser is Safari iOS / iPad / iPhone
+ * Source: https://github.com/DamonOehlman/detect-browser/blob/master/src/index.ts
+ */
+export const isSafariMobile = (): boolean => {
+  return !!navigator.userAgent.match(/Version\/([0-9\._]+).*Mobile.*Safari.*/); // ios
+};
+
+/**
+ * Create a worker from a string
+ */
+export const createWorker = (workerCode: string | Blob): Worker => {
+  const workerURL = URL.createObjectURL(
+    isString(workerCode)
+      ? new Blob([workerCode], { type: 'text/javascript' })
+      : (workerCode as Blob)
+  );
+  return new Worker(workerURL, { type: 'module' });
+};
+
+/**
+ * Convert callback to async iterator
+ */
+export const cbToAsyncIter =
+  <A extends any[], T>(
+    fn: (
+      ...args: [
+        ...args: A,
+        callback: (val?: T, done?: boolean, err?: Error) => void,
+      ]
+    ) => void
+  ) =>
+  (...args: A): AsyncIterable<T> => {
+    let values: Promise<[T, boolean]>[] = [];
+    let resolve: (x: [T, boolean]) => void;
+    let reject: (e: Error) => void;
+    values.push(
+      new Promise((res, rej) => {
+        resolve = res;
+        reject = rej;
+      })
+    );
+    fn(...args, (val?: T, done?: boolean, err?: Error) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+      resolve([val!, done!]);
+      values.push(
+        new Promise((res, rej) => {
+          resolve = res;
+          reject = rej;
+        })
+      );
+    });
+    return (async function* () {
+      let val: T;
+      for (let i = 0, done = false; !done; i++) {
+        [val, done] = await values[i];
+        delete values[i];
+        if (val !== undefined) yield val;
+      }
+    })();
+  };
+
+/**
+ * Check if we can use async file read, where the wasm env can asynchronously read a Blob.
+ * Please refer to README-dev.md for more details.
+ */
+export const canUseAsyncFileRead = (compat: boolean) =>
+  isSupportJSPI() || compat;
+
+export const needCompat = () => !isSupportJSPI() || !isSupportMem64();