@simulatte/doppler 0.1.8 → 0.2.0

package/src/browser/browser-converter.js

@@ -1,977 +0,0 @@
-
-
-import { parseGGUFHeaderFromSource } from './gguf-parser-browser.js';
-import { isTensorSource, normalizeTensorSource } from './tensor-source-file.js';
-import { createRemoteTensorSource } from './tensor-source-download.js';
-import {
-  parseSafetensorsFile,
-  parseSafetensorsSharded,
-  parseConfigJson,
-  parseTokenizerConfigJson,
-  parseTokenizerJson,
-  parseIndexJson,
-  streamTensorData,
-  detectModelFormat,
-  getAuxiliaryFiles,
-} from './safetensors-parser-browser.js';
-import {
-  initStorage,
-  openModelStore,
-  saveManifest,
-  saveTokenizer,
-  saveTokenizerModel,
-  saveAuxFile,
-  deleteModel,
-  createConversionShardWriter,
-  computeHash,
-  createStreamingHasher,
-  getStorageBackendType,
-} from '../storage/shard-manager.js';
-import { classifyTensorRole } from '../formats/rdrr/index.js';
-import { registerModel } from '../storage/registry.js';
-import {
-  checkSpaceAvailable,
-  requestPersistence,
-  isOPFSAvailable,
-  isIndexedDBAvailable,
-  QuotaExceededError,
-} from '../storage/quota.js';
-
-// Import shared shard packing logic
-import {
-  ShardPacker,
-} from '../converter/shard-packer.js';
-import {
-  resolveTensorDtype,
-  resolveQ4KLayout,
-  getQ4KOutputSize,
-  createQ4KChunkStream,
-  createF16ChunkStream,
-  resolveEffectiveQuantizationInfo,
-  resolveManifestQuantization,
-} from './quantization.js';
-
-// Import shared types and functions from convert-core
-import {
-  ConvertStage,
-  formatBytes,
-  extractArchitecture,
-  createManifest,
-} from '../converter/core.js';
-import {
-  inferSourceWeightQuantization,
-  resolveConversionPlan,
-  resolveConvertedModelId,
-} from '../converter/conversion-plan.js';
-import { parseDiffusionModel as parseSharedDiffusionModel } from '../converter/parsers/diffusion.js';
-import { parseGGUFModel as parseSharedGGUFModel } from '../converter/parsers/gguf.js';
-
-import { createConverterConfig } from '../config/index.js';
-import { MB, GB } from '../config/schema/units.schema.js';
-import { log, trace } from '../debug/index.js';
-
-// Re-export types for consumers
-export {
-  ConvertStage,
-};
-
-export function isConversionSupported() {
-  return isOPFSAvailable() || isIndexedDBAvailable();
-}
-
-const LARGE_MODEL_THRESHOLD_BYTES = 8 * GB;
-const LARGE_MODEL_CHUNK_BYTES = 128 * MB;
-const LARGE_MODEL_SHARD_BYTES = 256 * MB;
-
-function tuneConverterConfig(config, totalInputBytes, modelType) {
-  if (!config) return;
-  const isLarge = Number.isFinite(totalInputBytes) && totalInputBytes >= LARGE_MODEL_THRESHOLD_BYTES;
-  const isDiffusion = modelType === 'diffusion';
-  if (!isLarge && !isDiffusion) return;
-
-  if (config.streaming?.chunkSizeBytes) {
-    config.streaming.chunkSizeBytes = Math.max(config.streaming.chunkSizeBytes, LARGE_MODEL_CHUNK_BYTES);
-  }
-  if (config.sharding?.shardSizeBytes) {
-    config.sharding.shardSizeBytes = Math.max(config.sharding.shardSizeBytes, LARGE_MODEL_SHARD_BYTES);
-  }
-}
-
-function createStageTimer(label) {
-  const start = performance.now();
-  return {
-    stop: (extra, data) => {
-      const elapsed = performance.now() - start;
-      const suffix = extra ? ` - ${extra}` : '';
-      log.info('Convert', `${label}: ${elapsed.toFixed(0)}ms${suffix}`);
-      trace.perf(`Convert ${label}: ${elapsed.toFixed(0)}ms`, data);
-      return elapsed;
-    },
-  };
-}
-
-function resolveRemoteOptions(options) {
-  const http = options?.converterConfig?.http || null;
-  return {
-    headers: options?.headers,
-    signal: options?.signal,
-    name: options?.name,
-    allowDownloadFallback: http?.allowDownloadFallback,
-    maxDownloadBytes: http?.maxDownloadBytes,
-  };
-}
-
-export async function createRemoteModelSources(urls, options = {}) {
-  if (!Array.isArray(urls) || urls.length === 0) {
-    throw new Error('Remote conversion requires at least one URL.');
-  }
-
-  const sources = [];
-  const remoteOptions = resolveRemoteOptions(options);
-  for (const url of urls) {
-    if (typeof url !== 'string' || url.length === 0) {
-      throw new Error('Remote conversion URLs must be non-empty strings.');
-    }
-    const result = await createRemoteTensorSource(url, remoteOptions);
-    sources.push(result.source);
-  }
-
-  return sources;
-}
-
-// ============================================================================
-// Main Convert Function
-// ============================================================================
-
-
-function getFilePath(file) {
-  if (!file) return '';
-  if (typeof file.relativePath === 'string' && file.relativePath.length > 0) {
-    return file.relativePath;
-  }
-  if (typeof file.webkitRelativePath === 'string' && file.webkitRelativePath.length > 0) {
-    return file.webkitRelativePath;
-  }
-  if (typeof file.name === 'string') return file.name;
-  return '';
-}
-
-function normalizePath(path) {
-  return String(path || '').replace(/\\/g, '/');
-}
-
-function attachRelativePath(file, relativePath) {
-  if (!file || !relativePath) return;
-  try {
-    Object.defineProperty(file, 'relativePath', {
-      value: relativePath,
-      configurable: true,
-    });
-  } catch {
-    // Ignore if File is non-extensible in this environment.
-  }
-}
-
-function getBaseName(path) {
-  const normalized = normalizePath(path);
-  if (!normalized) return '';
-  const parts = normalized.split('/');
-  return parts[parts.length - 1] || '';
-}
-
-function pathEndsWith(path, suffix) {
-  const normalized = normalizePath(path);
-  return normalized.endsWith(suffix);
-}
-
-function compareNullableStrings(left, right) {
-  const a = typeof left === 'string' ? left : '';
-  const b = typeof right === 'string' ? right : '';
-  return a.localeCompare(b);
-}
-
-function sortTensorsByDeterministicLocality(tensors) {
-  if (!Array.isArray(tensors) || tensors.length <= 1) {
-    return tensors;
-  }
-  tensors.sort((left, right) => {
-    const leftPath = left?.source?.name ?? left?.shardFile ?? left?.file?.name ?? '';
-    const rightPath = right?.source?.name ?? right?.shardFile ?? right?.file?.name ?? '';
-    const sourcePathCmp = compareNullableStrings(leftPath, rightPath);
-    if (sourcePathCmp !== 0) return sourcePathCmp;
-    const leftOffset = Number.isFinite(left?.offset) ? Number(left.offset) : 0;
-    const rightOffset = Number.isFinite(right?.offset) ? Number(right.offset) : 0;
-    if (leftOffset !== rightOffset) {
-      return leftOffset - rightOffset;
-    }
-    return compareNullableStrings(left?.name, right?.name);
-  });
-  return tensors;
-}
-
-function findFileBySuffix(files, suffix) {
-  return files.find((file) => pathEndsWith(getFilePath(file), suffix)) || null;
-}
-
-async function readTextFile(file, label = 'file') {
-  if (!file || typeof file.text !== 'function') {
-    throw new Error(`Missing ${label}`);
-  }
-  return file.text();
-}
-
-async function parseJsonFile(file, label = 'json') {
-  const text = await readTextFile(file, label);
-  try {
-    return JSON.parse(text);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    throw new Error(`[${label}] Failed to parse JSON: ${message}`);
-  }
-}
-
-function isDiffusionInput(files) {
-  return files.some((file) => getBaseName(getFilePath(file)) === 'model_index.json');
-}
-
-function pickFirstBySuffix(files, suffixes) {
-  for (const suffix of suffixes) {
-    const match = findFileBySuffix(files, suffix);
-    if (match) return match;
-  }
-  return null;
-}
-
-async function parseDiffusionModel(files, onProgress, signal) {
-  const modelIndexFile = findFileBySuffix(files, 'model_index.json');
-  if (!modelIndexFile) return null;
-  return parseSharedDiffusionModel({
-    onProgress,
-    signal,
-    findExistingSuffix(suffixes) {
-      const file = pickFirstBySuffix(files, suffixes);
-      return file ? normalizePath(getFilePath(file)) : null;
-    },
-    async readJson(suffix, label = 'json') {
-      const file = findFileBySuffix(files, suffix);
-      if (!file) {
-        throw new Error(`Missing ${label} (${suffix})`);
-      }
-      return parseJsonFile(file, label);
-    },
-    async readText(suffix, label = 'text') {
-      const file = findFileBySuffix(files, suffix);
-      if (!file) {
-        throw new Error(`Missing ${label} (${suffix})`);
-      }
-      return readTextFile(file, label);
-    },
-    async readBinary(suffix, label = 'binary') {
-      const file = findFileBySuffix(files, suffix);
-      if (!file || typeof file.arrayBuffer !== 'function') {
-        throw new Error(`Missing ${label} (${suffix})`);
-      }
-      return file.arrayBuffer();
-    },
-    async parseSingleSafetensors(suffix) {
-      const file = findFileBySuffix(files, suffix);
-      if (!file) {
-        throw new Error(`Missing safetensors file (${suffix})`);
-      }
-      return parseSafetensorsFile(file);
-    },
-    async parseShardedSafetensors(indexSuffix, indexJson, componentId) {
-      const weightMap = indexJson?.weight_map || {};
-      const shardNames = Array.from(new Set(Object.values(weightMap)));
-      if (shardNames.length === 0) {
-        throw new Error(`No shards listed in ${componentId} index file`);
-      }
-      const baseDir = indexSuffix.includes('/') ? indexSuffix.split('/').slice(0, -1).join('/') : '';
-      const shardFiles = shardNames.map((name) => {
-        const suffix = baseDir ? `${baseDir}/${name}` : name;
-        return findFileBySuffix(files, suffix);
-      }).filter(Boolean);
-      if (shardFiles.length !== shardNames.length) {
-        throw new Error(`Missing shard files for ${componentId} (${shardFiles.length}/${shardNames.length} found)`);
-      }
-      return parseSafetensorsSharded(shardFiles, indexJson);
-    },
-  });
-}
-
-export async function convertModel(files, options = {}) {
-  const { modelId: userModelId, onProgress, signal, converterConfig } = options;
-  const resolvedConverterConfig = converterConfig || createConverterConfig();
-  const progressState = {
-    lastStage: null,
-    lastMessage: null,
-    lastPercentBucket: null,
-  };
-  const reportProgress = (update) => {
-    if (update) {
-      const stage = update.stage ?? null;
-      const message = update.message ? String(update.message) : '';
-      const percent = Number.isFinite(update.percent) ? update.percent : null;
-      if (stage && stage !== progressState.lastStage) {
-        progressState.lastStage = stage;
-        progressState.lastMessage = null;
-        progressState.lastPercentBucket = null;
-        log.info('Convert', message ? `${stage}: ${message}` : `${stage}`);
-      } else if (stage === ConvertStage.WRITING && percent !== null) {
-        const bucket = Math.floor(percent / 5) * 5;
-        if (bucket !== progressState.lastPercentBucket) {
-          progressState.lastPercentBucket = bucket;
-          const counts = Number.isFinite(update.current) && Number.isFinite(update.total)
-            ? ` (${update.current}/${update.total})`
-            : '';
-          log.info('Convert', `Writing ${bucket}%${counts}`);
-        }
-      } else if (message && message !== progressState.lastMessage) {
-        progressState.lastMessage = message;
-        log.info('Convert', message);
-      }
-    }
-    onProgress?.(update);
-  };
-
-  let modelId = null;
-  const shardInfos = [];
-  const cleanupTasks = [];
-  const inputFiles = Array.isArray(files) ? files : [];
-  const totalInputBytes = inputFiles.reduce((sum, file) => sum + (file?.size || 0), 0);
-  const hasOnlyFiles = inputFiles.every((file) => !isTensorSource(file));
-  const diffusionCandidate = hasOnlyFiles && isDiffusionInput(inputFiles);
-  let diffusionInfo = null;
-  let diffusionAuxFiles = null;
-  let diffusionArchitecture = null;
-  let diffusionEosTokenId = undefined;
-  const conversionStart = performance.now();
-  log.info(
-    'Convert',
-    `Start: ${inputFiles.length} files, ${formatBytes(totalInputBytes)}`
-  );
-
-  try {
-    if (!isOPFSAvailable() && !isIndexedDBAvailable()) {
-      throw new Error('No supported storage backend available for browser conversion. Supported: opfs, indexeddb.');
-    }
-
-    // Initialize storage
-    await initStorage();
-    const persistence = await requestPersistence();
-    const backendType = getStorageBackendType();
-    reportProgress({
-      stage: ConvertStage.DETECTING,
-      message: `Storage backend: ${backendType ?? 'unknown'}`,
-      backend: backendType,
-      persistence,
-    });
-
-    // Detect format
-    reportProgress({
-      stage: ConvertStage.DETECTING,
-      message: 'Detecting model format...',
-    });
-
-    if (signal?.aborted) throw new DOMException('Cancelled', 'AbortError');
-
-    const detectTimer = createStageTimer('Detect format');
-    if (diffusionCandidate) {
-      diffusionInfo = await parseDiffusionModel(inputFiles, reportProgress, signal);
-      diffusionAuxFiles = diffusionInfo?.auxFiles ?? null;
-      diffusionArchitecture = diffusionInfo?.architecture ?? null;
-      diffusionEosTokenId = null;
-    }
-
-    const format = diffusionInfo ? { type: 'diffusion' } : detectModelFormat(files);
-    detectTimer.stop(`type=${format.type}`);
-    const auxiliary = diffusionInfo ? null : getAuxiliaryFiles(files);
-    for (const file of files) {
-      if (isTensorSource(file) && typeof file.cleanup === 'function') {
-        cleanupTasks.push(file.cleanup);
-      }
-    }
-    if (!diffusionInfo) {
-      const hasTokenizerJson = !!auxiliary.tokenizer;
-      const hasTokenizerModel = !!auxiliary.tokenizerModel;
-      if (!hasTokenizerJson && !hasTokenizerModel) {
-        throw new Error('Missing tokenizer.json or tokenizer.model for browser conversion.');
-      }
-    }
-
-    reportProgress({
-      stage: ConvertStage.DETECTING,
-      message: `Format: ${format.type}`,
-      format: format.type,
-    });
-
-    // Parse based on format
-    let modelInfo;
-    let config = null;
-    let generationConfig = null;
-    let tokenizerJson = null;
-    let tokenizerConfig = null;
-    let tokenizerModel = null;
-
-    const parseTimer = createStageTimer('Parse tensors');
-    if (format.type === 'diffusion') {
-      modelInfo = {
-        tensors: diffusionInfo.tensors,
-        config: diffusionInfo.config,
-        architecture: diffusionArchitecture,
-        format: 'safetensors',
-      };
-      config = diffusionInfo.config;
-    } else if (format.type === 'gguf') {
-      modelInfo = await parseGGUFModel(format.ggufFile, reportProgress, signal);
-    } else if (format.type === 'single') {
-      const parsed = await parseSafetensorsFile(format.safetensorsFile);
-      modelInfo = { tensors: parsed.tensors, config: parsed.config };
-      if (auxiliary.config) {
-        config = await parseConfigJson(auxiliary.config);
-        modelInfo.config = config;
-      }
-    } else if (format.type === 'sharded' || format.type === 'sharded-no-index') {
-      let indexJson = null;
-      if (format.indexFile) {
-        indexJson = await parseIndexJson(format.indexFile);
-      }
-      const parsed = await parseSafetensorsSharded(format.safetensorsFiles, indexJson);
-      modelInfo = { tensors: parsed.tensors, config: parsed.config };
-      if (auxiliary.config) {
-        config = await parseConfigJson(auxiliary.config);
-        modelInfo.config = config;
-      }
-    } else {
-      throw new Error(`Unsupported format: ${format.type}`);
-    }
-
-    // Parse tokenizer if available (text models only)
-    if (!diffusionInfo) {
-      if (auxiliary.tokenizer) {
-        tokenizerJson = await parseTokenizerJson(auxiliary.tokenizer);
-        modelInfo.tokenizerJson = tokenizerJson;
-      }
-      if (auxiliary.tokenizerConfig) {
-        tokenizerConfig = await parseTokenizerConfigJson(auxiliary.tokenizerConfig);
-        modelInfo.tokenizerConfig = tokenizerConfig;
-      }
-      if (auxiliary.generationConfig) {
-        generationConfig = await parseConfigJson(auxiliary.generationConfig);
-        modelInfo.generationConfig = generationConfig;
-      }
-      if (auxiliary.tokenizerModel) {
-        const source = normalizeTensorSource(auxiliary.tokenizerModel);
-        tokenizerModel = await source.readRange(0, source.size);
-        modelInfo.tokenizerModel = tokenizerModel;
-      }
-    }
-    parseTimer.stop(`${modelInfo.tensors.length} tensors`);
-
-    if (signal?.aborted) throw new DOMException('Cancelled', 'AbortError');
-
-    const rawConfig = (config || modelInfo.config || {});
-    const tensors = sortTensorsByDeterministicLocality(modelInfo.tensors || []);
-    modelInfo.tensors = tensors;
-    const totalTensorBytes = tensors.reduce((sum, tensor) => sum + (tensor.size || 0), 0);
-    let architectureConfig = null;
-    if (!diffusionInfo) {
-      const hfConfig = (config || (modelInfo.format === 'gguf' ? null : modelInfo.config));
-      const ggufConfig = modelInfo.format === 'gguf' ? modelInfo.config : undefined;
-      architectureConfig = extractArchitecture(hfConfig || {}, ggufConfig);
-    }
-    const weightOverride = resolvedConverterConfig.quantization?.weights ?? null;
-    const sourceQuantization = modelInfo.quantization || weightOverride || inferSourceWeightQuantization(tensors);
-    const plan = resolveConversionPlan({
-      rawConfig,
-      tensors,
-      converterConfig: resolvedConverterConfig,
-      sourceQuantization,
-      modelKind: diffusionInfo ? 'diffusion' : 'transformer',
-      architectureHint: modelInfo.architecture,
-      architectureConfig,
-    });
-    const modelType = plan.modelType;
-    const quantizationInfo = plan.quantizationInfo;
-    const manifestQuantization = plan.manifestQuantization;
-    const manifestInference = plan.manifestInference;
-
-    const detectedModelId = extractModelId(files, config);
-    modelId = resolveConvertedModelId({
-      explicitModelId: userModelId,
-      converterConfig: resolvedConverterConfig,
-      detectedModelId,
-      quantizationInfo,
-    });
-    if (!modelId) {
-      throw new Error(
-        'Missing modelId. Provide modelId explicitly or set converterConfig.output.modelBaseId.'
-      );
-    }
-
-    tuneConverterConfig(resolvedConverterConfig, totalTensorBytes, modelType);
-
-    const chunkSizeBytes = resolvedConverterConfig.streaming.chunkSizeBytes;
-    if (!chunkSizeBytes || chunkSizeBytes <= 0) {
-      throw new Error('Invalid converter streaming chunk size');
-    }
-
-    const collectChunks = async (chunks) => {
-      const buffers = [];
-      let total = 0;
-      for await (const chunk of chunks) {
-        const bytes = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk);
-        if (bytes.length === 0) continue;
-        buffers.push(bytes);
-        total += bytes.length;
-      }
-      const output = new Uint8Array(total);
-      let offset = 0;
-      for (const buffer of buffers) {
-        output.set(buffer, offset);
-        offset += buffer.length;
-      }
-      return output;
-    };
-
-    const sourceChunksFor = (tensor) => {
-      if (modelInfo.format === 'gguf' && tensor.source) {
-        return streamSourceRange(tensor.source, tensor.offset, tensor.size, chunkSizeBytes);
-      }
-      return streamTensorData(tensor, chunkSizeBytes);
-    };
-
-    const tensorPlans = modelInfo.tensors.map((tensor) => {
-      const targetDtype = resolveTensorDtype(tensor.name, tensor.shape, tensor.dtype, quantizationInfo);
-      const q4kLayout = targetDtype === 'Q4_K_M'
-        ? resolveQ4KLayout(tensor.name, tensor.shape, quantizationInfo)
-        : null;
-      const numElements = tensor.shape.reduce((a, b) => a * b, 1);
-      const targetSize = targetDtype === 'Q4_K_M'
-        ? getQ4KOutputSize(tensor.shape, q4kLayout)
-        : targetDtype === 'F16'
-          ? numElements * 2
-          : targetDtype === 'F32'
-            ? numElements * 4
-            : tensor.size;
-
-      const getQ4KData = async () => {
-        const chunks = createQ4KChunkStream(
-          sourceChunksFor(tensor),
-          tensor.dtype,
-          tensor.shape,
-          q4kLayout,
-          chunkSizeBytes
-        );
-        return collectChunks(chunks);
-      };
-
-      const getF16Data = async () => {
-        const chunks = createF16ChunkStream(sourceChunksFor(tensor), tensor.dtype);
-        return collectChunks(chunks);
-      };
-
-      let getData;
-      let getChunks;
-
-      if (targetDtype === 'Q4_K_M') {
-        getData = getQ4KData;
-        getChunks = () => createQ4KChunkStream(
-          sourceChunksFor(tensor),
-          tensor.dtype,
-          tensor.shape,
-          q4kLayout,
-          chunkSizeBytes
-        );
-      } else if (targetDtype === 'F16' && tensor.dtype !== 'F16') {
-        getData = getF16Data;
-        getChunks = () => createF16ChunkStream(sourceChunksFor(tensor), tensor.dtype);
-      } else {
-        getData = async () => {
-          const data = await readTensorData(tensor);
-          return new Uint8Array(data);
-        };
-        getChunks = () => sourceChunksFor(tensor);
-      }
-
-      return {
-        name: tensor.name,
-        shape: tensor.shape,
-        dtype: targetDtype,
-        role: classifyTensorRole(tensor.name),
-        layout: q4kLayout,
-        size: targetSize,
-        offset: tensor.offset,
-        getData,
-        getChunks,
-      };
-    });
-
-    const totalSizeBytes = tensorPlans.reduce((sum, tensor) => sum + tensor.size, 0);
-    const spaceCheck = await checkSpaceAvailable(totalSizeBytes);
-    if (!spaceCheck.hasSpace) {
-      throw new QuotaExceededError(totalSizeBytes, spaceCheck.info.available);
-    }
-
-    reportProgress({
-      stage: ConvertStage.PARSING,
-      message: `Model: ${modelId}`,
-      modelId,
-      tensorCount: modelInfo.tensors.length,
-      totalSize: formatBytes(totalSizeBytes),
-    });
-
-    await openModelStore(modelId);
-
-    if (diffusionAuxFiles && diffusionAuxFiles.length > 0) {
-      for (const asset of diffusionAuxFiles) {
-        await saveAuxFile(asset.name, asset.data);
-      }
-    }
-
-    const hashAlgorithm = resolvedConverterConfig.manifest.hashAlgorithm;
-
-    // Create shard I/O adapter
-    const shardIO = {
-      writeShard: async (index, data) => {
-        const writer = await createConversionShardWriter(index);
-        try {
-          await writer.write(data);
-          await writer.close();
-        } catch (error) {
-          await writer.abort();
-          throw error;
-        }
-        return computeHash(data, hashAlgorithm);
-      },
-      computeHash: (data) => computeHash(data, hashAlgorithm),
-      createShardWriter: (index) => createConversionShardWriter(index),
-      createHasher: () => createStreamingHasher(hashAlgorithm),
-    };
-
-    // Create shard packer
-    const packer = new ShardPacker(shardIO, {
-      modelType,
-      shardSize: resolvedConverterConfig.sharding.shardSizeBytes,
-      hashAlgorithm,
-    });
-
-    // Prepare tensors for packing
-    const packerTensors = tensorPlans;
-
-    // Pack tensors into shards
-    reportProgress({
-      stage: ConvertStage.WRITING,
-      message: 'Packing tensors...',
-    });
-
-    const packTimer = createStageTimer('Pack shards');
-    const packResult = await packer.pack(packerTensors, {
-      onProgress: (current, total, tensorName) => {
-        reportProgress({
-          stage: ConvertStage.WRITING,
-          message: `Processing ${tensorName}`,
-          current,
-          total,
-          percent: Math.round((current / total) * 100),
-        });
-      },
-      signal,
-    });
-    packTimer.stop(
-      `${packResult.shards.length} shards, ${formatBytes(packResult.totalSize)}`
-    );
-
-    // Convert pack result to expected format
-    const result = {
-      totalSize: packResult.totalSize,
-      tensorLocations: packResult.tensors,
-    };
-
-    // Copy shard infos
-    for (const shard of packResult.shards) {
-      shardInfos.push(shard);
-    }
-
-    if (signal?.aborted) throw new DOMException('Cancelled', 'AbortError');
-
-    // Create manifest using shared function
-    reportProgress({
-      stage: ConvertStage.MANIFEST,
-      message: 'Creating manifest...',
-    });
-
-    // Convert to ParsedModel format for createManifest
-    const parsedModel = {
-      tensors: tensorPlans.map(t => ({
-        name: t.name,
-        shape: t.shape,
-        dtype: t.dtype,
-        size: t.size,
-        offset: t.offset,
-      })),
-      config: (config || modelInfo.config || {}),
-      architecture: modelInfo.architecture,
-      quantization: manifestQuantization,
-      tokenizerJson,
-      tokenizerConfig,
-      tokenizerModel: tokenizerModel ? 'tokenizer.model' : null,
-    };
-
-    const manifestTimer = createStageTimer('Manifest');
-    const effectiveQuantizationInfo = resolveEffectiveQuantizationInfo(
-      quantizationInfo,
-      tensorPlans
-    );
-    const effectiveManifestQuantization = resolveManifestQuantization(
-      effectiveQuantizationInfo.weights,
-      manifestQuantization
-    );
-
-    const manifest = createManifest(
-      modelId,
-      parsedModel,
-      shardInfos,
-      result.tensorLocations,
-      {
-        source: 'browser-converter',
-        inference: manifestInference,
-        modelType,
-        quantization: effectiveManifestQuantization,
-        quantizationInfo: effectiveQuantizationInfo,
-        hashAlgorithm,
-        architecture: diffusionArchitecture ?? undefined,
-        eosTokenId: diffusionEosTokenId,
-        convertedAt: resolvedConverterConfig?.manifest?.conversion?.convertedAt ?? null,
-        conversionInfo: resolvedConverterConfig?.manifest?.conversion ?? null,
-      }
-    );
-
-    manifest.groups = packResult.groups;
-    manifest.tensorCount = packResult.tensorCount;
-    if (manifest.tokenizer) {
-      if (manifest.tokenizer.type === 'bundled' || manifest.tokenizer.type === 'huggingface') {
-        manifest.tokenizer.file = manifest.tokenizer.file ?? 'tokenizer.json';
-      }
-      if (manifest.tokenizer.type === 'sentencepiece') {
-        manifest.tokenizer.sentencepieceModel = manifest.tokenizer.sentencepieceModel ?? 'tokenizer.model';
-      }
-    }
-
-    if (tokenizerJson) {
-      await saveTokenizer(JSON.stringify(tokenizerJson));
-    }
-    if (tokenizerModel) {
-      await saveTokenizerModel(tokenizerModel);
-    }
-
-    // Save manifest
-    await saveManifest(JSON.stringify(manifest, null, 2));
-
-    try {
-      await registerModel({
-        modelId,
-        totalSize: manifest.totalSize ?? result.totalSize,
-        quantization: manifest.quantization,
-        hashAlgorithm: manifest.hashAlgorithm,
-        backend: getStorageBackendType(),
-      });
-    } catch {
-      // Registry is optional; ignore failures
-    }
-    manifestTimer.stop();
-
-    reportProgress({
-      stage: ConvertStage.COMPLETE,
-      message: 'Conversion complete!',
-      modelId,
-      shardCount: shardInfos.length,
-      totalSize: formatBytes(result.totalSize),
-    });
-
-    if (cleanupTasks.length > 0) {
-      await Promise.allSettled(cleanupTasks.map((task) => task()));
-    }
-
-    const totalMs = performance.now() - conversionStart;
-    log.info(
-      'Convert',
-      `Complete: ${formatBytes(result.totalSize)} in ${totalMs.toFixed(0)}ms`
-    );
-    trace.perf('Convert total', {
-      ms: totalMs,
-      tensors: modelInfo.tensors.length,
-      totalSize: result.totalSize,
-      shardCount: shardInfos.length,
-    });
-
-    return modelId;
-  } catch (error) {
-    const totalMs = performance.now() - conversionStart;
-    log.error('Convert', `Failed after ${totalMs.toFixed(0)}ms: ${error.message}`);
-    // Cleanup on error
-    if (modelId) {
-      try {
-        await deleteModel(modelId);
-      } catch {
-        // Ignore cleanup errors
-      }
-    }
-
-    reportProgress({
-      stage: ConvertStage.ERROR,
-      message: error.message,
-      error: error,
-    });
-
-    throw error;
-  }
-}
-
-
-async function parseGGUFModel(
-  file,
-  onProgress,
-  signal
-) {
-  return parseSharedGGUFModel({
-    file,
-    parseGGUFHeaderFromSource,
-    normalizeTensorSource,
-    onProgress,
-    signal,
-  });
-}
-
-
-async function* streamSourceRange(source, offset, size, chunkSize) {
-  let cursor = offset;
-  const end = offset + size;
-
-  while (cursor < end) {
-    const next = Math.min(cursor + chunkSize, end);
-    const buffer = await source.readRange(cursor, next - cursor);
-    yield new Uint8Array(buffer);
-    cursor = next;
-  }
-}
-
-
-function extractModelId(files, config) {
-  // Try config first
-  if (config?._name_or_path) {
-    const parts = config._name_or_path.split('/');
-    return parts[parts.length - 1];
-  }
-
-  const safetensorsFiles = files.filter((f) => getBaseName(getFilePath(f)).toLowerCase().endsWith('.safetensors'));
-  const rootSafetensors = safetensorsFiles.find((f) => !normalizePath(getFilePath(f)).includes('/'));
-  const stFile = rootSafetensors || safetensorsFiles.find((f) => {
-    const base = getBaseName(getFilePath(f)).toLowerCase();
-    return !base.startsWith('model-') && !base.includes('-of-');
-  }) || safetensorsFiles[0];
-  if (stFile) {
-    const base = getBaseName(getFilePath(stFile));
-    return base.replace(/\.safetensors$/, '').replace(/model[-_.]?/, '');
-  }
-
-  // Try GGUF file name
-  const ggufFile = files.find((f) => f.name.endsWith('.gguf'));
-  if (ggufFile) {
-    return ggufFile.name.replace(/\.gguf$/, '');
-  }
-
-  return null;
-}
-
-// ============================================================================
-// File Picker Utilities
-// ============================================================================
-
-
-export async function pickModelFiles() {
-  const browserGlobal = typeof globalThis !== 'undefined' ? globalThis : null;
-  if (!browserGlobal) {
-    throw new Error('File picker APIs are unavailable outside browser contexts');
-  }
-
-  // Try directory picker first (for HuggingFace models)
-  if ('showDirectoryPicker' in browserGlobal) {
-    try {
-      const dirHandle = await browserGlobal.showDirectoryPicker({
-        mode: 'read',
-      });
-      return await collectFilesFromDirectory(dirHandle);
-    } catch (e) {
-      if (e.name === 'AbortError') throw e;
-      // Fall back to file picker
-    }
-  }
-
-  // Fall back to file picker
-  if ('showOpenFilePicker' in browserGlobal) {
-    const handles = await browserGlobal.showOpenFilePicker({
-      multiple: true,
-      types: [
-        {
-          description: 'Model files',
-          accept: {
-            'application/octet-stream': ['.gguf', '.safetensors', '.bin'],
-            'application/json': ['.json'],
-          },
-        },
-      ],
-    });
-    return Promise.all(handles.map((h) => h.getFile()));
-  }
-
-  // Ultimate fallback: input element
-  return new Promise((resolve) => {
-    const input = document.createElement('input');
-    input.type = 'file';
-    input.multiple = true;
-    input.accept = '.gguf,.safetensors,.json,.bin';
-    input.onchange = () => {
-      resolve(Array.from(input.files || []));
-    };
-    input.click();
-  });
-}
-
-
-async function collectFilesFromDirectory(
-  dirHandle,
-  files = [],
-  basePath = '',
-  depth = 0
-) {
-  if (depth > 4) {
-    throw new Error(
-      `Model directory exceeds supported depth (4) near "${basePath || dirHandle?.name || '.'}". ` +
-      'Choose a shallower directory root or flatten the model files.'
-    );
-  }
-  const entries = dirHandle.values();
-  for await (const entry of entries) {
-    if (entry.kind === 'file') {
-      const file = await entry.getFile();
-      // Only include relevant files
-      if (
-        file.name.endsWith('.safetensors') ||
-        file.name.endsWith('.gguf') ||
-        file.name.endsWith('.json') ||
-        file.name === 'tokenizer.model'
-      ) {
-        const relativePath = basePath ? `${basePath}/${entry.name}` : entry.name;
-        attachRelativePath(file, relativePath);
-        files.push(file);
-      }
-    } else if (entry.kind === 'directory') {
-      const nextBasePath = basePath ? `${basePath}/${entry.name}` : entry.name;
-      await collectFilesFromDirectory(entry, files, nextBasePath, depth + 1);
-    }
-  }
-  return files;
-}
-
-export default convertModel;