@simulatte/doppler 0.1.6 → 0.1.8

package/src/storage/downloader.js

@@ -2,15 +2,20 @@
 
 import {
   parseManifest,
+  getExpectedShardHash,
   getManifestUrl,
 } from '../formats/rdrr/index.js';
 
 import {
   openModelStore,
+  createFileWriter,
+  createStreamingHasher,
   writeShard,
   shardExists,
   loadShard,
+  loadFileFromStore,
   deleteShard,
+  deleteFileFromStore,
   saveManifest,
   saveTokenizer,
   saveTokenizerModel,
@@ -40,6 +45,7 @@ import {
 } from './download-types.js';
 
 import { downloadShard as downloadShardFromDistribution } from '../distribution/shard-delivery.js';
+import { resolveSourceArtifact, normalizeSourceArtifactPath } from './source-artifact-store.js';
 
 // ============================================================================
 // Module State
@@ -51,6 +57,7 @@ let db = null;
 const activeDownloads = new Map();
 
 function buildManifestVersionSet(manifest) {
+  const sourceArtifact = resolveSourceArtifact(manifest);
   if (!manifest || typeof manifest !== 'object') return 'manifest:invalid';
   const shards = Array.isArray(manifest.shards)
     ? manifest.shards.map((shard, index) => ({
@@ -67,6 +74,7 @@ function buildManifestVersionSet(manifest) {
     tensorCount: manifest.tensorCount ?? null,
     totalSize: manifest.totalSize ?? null,
     shards,
+    sourceRuntime: sourceArtifact?.sourceRuntime ?? null,
   };
   return JSON.stringify(payload);
 }
@@ -93,6 +101,31 @@ function normalizeSourceStats(value) {
   };
 }
 
+function isTokenizerJsonRequired(tokenizer) {
+  return Boolean(
+    tokenizer
+    && (tokenizer.type === 'bundled' || tokenizer.type === 'huggingface')
+    && typeof tokenizer.file === 'string'
+    && tokenizer.file.length > 0
+  );
+}
+
+function getTokenizerModelPath(tokenizer) {
+  if (!tokenizer || typeof tokenizer !== 'object') {
+    return null;
+  }
+  const explicit = typeof tokenizer.sentencepieceModel === 'string'
+    ? tokenizer.sentencepieceModel
+    : null;
+  if (explicit && explicit.length > 0) {
+    return explicit;
+  }
+  if (tokenizer.type === 'sentencepiece') {
+    return 'tokenizer.model';
+  }
+  return null;
+}
+
 // ============================================================================
 // IndexedDB Operations
 // ============================================================================
@@ -230,6 +263,12 @@ function isDatabaseClosingError(error) {
     ||  (error)?.name === 'InvalidStateError';
 }
 
+function createAbortError(message = 'Download aborted') {
+  const error = new Error(message);
+  error.name = 'AbortError';
+  return error;
+}
+
 // ============================================================================
 // Fetch Operations
 // ============================================================================
@@ -276,6 +315,106 @@ async function fetchWithRetry(url, options = {}) {
   throw  (lastError);
 }
 
+function joinArtifactUrl(baseUrl, relativePath) {
+  const root = String(baseUrl || '').trim();
+  const rel = normalizeSourceArtifactPath(relativePath);
+  if (!root || !rel) {
+    throw new Error('joinArtifactUrl requires baseUrl and relativePath.');
+  }
+  return new URL(rel, root.endsWith('/') ? root : `${root}/`).href;
+}
+
+async function fileExistsInStore(path) {
+  try {
+    await loadFileFromStore(path);
+    return true;
+  } catch (error) {
+    const message = String(error?.message || '');
+    return error?.name === 'NotFoundError' || message.toLowerCase().includes('not found')
+      ? false
+      : Promise.reject(error);
+  }
+}
+
+async function computeAssetHash(payload, algorithm = 'sha256') {
+  const bytes = payload instanceof Uint8Array ? payload : new Uint8Array(payload);
+  const hasher = await createStreamingHasher(String(algorithm || 'sha256').trim().toLowerCase());
+  hasher.update(bytes);
+  const digest = await hasher.finalize();
+  return Array.from(digest)
+    .map((byte) => byte.toString(16).padStart(2, '0'))
+    .join('');
+}
+
+async function downloadSourceAsset(url, asset, options = {}) {
+  const response = await fetchWithRetry(url, { signal: options.signal });
+  const expectedSize = Number.isFinite(asset?.size) ? Math.max(0, Math.floor(Number(asset.size))) : null;
+  const expectedHash = typeof asset?.hash === 'string' && asset.hash.trim() ? asset.hash.trim().toLowerCase() : null;
+  const hashAlgorithm = typeof asset?.hashAlgorithm === 'string' && asset.hashAlgorithm.trim()
+    ? asset.hashAlgorithm.trim().toLowerCase()
+    : 'sha256';
+  const writer = await createFileWriter(asset.path);
+  const hasher = expectedHash ? await createStreamingHasher(hashAlgorithm) : null;
+  let receivedBytes = 0;
+  try {
+    if (response.body && typeof response.body.getReader === 'function') {
+      const reader = response.body.getReader();
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) {
+          break;
+        }
+        if (!(value instanceof Uint8Array) || value.byteLength <= 0) {
+          continue;
+        }
+        await writer.write(value);
+        hasher?.update(value);
+        receivedBytes += value.byteLength;
+        options.onProgress?.(receivedBytes);
+      }
+    } else {
+      const bytes = new Uint8Array(await response.arrayBuffer());
+      await writer.write(bytes);
+      hasher?.update(bytes);
+      receivedBytes += bytes.byteLength;
+      options.onProgress?.(receivedBytes);
+    }
+    await writer.close();
+
+    if (expectedSize != null && receivedBytes !== expectedSize) {
+      throw new Error(
+        `Asset size mismatch for ${asset.path}: expected ${expectedSize}, got ${receivedBytes}`
+      );
+    }
+
+    if (hasher && expectedHash) {
+      const computedHashBytes = await hasher.finalize();
+      const computedHash = Array.from(computedHashBytes)
+        .map((byte) => byte.toString(16).padStart(2, '0'))
+        .join('');
+      if (computedHash !== expectedHash) {
+        throw new Error(
+          `Asset hash mismatch for ${asset.path}: expected ${expectedHash}, got ${computedHash}`
+        );
+      }
+    }
+
+    return {
+      source: 'http',
+      path: asset.path,
+      bytes: receivedBytes,
+    };
+  } catch (error) {
+    try {
+      await writer.abort();
+    } catch {}
+    try {
+      await deleteFileFromStore(asset.path);
+    } catch {}
+    throw error;
+  }
+}
+
 
 async function downloadShard(
   baseUrl,
@@ -327,9 +466,14 @@ export async function downloadModel(
   const {
     concurrency = getDefaultConcurrency(),
     requestPersist = true,
-    modelId: overrideModelId = undefined
+    modelId: overrideModelId = undefined,
+    signal: externalSignal = null,
   } = options;
 
+  if (externalSignal?.aborted) {
+    throw createAbortError();
+  }
+
   // Request persistent storage if needed
   if (requestPersist) {
     await requestPersistence();
@@ -340,14 +484,19 @@ export async function downloadModel(
   const manifestResponse = await fetchWithRetry(manifestUrl);
   const manifestJson = await manifestResponse.text();
   const manifest = parseManifest(manifestJson);
+  const directSourceArtifact = resolveSourceArtifact(manifest);
+  const trackedShards = directSourceArtifact ? directSourceArtifact.sourceFiles : manifest.shards;
+  const trackedTotalBytes = directSourceArtifact
+    ? directSourceArtifact.totalBytes
+    : manifest.totalSize;
 
   // Use override modelId for storage, or fall back to manifest's modelId
   const storageModelId = overrideModelId || manifest.modelId;
 
   // Check available space
-  const spaceCheck = await checkSpaceAvailable(manifest.totalSize);
+  const spaceCheck = await checkSpaceAvailable(trackedTotalBytes);
   if (!spaceCheck.hasSpace) {
-    throw new QuotaExceededError(manifest.totalSize, spaceCheck.info.available);
+    throw new QuotaExceededError(trackedTotalBytes, spaceCheck.info.available);
   }
 
   // Open model directory
@@ -377,7 +526,14 @@ export async function downloadModel(
     if (savedVersionSet !== manifestVersionSet) {
       log.warn('Downloader', `Manifest version-set changed for ${storageModelId}, resetting cached shards`);
       for (const idx of state.completedShards) {
-        await deleteShard(idx);
+        if (directSourceArtifact) {
+          const sourceEntry = directSourceArtifact.sourceFiles[idx];
+          if (sourceEntry) {
+            await deleteFileFromStore(sourceEntry.path);
+          }
+        } else {
+          await deleteShard(idx);
+        }
       }
       state.completedShards.clear();
     }
@@ -389,6 +545,13 @@ export async function downloadModel(
     state.lastSourcePath = typeof state.lastSourcePath === 'string' ? state.lastSourcePath : null;
     // Check which shards actually exist (in case OPFS was cleared)
     for (const idx of state.completedShards) {
+      if (directSourceArtifact) {
+        const sourceEntry = directSourceArtifact.sourceFiles[idx];
+        if (!sourceEntry || !(await fileExistsInStore(sourceEntry.path))) {
+          state.completedShards.delete(idx);
+        }
+        continue;
+      }
       if (!(await shardExists(idx))) {
         state.completedShards.delete(idx);
       }
@@ -396,23 +559,50 @@ export async function downloadModel(
     // Verify hashes for completed shards; drop and re-download corrupt shards
     for (const idx of Array.from(state.completedShards)) {
       try {
-        await loadShard(idx, { verify: true });
+        if (directSourceArtifact) {
+          const sourceEntry = directSourceArtifact.sourceFiles[idx];
+          if (!sourceEntry?.hash) {
+            continue;
+          }
+          const payload = await loadFileFromStore(sourceEntry.path);
+          const computedHash = await computeAssetHash(payload, sourceEntry.hashAlgorithm);
+          if (computedHash !== sourceEntry.hash) {
+            throw new Error(
+              `Hash mismatch for source asset ${sourceEntry.path}: expected ${sourceEntry.hash}, got ${computedHash}`
+            );
+          }
+        } else {
+          await loadShard(idx, { verify: true });
+        }
       } catch (err) {
         log.warn('Downloader', `Shard ${idx} failed verification, re-downloading`);
         state.completedShards.delete(idx);
-        await deleteShard(idx);
+        if (directSourceArtifact) {
+          const sourceEntry = directSourceArtifact.sourceFiles[idx];
+          if (sourceEntry) {
+            await deleteFileFromStore(sourceEntry.path);
+          }
+        } else {
+          await deleteShard(idx);
+        }
       }
     }
   }
 
   // Create abort controller
   const abortController = new AbortController();
+  const abortFromExternalSignal = () => {
+    abortController.abort();
+  };
+  if (externalSignal && typeof externalSignal.addEventListener === 'function') {
+    externalSignal.addEventListener('abort', abortFromExternalSignal, { once: true });
+  }
   activeDownloads.set(storageModelId, {
     state,
     abortController
   });
 
-  const totalShards = manifest.shards.length;
+  const totalShards = trackedShards.length;
   const requiredEncoding = getRequiredContentEncoding();
   
   const pendingShards = [];
@@ -427,7 +617,7 @@ export async function downloadModel(
   // Progress tracking
   let downloadedBytes = 0;
   for (const idx of state.completedShards) {
-    const info = manifest.shards[idx];
+    const info = trackedShards[idx];
     if (info) downloadedBytes += info.size;
   }
 
@@ -464,9 +654,9 @@ export async function downloadModel(
         manifest,
         totalShards,
         completedShards:  (state).completedShards.size,
-        totalBytes: manifest.totalSize,
+        totalBytes: trackedTotalBytes,
         downloadedBytes,
-        percent: (downloadedBytes / manifest.totalSize) * 100,
+        percent: trackedTotalBytes > 0 ? (downloadedBytes / trackedTotalBytes) * 100 : 0,
         status:  (state).status,
         currentShard,
         speed: speedTracker.speed,
@@ -492,58 +682,96 @@ export async function downloadModel(
     updateProgress(shardIndex);
 
     try {
-      const shardInfo = manifest.shards[shardIndex];
-      if (!shardInfo) {
-        throw new Error(`Invalid shard index: ${shardIndex}`);
-      }
-      const algorithm = manifest.hashAlgorithm;
-      if (!algorithm) {
-        throw new Error('Manifest missing hashAlgorithm for download verification.');
-      }
-      const expectedHash = shardInfo.hash;
-      if (!expectedHash) {
-        throw new Error(`Shard ${shardIndex} is missing hash in manifest`);
-      }
-      const expectedSize = Number.isFinite(shardInfo.size) ? Math.floor(shardInfo.size) : null;
-      const result = await downloadShard(baseUrl, shardIndex, shardInfo, {
-        signal: abortController.signal,
-        algorithm,
-        requiredEncoding,
-        expectedHash,
-        expectedSize,
-        expectedManifestVersionSet: manifestVersionSet,
-        writeToStore: true,
-        onProgress: ( p) => {
-          const prev = shardProgress.get(shardIndex) || 0;
-          const delta = Math.max(0, p.receivedBytes - prev);
-          shardProgress.set(shardIndex, p.receivedBytes);
-          downloadedBytes += delta;
-          updateProgress(shardIndex);
+      if (directSourceArtifact) {
+        const sourceAsset = directSourceArtifact.sourceFiles[shardIndex];
+        if (!sourceAsset) {
+          throw new Error(`Invalid source asset index: ${shardIndex}`);
         }
-      });
+        const result = await downloadSourceAsset(
+          joinArtifactUrl(baseUrl, sourceAsset.path),
+          sourceAsset,
+          {
+            signal: abortController.signal,
+            onProgress: (receivedBytes) => {
+              const prev = shardProgress.get(shardIndex) || 0;
+              const delta = Math.max(0, receivedBytes - prev);
+              shardProgress.set(shardIndex, receivedBytes);
+              downloadedBytes += delta;
+              updateProgress(shardIndex);
+            },
+          }
+        );
 
-      if (result.hash !== expectedHash) {
-        await deleteShard(shardIndex);
-        throw new Error(`Hash mismatch for shard ${shardIndex}: expected ${expectedHash}, got ${result.hash}`);
-      }
+        const source = typeof result.source === 'string' ? result.source : 'unknown';
+        const sourceStats = normalizeSourceStats(state.sourceStats);
+        if (source in sourceStats) {
+          sourceStats[source] += 1;
+        } else {
+          sourceStats.unknown += 1;
+        }
+        state.sourceStats = sourceStats;
+        state.lastSource = source;
+        state.lastSourcePath = typeof result.path === 'string' ? result.path : null;
+
+        const observedBytes = shardProgress.get(shardIndex) || 0;
+        const shardBytes = sourceAsset.size ?? result.bytes ?? observedBytes;
+        if (shardBytes > observedBytes) {
+          downloadedBytes += shardBytes - observedBytes;
+        }
+      } else {
+        const shardInfo = manifest.shards[shardIndex];
+        if (!shardInfo) {
+          throw new Error(`Invalid shard index: ${shardIndex}`);
+        }
+        const algorithm = manifest.hashAlgorithm;
+        if (!algorithm) {
+          throw new Error('Manifest missing hashAlgorithm for download verification.');
+        }
+        const expectedHash = getExpectedShardHash(shardInfo, algorithm);
+        if (!expectedHash) {
+          throw new Error(`Shard ${shardIndex} is missing hash in manifest`);
+        }
+        const expectedSize = Number.isFinite(shardInfo.size) ? Math.floor(shardInfo.size) : null;
+        const result = await downloadShard(baseUrl, shardIndex, shardInfo, {
+          signal: abortController.signal,
+          algorithm,
+          requiredEncoding,
+          expectedHash,
+          expectedSize,
+          expectedManifestVersionSet: manifestVersionSet,
+          writeToStore: true,
+          onProgress: ( p) => {
+            const prev = shardProgress.get(shardIndex) || 0;
+            const delta = Math.max(0, p.receivedBytes - prev);
+            shardProgress.set(shardIndex, p.receivedBytes);
+            downloadedBytes += delta;
+            updateProgress(shardIndex);
+          }
+        });
 
-      await persistDownloadedShardIfNeeded(result, shardIndex);
+        if (result.hash !== expectedHash) {
+          await deleteShard(shardIndex);
+          throw new Error(`Hash mismatch for shard ${shardIndex}: expected ${expectedHash}, got ${result.hash}`);
+        }
 
-      const source = typeof result.source === 'string' ? result.source : 'unknown';
-      const sourceStats = normalizeSourceStats(state.sourceStats);
-      if (source in sourceStats) {
-        sourceStats[source] += 1;
-      } else {
-        sourceStats.unknown += 1;
-      }
-      state.sourceStats = sourceStats;
-      state.lastSource = source;
-      state.lastSourcePath = typeof result.path === 'string' ? result.path : null;
-
-      const observedBytes = shardProgress.get(shardIndex) || 0;
-      const shardBytes = shardInfo.size ?? result.bytes ?? observedBytes;
-      if (shardBytes > observedBytes) {
-        downloadedBytes += shardBytes - observedBytes;
+        await persistDownloadedShardIfNeeded(result, shardIndex);
+
+        const source = typeof result.source === 'string' ? result.source : 'unknown';
+        const sourceStats = normalizeSourceStats(state.sourceStats);
+        if (source in sourceStats) {
+          sourceStats[source] += 1;
+        } else {
+          sourceStats.unknown += 1;
+        }
+        state.sourceStats = sourceStats;
+        state.lastSource = source;
+        state.lastSourcePath = typeof result.path === 'string' ? result.path : null;
+
+        const observedBytes = shardProgress.get(shardIndex) || 0;
+        const shardBytes = shardInfo.size ?? result.bytes ?? observedBytes;
+        if (shardBytes > observedBytes) {
+          downloadedBytes += shardBytes - observedBytes;
+        }
       }
 
       // Update state
@@ -599,6 +827,10 @@ export async function downloadModel(
     // Wait for any remaining downloads to complete
     await Promise.all([...downloadPromises]);
 
+    if (abortController.signal.aborted) {
+      throw createAbortError();
+    }
+
     // Verify all shards completed
     if (state.completedShards.size === totalShards) {
       state.status = 'completed';
@@ -606,35 +838,50 @@ export async function downloadModel(
       // Save manifest to OPFS
       await saveManifest(manifestJson);
 
-      // Download tokenizer assets if specified
-      const tokenizer =  (manifest.tokenizer);
-      const hasBundledTokenizer = (tokenizer?.type === 'bundled' || tokenizer?.type === 'huggingface') && tokenizer?.file;
-      if (hasBundledTokenizer) {
-        try {
+      if (directSourceArtifact) {
+        for (const asset of directSourceArtifact.auxiliaryFiles) {
+          const alreadyPresent = await fileExistsInStore(asset.path);
+          if (alreadyPresent) {
+            continue;
+          }
+          await downloadSourceAsset(joinArtifactUrl(baseUrl, asset.path), asset, {
+            signal: abortController.signal,
+            onProgress: (receivedBytes) => {
+              const previous = shardProgress.get(asset.path) || 0;
+              const delta = Math.max(0, receivedBytes - previous);
+              shardProgress.set(asset.path, receivedBytes);
+              downloadedBytes += delta;
+              updateProgress(null);
+            },
+          });
+          const observedBytes = shardProgress.get(asset.path) || 0;
+          shardProgress.delete(asset.path);
+          const assetBytes = asset.size ?? observedBytes;
+          if (assetBytes > observedBytes) {
+            downloadedBytes += assetBytes - observedBytes;
+          }
+          updateProgress(null, true);
+        }
+      } else {
+        // Download tokenizer assets if specified
+        const tokenizer =  (manifest.tokenizer);
+        if (isTokenizerJsonRequired(tokenizer)) {
           const tokenizerUrl = `${baseUrl}/${ (tokenizer).file}`;
           log.verbose('Downloader', `Fetching bundled tokenizer from ${tokenizerUrl}`);
           const tokenizerResponse = await fetchWithRetry(tokenizerUrl);
           const tokenizerJson = await tokenizerResponse.text();
           await saveTokenizer(tokenizerJson);
           log.verbose('Downloader', 'Saved bundled tokenizer.json');
-        } catch (err) {
-          log.warn('Downloader', `Failed to download tokenizer.json: ${ (err).message}`);
-          // Non-fatal - model will fall back to HuggingFace tokenizer
         }
-      }
 
-      const sentencepieceModel = tokenizer?.sentencepieceModel
-        ?? (tokenizer?.type === 'sentencepiece' ? 'tokenizer.model' : null);
-      if (sentencepieceModel) {
-        try {
+        const sentencepieceModel = getTokenizerModelPath(tokenizer);
+        if (sentencepieceModel) {
           const modelUrl = `${baseUrl}/${sentencepieceModel}`;
           log.verbose('Downloader', `Fetching sentencepiece model from ${modelUrl}`);
           const modelResponse = await fetchWithRetry(modelUrl);
           const modelBuffer = await modelResponse.arrayBuffer();
           await saveTokenizerModel(modelBuffer);
           log.verbose('Downloader', 'Saved tokenizer.model');
-        } catch (err) {
-          log.warn('Downloader', `Failed to download tokenizer.model: ${ (err).message}`);
         }
       }
 
@@ -660,6 +907,9 @@ export async function downloadModel(
     throw error;
 
   } finally {
+    if (externalSignal && typeof externalSignal.removeEventListener === 'function') {
+      externalSignal.removeEventListener('abort', abortFromExternalSignal);
+    }
     activeDownloads.delete(storageModelId);
   }
 }
@@ -684,7 +934,10 @@ export async function resumeDownload(
     throw new Error(`No download state found for model: ${modelId}`);
   }
 
-  return downloadModel(state.baseUrl, onProgress, options);
+  return downloadModel(state.baseUrl, onProgress, {
+    ...options,
+    modelId: options.modelId ?? state.modelId,
+  });
 }
 
 
@@ -694,11 +947,13 @@ export async function getDownloadProgress(modelId) {
   if (active) {
     const state = active.state;
     const manifest = state.manifest;
-    const totalShards = manifest?.shards?.length || 0;
+    const directSourceArtifact = resolveSourceArtifact(manifest);
+    const trackedShards = directSourceArtifact ? directSourceArtifact.sourceFiles : (manifest?.shards || []);
+    const totalShards = trackedShards.length;
 
     let downloadedBytes = 0;
     for (const idx of state.completedShards) {
-      const info = manifest?.shards?.[idx];
+      const info = trackedShards[idx];
       if (info) downloadedBytes += info.size;
     }
 
@@ -706,9 +961,14 @@ export async function getDownloadProgress(modelId) {
       modelId,
       totalShards,
       completedShards: state.completedShards.size,
-      totalBytes: manifest?.totalSize || 0,
+      totalBytes: directSourceArtifact ? directSourceArtifact.totalBytes : (manifest?.totalSize || 0),
       downloadedBytes,
-      percent: manifest ? (downloadedBytes / manifest.totalSize) * 100 : 0,
+      percent: manifest
+        ? (
+          downloadedBytes
+          / (directSourceArtifact ? directSourceArtifact.totalBytes : manifest.totalSize || 1)
+        ) * 100
+        : 0,
       status: state.status,
       currentShard: null,
       speed: 0,
@@ -721,20 +981,25 @@ export async function getDownloadProgress(modelId) {
   // Check saved state
   const state = await loadDownloadState(modelId);
   if (!state) return null;
+  const directSourceArtifact = resolveSourceArtifact(state.manifest);
+  const trackedShards = directSourceArtifact ? directSourceArtifact.sourceFiles : state.manifest.shards;
 
   let downloadedBytes = 0;
   for (const idx of state.completedShards) {
-    const shard = state.manifest.shards[idx];
+    const shard = trackedShards[idx];
     if (shard) downloadedBytes += shard.size;
   }
 
   return {
     modelId,
-    totalShards: state.manifest.shards.length,
+    totalShards: trackedShards.length,
     completedShards: state.completedShards.size,
-    totalBytes: state.manifest.totalSize,
+    totalBytes: directSourceArtifact ? directSourceArtifact.totalBytes : state.manifest.totalSize,
     downloadedBytes,
-    percent: (downloadedBytes / state.manifest.totalSize) * 100,
+    percent: (
+      downloadedBytes
+      / (directSourceArtifact ? directSourceArtifact.totalBytes : state.manifest.totalSize || 1)
+    ) * 100,
     status: state.status,
     currentShard: null,
     speed: 0,
@@ -790,7 +1055,8 @@ export async function checkDownloadNeeded(modelId) {
     };
   }
 
-  const totalShards = state.manifest.shards.length;
+  const directSourceArtifact = resolveSourceArtifact(state.manifest);
+  const totalShards = directSourceArtifact ? directSourceArtifact.sourceFiles.length : state.manifest.shards.length;
   
   const missingShards = [];
 
@@ -808,6 +1074,22 @@ export async function checkDownloadNeeded(modelId) {
     };
   }
 
+  if (directSourceArtifact) {
+    const missingAuxiliaryFiles = [];
+    for (const entry of directSourceArtifact.auxiliaryFiles) {
+      if (!(await fileExistsInStore(entry.path))) {
+        missingAuxiliaryFiles.push(entry.path);
+      }
+    }
+    if (missingAuxiliaryFiles.length > 0) {
+      return {
+        needed: true,
+        reason: `Missing ${missingAuxiliaryFiles.length} direct-source auxiliary file(s)`,
+        missingShards: [],
+      };
+    }
+  }
+
   return {
     needed: false,
     reason: 'Model fully downloaded',

package/src/storage/index.d.ts

@@ -20,6 +20,7 @@ export {
   writeShard,
   createShardWriter,
   createConversionShardWriter,
+  createFileWriter,
   loadShard,
   loadShardRange,
   streamShardRange,
@@ -32,6 +33,7 @@ export {
   listModels,
   listFilesInStore,
   loadFileFromStore,
+  loadFileRangeFromStore,
   streamFileFromStore,
   getModelInfo,
   modelExists,
@@ -46,6 +48,7 @@ export {
   saveAuxFile,
   loadAuxFile,
   loadAuxText,
+  deleteFileFromStore,
   cleanup,
 } from './shard-manager.js';
 export type {