@qvac/sdk 0.8.1 → 1.1.0

package/dist/server/rpc/handlers/load-model/hyperdrive.js

@@ -1,41 +1,42 @@
 import fs, { promises as fsPromises } from "bare-fs";
 import path from "bare-path";
+import crypto from "bare-crypto";
 import Corestore from "corestore";
 import Hyperswarm, {} from "hyperswarm";
 import Hyperdrive from "hyperdrive";
 import {} from "bare-stream";
 import { AbortController } from "bare-abort-controller";
-import { getEnv } from "../../../env.js";
-import { getModelsCacheDir, generateShortHash, detectShardedModel, getShardedModelCacheDir, getShardPath, checkShardCompleteness, measureChecksum, extractTensorsFromShards, calculatePercentage, } from "../../../utils/index.js";
-import { getModelBySrc } from "../../../../models/registry/index.js";
+import { getEnv } from "../../../worker.js";
+import { getModelsCacheDir, generateShortHash } from "../../../utils/index.js";
+import { getModelBySrc } from "../../../../models/hyperdrive/index.js";
 import { getActiveDownload, registerDownload, unregisterDownload, createHyperdriveDownloadKey, shouldClearCache, clearClearCacheFlag, } from "./download-manager.js";
-import { getSDKConfig } from "../../../bare/registry/config-registry.js";
-import { FileNotFoundError, ChecksumValidationFailedError, DownloadCancelledError, HyperdriveDownloadFailedError, ModelLoadFailedError, NoBlobFoundError, } from "../../../../utils/errors-server.js";
-import { getServerLogger } from "../../../../logging/index.js";
-const logger = getServerLogger();
-function getCorestoreDir(hyperdriveKey) {
-    return path.join(getEnv().HOME_DIR, ".qvac", "corestore", hyperdriveKey);
+function createProgressHandler(blob, progressCallback, downloadKey, initialBytes = 0) {
+    let downloadedBytes = initialBytes;
+    return (index, bytes) => {
+        if (index >= blob.blockOffset &&
+            index < blob.blockOffset + blob.blockLength) {
+            downloadedBytes += bytes;
+            if (progressCallback && downloadKey) {
+                progressCallback({
+                    type: "modelProgress",
+                    downloaded: downloadedBytes,
+                    total: blob.byteLength,
+                    percentage: blob.byteLength > 0 ? (downloadedBytes / blob.byteLength) * 100 : 0,
+                    downloadKey,
+                });
+            }
+        }
+    };
 }
 async function setupHyperdrive(hyperdriveKey, corestoreDir) {
     const corestore = new Corestore(corestoreDir);
     await corestore.ready();
     const drive = new Hyperdrive(corestore, Buffer.from(hyperdriveKey, "hex"));
     await drive.ready();
-    const getRelays = () => {
-        const config = getSDKConfig();
-        const relayPublicKeys = config.swarmRelays;
-        if (!relayPublicKeys || relayPublicKeys.length === 0) {
-            return null;
-        }
-        return relayPublicKeys.map((key) => Buffer.from(key, "hex"));
-    };
-    const swarmOptions = {
-        relayThrough: getRelays,
-    };
-    const swarm = new Hyperswarm(swarmOptions);
+    const swarm = new Hyperswarm();
     const doneFindingPeers = drive.findingPeers();
     swarm.on("connection", (connection) => {
-        logger.debug(`🤝 Connected to peer for hyperdrive download`);
+        console.log(`🤝 Connected to peer for hyperdrive download`);
         drive.replicate(connection);
     });
     swarm.join(drive.discoveryKey, { server: false, client: true });
@@ -43,7 +44,7 @@ async function setupHyperdrive(hyperdriveKey, corestoreDir) {
     return { corestore, drive, swarm, corestoreDir };
 }
 async function cleanupHyperdrive(setup) {
-    logger.debug("🧹 Cleaning up hyperdrive setup");
+    console.log("🧹 Cleaning up hyperdrive setup");
     if (!setup.swarm.suspended) {
         try {
             await setup.swarm.suspend();
@@ -57,12 +58,12 @@ async function cleanupHyperdrive(setup) {
     await setup.corestore.close();
 }
 export async function deleteCorestoreDirectory(corestoreDir) {
-    logger.info(`🗑️ DELETING corestore directory... ${corestoreDir}`);
+    console.log(`🗑️ DELETING corestore directory... ${corestoreDir}`);
     try {
         await fsPromises.rm(corestoreDir, { recursive: true, force: true });
     }
     catch (e) {
-        logger.error(`Error deleting corestore directory: ${e instanceof Error ? e.message : String(e)}`);
+        console.error(`Error deleting corestore directory: ${e instanceof Error ? e.message : String(e)}`);
     }
 }
 async function findFileInHyperdrive(drive, modelFileName) {
@@ -102,7 +103,7 @@ async function findFileInHyperdrive(drive, modelFileName) {
     const filesInfo = availableFiles.length > 0
         ? `Available files: ${availableFiles.join(", ")}`
         : "Could not list available files";
-    throw new FileNotFoundError(`${modelFileName} in hyperdrive. ${filesInfo}`);
+    throw new Error(`File ${modelFileName} not found in hyperdrive. ${filesInfo}`);
 }
 async function checkFileDownloaded(drive, entry) {
     const blobs = await drive.getBlobs();
@@ -142,7 +143,40 @@ async function validateFileExistence(drive, file) {
         totalBytes,
     };
 }
-async function validateCachedFile(modelPath, modelFileName, expectedSize, expectedChecksum, hooks) {
+async function validateFileChecksum(filePath, expectedChecksum) {
+    console.log(`🔍 Validating SHA256 checksum...`);
+    return new Promise((resolve) => {
+        try {
+            const hash = crypto.createHash("sha256");
+            const stream = fs.createReadStream(filePath);
+            stream.on("data", (chunk) => {
+                hash.update(chunk);
+            });
+            stream.on("end", () => {
+                const actualChecksum = hash.digest("hex");
+                console.log(`Expected: ${String(expectedChecksum)}`);
+                console.log(`Actual:   ${String(actualChecksum)}`);
+                const isValid = actualChecksum === expectedChecksum;
+                if (isValid) {
+                    console.log(`✅ Checksum validation passed!`);
+                }
+                else {
+                    console.log(`❌ Checksum validation FAILED! File may be corrupted.`);
+                }
+                resolve(isValid);
+            });
+            stream.on("error", (error) => {
+                console.error(`❌ Error validating checksum: ${error.message}`);
+                resolve(false);
+            });
+        }
+        catch (error) {
+            console.error(`❌ Error validating checksum: ${error instanceof Error ? error.message : String(error)}`);
+            resolve(false);
+        }
+    });
+}
+async function validateCachedFile(modelPath, modelFileName, expectedSize, expectedChecksum) {
     try {
         await fsPromises.access(modelPath);
         const localStats = await fsPromises.stat(modelPath);
@@ -150,16 +184,21 @@ async function validateCachedFile(modelPath, modelFileName, expectedSize, expect
         // For hyperdrive: only validate if file is complete
         // Partial files are ignored - corestore state is the source of truth for resume
         if (localSize === expectedSize) {
-            logger.info(`✅ Model cached with correct size: ${modelPath}`);
+            console.log(`✅ Model cached with correct size: ${modelPath}`);
             // Always validate checksum if provided, even when size matches
             if (expectedChecksum && expectedChecksum.length === 64) {
-                const checksum = await measureChecksum(modelPath, hooks);
-                if (checksum !== expectedChecksum) {
-                    throw new ChecksumValidationFailedError(`${modelFileName}. Expected: ${expectedChecksum}. Actual: ${checksum}. File may be corrupted`);
+                const isValid = await validateFileChecksum(modelPath, expectedChecksum);
+                if (isValid) {
+                    return modelPath;
+                }
+                else {
+                    throw new Error(`File ${modelFileName} is fully downloaded but checksum validation failed. Expected: ${expectedChecksum}. File may be corrupted.`);
                 }
             }
-            logger.info(`✅ Model already cached and size validated: ${modelPath}`);
-            return modelPath;
+            else {
+                console.log(`✅ Model already cached and size validated: ${modelPath}`);
+                return modelPath;
+            }
         }
         // File exists but incomplete - will be overwritten during download
         // Resume state will be determined by corestore blocks, not file size
@@ -170,66 +209,91 @@ async function validateCachedFile(modelPath, modelFileName, expectedSize, expect
         return null;
     }
 }
-async function downloadSingleFileToFilesystem(modelPath, modelFileName, hyperdriveKey, corestoreDir, downloadKey, expectedChecksum, progressCallback, seed, signal, hooks) {
+async function performHyperdriveDownload(modelPath, modelFileName, hyperdriveKey, corestoreDir, downloadKey, progressCallback, seed, signal) {
     // Check if already aborted
     if (signal?.aborted) {
-        throw new DownloadCancelledError();
+        throw new Error("Download cancelled");
     }
-    logger.info(`Downloading model from hyperdrive ${hyperdriveKey} to ${modelPath}`);
+    console.log(`Downloading model from hyperdrive ${hyperdriveKey} to ${modelPath}`);
     const setup = await setupHyperdrive(hyperdriveKey, corestoreDir);
+    // Set up abort handler
+    let readStream;
+    let writeStream;
     const cleanup = () => {
-        logger.info("🛑 Aborting hyperdrive download...");
+        console.log("🛑 Aborting hyperdrive download...");
         setup.swarm.suspended = true;
         setup.swarm.suspend().catch(() => {
             // Ignore errors during emergency suspend
         });
+        // Destroy streams if they exist
+        if (readStream) {
+            readStream.destroy();
+        }
+        if (writeStream) {
+            writeStream.destroy();
+        }
     };
     signal?.addEventListener("abort", cleanup);
-    let downloadSucceeded = false;
+    let downloadHandler;
     try {
-        const progressContext = progressCallback
-            ? {
-                downloadKey,
-                callback: progressCallback,
-            }
-            : undefined;
-        await downloadAndValidateFile(setup.drive, modelFileName, modelPath, 0, // Size will be determined from drive
-        expectedChecksum, progressContext, signal, hooks);
-        logger.info(`✅ Model downloaded successfully to ${modelPath}`);
-        // Send final 100% progress update
-        if (progressCallback) {
-            try {
-                const stats = await fsPromises.stat(modelPath);
-                progressCallback({
-                    type: "modelProgress",
-                    downloaded: stats.size,
-                    total: stats.size,
-                    percentage: 100,
-                    downloadKey,
-                });
-            }
-            catch {
-                // Ignore stat errors
-            }
+        const { entry, blobs, targetPath, blocksDownloaded, totalBlocks, totalBytes, } = await validateFileExistence(setup.drive, modelFileName);
+        const blob = entry?.value?.blob;
+        if (!blob) {
+            throw new Error(`No blob found for ${modelFileName}`);
+        }
+        // Calculate already downloaded bytes for resuming
+        const bytesPerBlock = totalBlocks > 0 ? totalBytes / totalBlocks : 0;
+        const initialBytes = Math.floor(blocksDownloaded * bytesPerBlock);
+        // Track only downloads for blocks within our file's blob range
+        downloadHandler = createProgressHandler(blob, progressCallback, downloadKey, initialBytes);
+        blobs.core.on("download", downloadHandler);
+        // Check if aborted before starting streams
+        if (signal?.aborted) {
+            throw new Error("Download cancelled");
+        }
+        // Trigger download for this file (hyperdrive will only download missing blocks)
+        setup.drive.download(targetPath);
+        readStream = setup.drive.createReadStream(targetPath);
+        writeStream = fs.createWriteStream(modelPath);
+        // Pipe the streams
+        readStream.pipe(writeStream);
+        // Wait for download to complete
+        await new Promise((resolve, reject) => {
+            writeStream.on("finish", resolve);
+            writeStream.on("error", reject);
+            readStream.on("error", reject);
+            // Also reject on abort
+            signal?.addEventListener("abort", () => {
+                reject(new Error("Download cancelled"));
+            }, { once: true });
+        });
+        console.log(`✅ Model downloaded successfully to ${modelPath}`);
+        // Delete corestore directory after successful download (only when not seeding, to save space)
+        if (!seed) {
+            await deleteCorestoreDirectory(setup.corestoreDir);
         }
-        downloadSucceeded = true;
     }
     catch (error) {
-        logger.error("❌ Error during hyperdrive download:", error instanceof Error ? error.message : String(error));
+        console.error("❌ Error during hyperdrive download:", error instanceof Error ? error.message : String(error));
         // Preserve cancellation errors
         if (error instanceof Error && error.message === "Download cancelled") {
             throw error;
         }
-        throw new HyperdriveDownloadFailedError(`Download failed for ${modelFileName} from hyperdrive: ${hyperdriveKey}`, error);
+        throw new Error(`Download failed for ${modelFileName} from hyperdrive: ${hyperdriveKey}`);
     }
     finally {
         signal?.removeEventListener("abort", cleanup);
+        if (downloadHandler) {
+            try {
+                const { blobs } = await validateFileExistence(setup.drive, modelFileName);
+                blobs.core.off("download", downloadHandler);
+            }
+            catch {
+                // If we can't get blobs, setup is already closed
+            }
+        }
         if (!seed) {
             await cleanupHyperdrive(setup);
-            // Delete only after close - Windows EBUSY on LOCK if deleted before corestore.close()
-            if (downloadSucceeded) {
-                await deleteCorestoreDirectory(corestoreDir);
-            }
         }
         // Only delete corestore and partial file if user explicitly requested clearCache
         if (signal?.aborted && shouldClearCache(downloadKey)) {
@@ -244,110 +308,56 @@ async function downloadSingleFileToFilesystem(modelPath, modelFileName, hyperdri
         clearClearCacheFlag(downloadKey);
     }
 }
-async function downloadAndValidateFile(drive, sourceFileName, targetFilePath, expectedSize, expectedChecksum, progressContext, signal, hooks) {
-    const { entry, blobs, targetPath, blocksDownloaded, totalBlocks, totalBytes, } = await validateFileExistence(drive, sourceFileName);
-    const blob = entry?.value?.blob;
-    if (!blob) {
-        throw new NoBlobFoundError(sourceFileName);
+export async function downloadModelFromHyperdrive(hyperdriveKey, modelFileName, seed, progressCallback, expectedChecksum) {
+    const downloadKey = createHyperdriveDownloadKey(hyperdriveKey, modelFileName);
+    // Check if already downloading
+    const existing = getActiveDownload(downloadKey);
+    if (existing) {
+        return existing.promise;
     }
-    // Use actual file size from drive if expectedSize is 0
-    const fileSizeToValidate = expectedSize || totalBytes;
-    // Setup progress tracking
-    let downloadedBytes = 0;
-    const bytesPerBlock = totalBlocks > 0 ? totalBytes / totalBlocks : 0;
-    const initialBytes = Math.floor(blocksDownloaded * bytesPerBlock);
-    downloadedBytes = initialBytes;
-    const progressHandler = (index, bytes) => {
-        if (index >= blob.blockOffset &&
-            index < blob.blockOffset + blob.blockLength) {
-            downloadedBytes += bytes;
-            const cappedDownloaded = Math.min(downloadedBytes, totalBytes);
-            if (progressContext) {
-                const baseProgress = {
-                    type: "modelProgress",
-                    downloaded: cappedDownloaded,
-                    total: totalBytes,
-                    percentage: calculatePercentage(cappedDownloaded, totalBytes),
-                    downloadKey: progressContext.downloadKey,
-                };
-                if (progressContext.shardInfo) {
-                    const currentOverall = progressContext.shardInfo.overallDownloaded + cappedDownloaded;
-                    progressContext.callback({
-                        ...baseProgress,
-                        shardInfo: {
-                            currentShard: progressContext.shardInfo.currentShard,
-                            totalShards: progressContext.shardInfo.totalShards,
-                            shardName: progressContext.shardInfo.shardName,
-                            overallDownloaded: currentOverall,
-                            overallTotal: progressContext.shardInfo.overallTotal,
-                            overallPercentage: progressContext.shardInfo.overallTotal > 0
-                                ? (currentOverall / progressContext.shardInfo.overallTotal) *
-                                    100
-                                : 0,
-                        },
-                    });
-                }
-                else {
-                    progressContext.callback(baseProgress);
-                }
-            }
-        }
-    };
-    blobs.core.on("download", progressHandler);
-    let readStream;
-    let writeStream;
-    try {
-        // Download blocks and stream to file
-        drive.download(targetPath);
-        readStream = drive.createReadStream(targetPath);
-        writeStream = fs.createWriteStream(targetFilePath);
-        readStream.pipe(writeStream);
-        await new Promise((resolve, reject) => {
-            writeStream.on("finish", resolve);
-            writeStream.on("error", reject);
-            readStream.on("error", reject);
-            signal?.addEventListener("abort", () => reject(new Error("Download cancelled")), { once: true });
-        });
-        blobs.core.off("download", progressHandler);
-        // Validate file size
-        const stats = await fsPromises.stat(targetFilePath);
-        if (stats.size !== fileSizeToValidate) {
-            throw new ChecksumValidationFailedError(`${path.basename(targetFilePath)}. File size mismatch: expected ${fileSizeToValidate}, got ${stats.size}`);
-        }
-        // Validate checksum
-        if (expectedChecksum && expectedChecksum.length === 64) {
-            const checksum = await measureChecksum(targetFilePath, hooks);
-            if (checksum !== expectedChecksum) {
-                await fsPromises.unlink(targetFilePath);
-                throw new ChecksumValidationFailedError(`${path.basename(targetFilePath)}. Expected: ${expectedChecksum}. Actual: ${checksum}`);
+    const corestoreDir = path.join(getEnv().HOME_DIR, ".qvac", "corestore", hyperdriveKey);
+    const cacheDir = getModelsCacheDir();
+    const sourceHash = generateShortHash(`${hyperdriveKey}/${modelFileName}`);
+    const modelPath = path.join(cacheDir, `${sourceHash}_${modelFileName}`);
+    // Get model definition to get expected size (if it's a known model)
+    const model = getModelBySrc(modelFileName, hyperdriveKey);
+    // First, check if we already have a valid cached file (only if we have model metadata)
+    if (model) {
+        const cachedPath = await validateCachedFile(modelPath, modelFileName, model.expectedSize, expectedChecksum || model.sha256Checksum);
+        if (cachedPath) {
+            // Delete any leftover corestore directory from previous runs (only when not seeding)
+            if (!seed) {
+                await deleteCorestoreDirectory(corestoreDir);
             }
+            return cachedPath;
         }
-        else if (!expectedChecksum) {
-            logger.warn(`⚠️  No checksum available for validation`);
-        }
-    }
-    catch (error) {
-        if (readStream) {
-            readStream.destroy();
-        }
-        if (writeStream) {
-            writeStream.destroy();
-        }
-        blobs.core.off("download", progressHandler);
-        throw error;
     }
-}
-function createManagedDownload(downloadKey, hyperdriveKey, modelFileName, downloadFn, progressCallback) {
+    // Create managed download with AbortController
     const abortController = new AbortController();
     const downloadPromise = (async () => {
         try {
-            return await downloadFn(abortController.signal);
+            // Download the file from hyperdrive
+            await performHyperdriveDownload(modelPath, modelFileName, hyperdriveKey, corestoreDir, downloadKey, progressCallback, seed, abortController.signal);
+            // Validate checksum using provided checksum or model's checksum (if available)
+            const checksumToValidate = expectedChecksum || model?.sha256Checksum;
+            if (checksumToValidate && checksumToValidate.length === 64) {
+                const isValid = await validateFileChecksum(modelPath, checksumToValidate);
+                if (!isValid) {
+                    await fsPromises.unlink(modelPath);
+                    throw new Error(`Checksum validation failed for ${modelFileName}`);
+                }
+            }
+            else {
+                console.log(`⚠️  No valid checksum available for validation`);
+            }
+            return modelPath;
         }
         finally {
+            // Cleanup from download manager
             unregisterDownload(downloadKey);
-            clearClearCacheFlag(downloadKey);
         }
     })();
+    // Register download
     const downloadEntry = {
         key: downloadKey,
         promise: downloadPromise,
@@ -361,206 +371,3 @@ function createManagedDownload(downloadKey, hyperdriveKey, modelFileName, downlo
     registerDownload(downloadKey, downloadEntry);
     return downloadPromise;
 }
-async function downloadShardedFilesToFilesystem(hyperdriveKey, firstShardFileName, shardMetadata, progressCallback, seed, signal, hooks) {
-    if (signal?.aborted) {
-        throw new DownloadCancelledError();
-    }
-    const shardInfo = detectShardedModel(firstShardFileName);
-    if (!shardInfo.isSharded || !shardInfo.totalShards) {
-        throw new ModelLoadFailedError(`Not a sharded model: ${firstShardFileName}`);
-    }
-    const shardDir = getShardedModelCacheDir(hyperdriveKey);
-    const allFiles = shardMetadata.map((s) => s.filename);
-    const numberedShardCount = shardInfo.totalShards;
-    logger.info(`Downloading sharded model with ${numberedShardCount} numbered shards + ${allFiles.length - numberedShardCount} companion files to ${shardDir}`);
-    // Check which files need to be downloaded
-    const invalidIndices = await checkShardCompleteness(hyperdriveKey, allFiles, shardMetadata, hooks ? (ms) => hooks.addChecksumValidationTimeMs(ms) : undefined);
-    if (invalidIndices.length === 0) {
-        logger.info(`✅ All ${allFiles.length} files already downloaded and validated`);
-        hooks?.markCacheHit();
-        if (progressCallback) {
-            const overallTotal = shardMetadata.reduce((sum, shard) => sum + shard.expectedSize, 0);
-            const lastFile = allFiles[allFiles.length - 1];
-            progressCallback({
-                type: "modelProgress",
-                downloaded: overallTotal,
-                total: overallTotal,
-                percentage: 100,
-                downloadKey: createHyperdriveDownloadKey(hyperdriveKey, firstShardFileName),
-                shardInfo: {
-                    currentShard: allFiles.length,
-                    totalShards: allFiles.length,
-                    shardName: lastFile,
-                    overallDownloaded: overallTotal,
-                    overallTotal,
-                    overallPercentage: 100,
-                },
-            });
-        }
-        await extractTensorsFromShards(shardDir, firstShardFileName);
-        return getShardPath(hyperdriveKey, allFiles[0]);
-    }
-    logger.info(`📥 Need to download ${invalidIndices.length} of ${allFiles.length} files`);
-    hooks?.markCacheMiss();
-    // Setup hyperdrive once for all shards
-    const corestoreDir = getCorestoreDir(hyperdriveKey);
-    const setup = await setupHyperdrive(hyperdriveKey, corestoreDir);
-    const cleanup = () => {
-        logger.info("🛑 Aborting sharded hyperdrive download...");
-        setup.swarm.suspended = true;
-        setup.swarm.suspend().catch(() => {
-            // Ignore errors during emergency suspend
-        });
-    };
-    signal?.addEventListener("abort", cleanup);
-    let downloadSucceeded = false;
-    try {
-        // Calculate overall progress
-        const overallTotal = shardMetadata.reduce((sum, shard) => sum + shard.expectedSize, 0);
-        let overallDownloaded = 0;
-        // Download each file sequentially
-        for (let i = 0; i < allFiles.length; i++) {
-            if (signal?.aborted) {
-                throw new DownloadCancelledError();
-            }
-            const file = allFiles[i];
-            const fileMeta = shardMetadata[i];
-            const filePath = getShardPath(hyperdriveKey, file);
-            // Skip if already valid
-            if (!invalidIndices.includes(i)) {
-                logger.debug(`✅ File ${i + 1}/${allFiles.length} already valid: ${file}`);
-                overallDownloaded += fileMeta.expectedSize;
-                // Report progress for skipped files
-                if (progressCallback) {
-                    progressCallback({
-                        type: "modelProgress",
-                        downloaded: fileMeta.expectedSize,
-                        total: fileMeta.expectedSize,
-                        percentage: 100,
-                        downloadKey: createHyperdriveDownloadKey(hyperdriveKey, firstShardFileName),
-                        shardInfo: {
-                            currentShard: i + 1,
-                            totalShards: allFiles.length,
-                            shardName: file,
-                            overallDownloaded,
-                            overallTotal,
-                            overallPercentage: calculatePercentage(overallDownloaded, overallTotal),
-                        },
-                    });
-                }
-                continue;
-            }
-            logger.info(`📥 Downloading file ${i + 1}/${allFiles.length}: ${file}`);
-            const progressContext = progressCallback
-                ? {
-                    downloadKey: createHyperdriveDownloadKey(hyperdriveKey, firstShardFileName),
-                    callback: progressCallback,
-                    shardInfo: {
-                        currentShard: i + 1,
-                        totalShards: allFiles.length,
-                        shardName: file,
-                        overallDownloaded,
-                        overallTotal,
-                    },
-                }
-                : undefined;
-            await downloadAndValidateFile(setup.drive, file, filePath, fileMeta.expectedSize, fileMeta.sha256Checksum, progressContext, signal, hooks);
-            if (signal?.aborted) {
-                throw new DownloadCancelledError();
-            }
-            logger.info(`✅ File ${i + 1}/${allFiles.length} downloaded and validated`);
-            overallDownloaded += fileMeta.expectedSize;
-        }
-        if (progressCallback && overallDownloaded === overallTotal) {
-            const lastFile = allFiles[allFiles.length - 1];
-            progressCallback({
-                type: "modelProgress",
-                downloaded: overallTotal,
-                total: overallTotal,
-                percentage: 100,
-                downloadKey: createHyperdriveDownloadKey(hyperdriveKey, firstShardFileName),
-                shardInfo: {
-                    currentShard: allFiles.length,
-                    totalShards: allFiles.length,
-                    shardName: lastFile,
-                    overallDownloaded,
-                    overallTotal,
-                    overallPercentage: 100,
-                },
-            });
-        }
-        await extractTensorsFromShards(shardDir, firstShardFileName);
-        downloadSucceeded = true;
-    }
-    catch (error) {
-        logger.error("❌ Error during sharded hyperdrive download:", error instanceof Error ? error.message : String(error));
-        // Preserve cancellation errors
-        if (error instanceof Error && error.message === "Download cancelled") {
-            throw error;
-        }
-        throw new HyperdriveDownloadFailedError(`Sharded download failed for ${firstShardFileName} from hyperdrive: ${hyperdriveKey}`, error);
-    }
-    finally {
-        signal?.removeEventListener("abort", cleanup);
-        if (!seed) {
-            await cleanupHyperdrive(setup);
-            // Delete only after close - Windows EBUSY on LOCK if deleted before corestore.close()
-            if (downloadSucceeded) {
-                await deleteCorestoreDirectory(corestoreDir);
-            }
-        }
-        // Only delete corestore and partial files if user explicitly requested clearCache
-        const downloadKey = createHyperdriveDownloadKey(hyperdriveKey, firstShardFileName);
-        if (signal?.aborted && shouldClearCache(downloadKey)) {
-            await deleteCorestoreDirectory(corestoreDir);
-        }
-        clearClearCacheFlag(downloadKey);
-    }
-    return getShardPath(hyperdriveKey, allFiles[0]);
-}
-export async function downloadModelFromHyperdrive(hyperdriveKey, modelFileName, seed, progressCallback, hooks, expectedChecksum) {
-    const downloadKey = createHyperdriveDownloadKey(hyperdriveKey, modelFileName);
-    // Check if already downloading
-    const existing = getActiveDownload(downloadKey);
-    if (existing) {
-        hooks?.markCacheMiss();
-        return existing.promise;
-    }
-    const shardInfo = detectShardedModel(modelFileName);
-    const model = getModelBySrc(modelFileName, hyperdriveKey);
-    if (shardInfo.isSharded && model?.shardMetadata) {
-        return createManagedDownload(downloadKey, hyperdriveKey, modelFileName, (signal) => downloadShardedFilesToFilesystem(hyperdriveKey, modelFileName, model.shardMetadata, progressCallback, seed, signal, hooks), progressCallback);
-    }
-    // Non-sharded model download
-    const corestoreDir = getCorestoreDir(hyperdriveKey);
-    const cacheDir = getModelsCacheDir();
-    const sourceHash = generateShortHash(`${hyperdriveKey}/${modelFileName}`);
-    const modelPath = path.join(cacheDir, `${sourceHash}_${modelFileName}`);
-    // First, check if we already have a valid cached file (only if we have model metadata)
-    if (model) {
-        const cachedPath = await validateCachedFile(modelPath, modelFileName, model.expectedSize, expectedChecksum || model.sha256Checksum, hooks);
-        if (cachedPath) {
-            hooks?.markCacheHit();
-            if (progressCallback) {
-                progressCallback({
-                    type: "modelProgress",
-                    downloaded: model.expectedSize,
-                    total: model.expectedSize,
-                    percentage: 100,
-                    downloadKey,
-                });
-            }
-            // Delete any leftover corestore directory from previous runs (only when not seeding)
-            if (!seed) {
-                await deleteCorestoreDirectory(corestoreDir);
-            }
-            return cachedPath;
-        }
-    }
-    hooks?.markCacheMiss();
-    return createManagedDownload(downloadKey, hyperdriveKey, modelFileName, async (signal) => {
-        const checksumToValidate = expectedChecksum || model?.sha256Checksum || "";
-        await downloadSingleFileToFilesystem(modelPath, modelFileName, hyperdriveKey, corestoreDir, downloadKey, checksumToValidate, progressCallback, seed, signal, hooks);
-        return modelPath;
-    }, progressCallback);
-}

package/dist/server/rpc/handlers/load-model/index.d.ts

@@ -1,4 +1,4 @@
-export { handleLoadModel } from "./handler";
+export { handleLoadModel } from "../load-model";
+export { downloadModelFromHyperdrive } from "./hyperdrive";
 export { resolveModelPath } from "./resolve";
-export { createCancelFunction } from "./download-manager";
 //# sourceMappingURL=index.d.ts.map

package/dist/server/rpc/handlers/load-model/index.js

@@ -1,3 +1,3 @@
-export { handleLoadModel } from "./handler.js";
+export { handleLoadModel } from "../load-model";
+export { downloadModelFromHyperdrive } from "./hyperdrive.js";
 export { resolveModelPath } from "./resolve.js";
-export { createCancelFunction } from "./download-manager.js";