@infersec/conduit 1.6.1 → 1.6.2

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-BS3RcUet.js';
+import { a as asError, s as startInferenceAgent } from './start-CQayYNYZ.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-BS3RcUet.js';
+import { s as startInferenceAgent, a as asError } from './start-CQayYNYZ.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/{start-BS3RcUet.js → start-CQayYNYZ.js}

@@ -31,7 +31,7 @@ import require$$1$5, { fileURLToPath } from 'node:url';
 import require$$1$6 from 'node:async_hooks';
 import require$$1$7 from 'node:console';
 import require$$0$b, { mkdir, readFile, writeFile, stat, unlink, rename, realpath, readlink, readdir, lstat } from 'node:fs/promises';
-import path$1, { dirname, join, win32, posix } from 'node:path';
+import path$1, { join, dirname, win32, posix } from 'node:path';
 import require$$1$8 from 'node:dns';
 import require$$2$4 from 'node:sqlite';
 import require$$0$c from 'path';
@@ -96521,7 +96521,9 @@ async function downloadModelViaHuggingFace({ format, huggingFaceToken, modelSlug
             modelSlug,
             revision
         });
-        if (rangeInfo.supportsRanges) {
+        const { partialPath } = getDownloadPaths({ filePath: file.path, targetDirectory });
+        const shouldAttemptRange = rangeInfo.supportsRanges || existsSync(partialPath);
+        if (shouldAttemptRange) {
             try {
                 await downloadFileWithRange({
                     accessToken,
@@ -96571,6 +96573,12 @@ function encodePathSegments(path) {
         .map(segment => encodeURIComponent(segment))
         .join("/");
 }
+function encodeRepoSlug(modelSlug) {
+    return modelSlug
+        .split("/")
+        .map(segment => encodeURIComponent(segment))
+        .join("/");
+}
 function parseModelRevision(modelSlugWithRevision) {
     const revisionIndex = modelSlugWithRevision.indexOf("@");
     if (revisionIndex === -1) {
@@ -96589,11 +96597,17 @@ function getDownloadPaths({ filePath, targetDirectory }) {
     };
 }
 function getResolveURL({ filePath, modelSlug, revision }) {
-    const encodedRepo = encodeURIComponent(modelSlug);
+    const encodedRepo = encodeRepoSlug(modelSlug);
     const encodedRevision = encodeURIComponent(revision);
     const encodedPath = encodePathSegments(filePath);
     return `https://huggingface.co/${encodedRepo}/resolve/${encodedRevision}/${encodedPath}`;
 }
+function getResolveDownloadURL({ filePath, modelSlug, revision }) {
+    const baseUrl = getResolveURL({ filePath, modelSlug, revision });
+    const resolvedUrl = new URL(baseUrl);
+    resolvedUrl.searchParams.set("download", "1");
+    return resolvedUrl.toString();
+}
 class RangeNotSupportedError extends Error {
     constructor(message) {
         super(message);
@@ -96608,39 +96622,128 @@ function getAuthHeaders(accessToken) {
         Authorization: `Bearer ${accessToken}`
     };
 }
-async function getRangeInfo({ accessToken, filePath, fileSize, modelSlug, revision }) {
-    const url = getResolveURL({ filePath, modelSlug, revision });
+function shouldSendAuthHeader(url) {
+    try {
+        const hostname = new URL(url).hostname.toLowerCase();
+        return hostname.endsWith("huggingface.co") || hostname.endsWith("hf.co");
+    }
+    catch {
+        return false;
+    }
+}
+function parseContentRangeTotal(contentRange) {
+    if (!contentRange) {
+        return null;
+    }
+    const totalMatch = /\/(\d+)$/.exec(contentRange);
+    if (!totalMatch) {
+        return null;
+    }
+    const totalSize = Number(totalMatch[1]);
+    return Number.isFinite(totalSize) ? totalSize : null;
+}
+async function getDownloadURL({ accessToken, filePath, modelSlug, revision }) {
+    const url = getResolveDownloadURL({ filePath, modelSlug, revision });
     try {
         const response = await undiciExports.fetch(url, {
-            method: "HEAD",
+            method: "GET",
+            redirect: "manual",
             headers: getAuthHeaders(accessToken)
         });
-        if (!response.ok) {
+        try {
+            if (response.status >= 300 && response.status < 400) {
+                const location = response.headers.get("location");
+                if (location) {
+                    return { reason: null, status: response.status, url: location };
+                }
+            }
+            if (response.ok) {
+                return { reason: null, status: response.status, url: response.url };
+            }
+            let responseBody = "<unavailable>";
+            try {
+                responseBody = await response.text();
+            }
+            catch {
+                responseBody = "<failed to read body>";
+            }
+            if (responseBody.length > 2000) {
+                responseBody = responseBody.slice(0, 2000);
+            }
+            console.warn(`Resolve request failed body for ${filePath}: ${responseBody}`);
             return {
-                reason: `HEAD request failed with status ${response.status}`,
-                supportsRanges: false,
-                totalSize: null
+                reason: `Resolve request failed with status ${response.status}`,
+                status: response.status,
+                url: null
             };
         }
-        const acceptRanges = response.headers.get("accept-ranges")?.toLowerCase() ?? "";
-        const totalSize = Number(response.headers.get("content-length"));
-        if (acceptRanges.includes("bytes") === false) {
+        finally {
+            if (response.body && response.bodyUsed === false && response.body.locked === false) {
+                await response.body.cancel();
+            }
+        }
+    }
+    catch (error) {
+        const parsed = asError(error);
+        return {
+            reason: `Resolve request failed: ${parsed.message}`,
+            status: null,
+            url: null
+        };
+    }
+}
+async function getRangeInfo({ accessToken, filePath, fileSize, modelSlug, revision }) {
+    const downloadUrlInfo = await getDownloadURL({
+        accessToken,
+        filePath,
+        modelSlug,
+        revision
+    });
+    if (!downloadUrlInfo.url) {
+        return {
+            reason: downloadUrlInfo.reason ?? "Resolve request failed",
+            supportsRanges: false,
+            totalSize: fileSize
+        };
+    }
+    try {
+        const response = await undiciExports.fetch(downloadUrlInfo.url, {
+            method: "GET",
+            headers: {
+                ...(shouldSendAuthHeader(downloadUrlInfo.url) ? getAuthHeaders(accessToken) : {}),
+                Range: "bytes=0-0"
+            }
+        });
+        try {
+            if (response.status === 206) {
+                const totalSize = parseContentRangeTotal(response.headers.get("content-range"));
+                return {
+                    reason: null,
+                    supportsRanges: true,
+                    totalSize: totalSize ?? fileSize
+                };
+            }
+            if (response.status === 200) {
+                return {
+                    reason: "Server returned 200 to range probe",
+                    supportsRanges: false,
+                    totalSize: fileSize
+                };
+            }
             return {
-                reason: "Server does not advertise range support",
+                reason: `Range probe failed with status ${response.status}`,
                 supportsRanges: false,
-                totalSize: Number.isFinite(totalSize) ? totalSize : fileSize
+                totalSize: fileSize
             };
         }
-        return {
-            reason: null,
-            supportsRanges: true,
-            totalSize: Number.isFinite(totalSize) ? totalSize : fileSize
-        };
+        finally {
+            await response.body?.cancel();
+        }
     }
     catch (error) {
         const parsed = asError(error);
         return {
-            reason: `HEAD request failed: ${parsed.message}`,
+            reason: `Range probe failed: ${parsed.message}`,
             supportsRanges: false,
             totalSize: fileSize
         };
@@ -96670,6 +96773,14 @@ async function downloadFileFull({ accessToken, filePath, fileSize, modelSlug, ta
             let progressTimeout = null;
             let lastPercentage = "0.0";
             let lastProgressBytes = 0;
+            const resetProgressTimeout = () => {
+                clearTimeout(progressTimeout);
+                progressTimeout = setTimeout(() => {
+                    input.destroy(new Error(`Timed out with no progress for ${DOWNLOAD_PROGRESS_TIMEOUT}ms while downloading ${filePath}. Last progress: ${lastPercentage}% (${lastProgressBytes} / ${fileSize})`));
+                }, DOWNLOAD_PROGRESS_TIMEOUT);
+            };
+            input.on("data", resetProgressTimeout);
+            resetProgressTimeout();
             meter.progress.on("progress", (totalBytes) => {
                 lastProgressBytes = totalBytes;
                 const percentComplete = ((totalBytes / fileSize) * 100).toFixed(1);
@@ -96677,10 +96788,6 @@ async function downloadFileFull({ accessToken, filePath, fileSize, modelSlug, ta
                     console.log(` => ${percentComplete}% (${totalBytes} / ${fileSize})`);
                 }
                 lastPercentage = percentComplete;
-                clearTimeout(progressTimeout);
-                progressTimeout = setTimeout(() => {
-                    input.destroy(new Error(`Timed out with no progress for ${DOWNLOAD_PROGRESS_TIMEOUT}ms while downloading ${filePath}. Last progress: ${lastPercentage}% (${lastProgressBytes} / ${fileSize})`));
-                }, DOWNLOAD_PROGRESS_TIMEOUT);
             });
             try {
                 await pipeline(input, meter, output);
@@ -96704,7 +96811,6 @@ async function downloadFileFull({ accessToken, filePath, fileSize, modelSlug, ta
 }
 async function downloadFileWithRange({ accessToken, filePath, fileSize, modelSlug, revision, targetDirectory }) {
     const { finalPath, partialPath } = getDownloadPaths({ filePath, targetDirectory });
-    const url = getResolveURL({ filePath, modelSlug, revision });
     let lastError = null;
     for (let attempt = 1; attempt <= DOWNLOAD_RETRY_ATTEMPTS_RANGE; attempt++) {
         let startOffset = 0;
@@ -96724,13 +96830,38 @@ async function downloadFileWithRange({ accessToken, filePath, fileSize, modelSlu
             const resumeLabel = startOffset > 0 ? `resuming at ${startOffset}` : "starting";
             console.log("Downloading:", filePath, fileSize, `(attempt ${attempt}, ${resumeLabel})`);
             await mkdir(dirname(finalPath), { recursive: true });
-            const response = await undiciExports.fetch(url, {
+            const downloadUrlInfo = await getDownloadURL({
+                accessToken,
+                filePath,
+                modelSlug,
+                revision
+            });
+            if (!downloadUrlInfo.url) {
+                if (downloadUrlInfo.status === 400) {
+                    throw new RangeNotSupportedError(downloadUrlInfo.reason ?? "Resolve request failed");
+                }
+                throw new Error(downloadUrlInfo.reason ?? "Resolve request failed");
+            }
+            const response = await undiciExports.fetch(downloadUrlInfo.url, {
                 headers: {
-                    ...getAuthHeaders(accessToken),
+                    ...(shouldSendAuthHeader(downloadUrlInfo.url)
+                        ? getAuthHeaders(accessToken)
+                        : {}),
                     Range: `bytes=${startOffset}-`
                 }
             });
             if (!response.ok) {
+                let responseBody = "<unavailable>";
+                try {
+                    responseBody = await response.text();
+                }
+                catch {
+                    responseBody = "<failed to read body>";
+                }
+                if (responseBody.length > 2000) {
+                    responseBody = responseBody.slice(0, 2000);
+                }
+                console.warn(`Range request failed response body for ${filePath}: ${responseBody}`);
                 throw new Error(`Range request failed with status ${response.status} for ${filePath}`);
             }
             if (startOffset > 0 && response.status !== 206) {
@@ -96745,6 +96876,14 @@ async function downloadFileWithRange({ accessToken, filePath, fileSize, modelSlu
             let progressTimeout = null;
             let lastPercentage = ((startOffset / fileSize) * 100).toFixed(1);
             let lastProgressBytes = startOffset;
+            const resetProgressTimeout = () => {
+                clearTimeout(progressTimeout);
+                progressTimeout = setTimeout(() => {
+                    input.destroy(new Error(`Timed out with no progress for ${DOWNLOAD_PROGRESS_TIMEOUT}ms while downloading ${filePath}. Last progress: ${lastPercentage}% (${lastProgressBytes} / ${fileSize})`));
+                }, DOWNLOAD_PROGRESS_TIMEOUT);
+            };
+            input.on("data", resetProgressTimeout);
+            resetProgressTimeout();
             meter.progress.on("progress", (totalBytes) => {
                 lastProgressBytes = startOffset + totalBytes;
                 const percentComplete = ((lastProgressBytes / fileSize) * 100).toFixed(1);
@@ -96752,10 +96891,6 @@ async function downloadFileWithRange({ accessToken, filePath, fileSize, modelSlu
                     console.log(` => ${percentComplete}% (${lastProgressBytes} / ${fileSize})`);
                 }
                 lastPercentage = percentComplete;
-                clearTimeout(progressTimeout);
-                progressTimeout = setTimeout(() => {
-                    input.destroy(new Error(`Timed out with no progress for ${DOWNLOAD_PROGRESS_TIMEOUT}ms while downloading ${filePath}. Last progress: ${lastPercentage}% (${lastProgressBytes} / ${fileSize})`));
-                }, DOWNLOAD_PROGRESS_TIMEOUT);
             });
             try {
                 await pipeline(input, meter, output);
@@ -96772,6 +96907,9 @@ async function downloadFileWithRange({ accessToken, filePath, fileSize, modelSlu
         }
         catch (error) {
             lastError = asError(error);
+            if (lastError instanceof RangeNotSupportedError) {
+                throw lastError;
+            }
             if (attempt < DOWNLOAD_RETRY_ATTEMPTS_RANGE) {
                 console.warn(`Retrying range download (${attempt}/${DOWNLOAD_RETRY_ATTEMPTS_RANGE}) for ${filePath} due to error: ${lastError.message}`);
                 continue;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.6.1",
+  "version": "1.6.2",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },