@juzi/file-box 1.7.20 → 1.8.1

package/src/misc.ts

@@ -2,20 +2,19 @@ import assert from 'assert'
 import { randomUUID } from 'crypto'
 import { once } from 'events'
 import { createReadStream, createWriteStream } from 'fs'
-import { rm } from 'fs/promises'
+import { rm, stat } from 'fs/promises'
 import http, { RequestOptions } from 'http'
 import https from 'https'
 import { HttpsProxyAgent } from 'https-proxy-agent'
 import { tmpdir } from 'os'
 import { join } from 'path'
 import type { Readable } from 'stream'
+import { Transform } from 'stream'
+import { pipeline } from 'stream/promises'
+import { setTimeout } from 'timers/promises'
 import { URL } from 'url'
 
-import {
-  HTTP_CHUNK_SIZE,
-  HTTP_REQUEST_TIMEOUT,
-  HTTP_RESPONSE_TIMEOUT,
-} from './config.js'
+import { CONFIG } from './config.js'
 
 const protocolMap: {
   [key: string]: { agent: http.Agent; request: typeof http.request }
@@ -27,6 +26,16 @@ const protocolMap: {
 const noop = () => { }
 const unsupportedRangeDomains = new Set<string>()
 
+// 自定义 Error：标记需要回退到非分片下载
+class FallbackError extends Error {
+
+  constructor (reason: string) {
+    super(`Fallback required: ${reason}`)
+    this.name = 'FallbackError'
+  }
+
+}
+
 function getProtocol (protocol: string) {
   assert(protocolMap[protocol], new Error('unknown protocol: ' + protocol))
   return protocolMap[protocol]!
@@ -65,8 +74,6 @@ export async function httpHeadHeader (url: string, headers: http.OutgoingHttpHea
       return res.headers
     }
 
-    // console.log('302 found for ' + url)
-
     if (!res.headers.location) {
       throw new Error('302 found but no location!')
     }
@@ -98,7 +105,7 @@ export async function httpStream (url: string, headers: http.OutgoingHttpHeaders
   if (headHeaders.location) {
     url = headHeaders.location
   }
-  const { protocol, hostname } = new URL(url)
+  const { protocol, hostname, port } = new URL(url)
   getProtocol(protocol)
 
   const options: http.RequestOptions = {
@@ -106,17 +113,17 @@ export async function httpStream (url: string, headers: http.OutgoingHttpHeaders
     method: 'GET',
   }
 
-  const fileSize = Number(headHeaders['content-length'])
-
-  // 运行时读取 env：方便测试/调用方动态调整
-  const noSliceDown = process.env['FILEBOX_NO_SLICE_DOWN'] === 'true'
-  const chunkSize = Number(process.env['FILEBOX_HTTP_CHUNK_SIZE']) || HTTP_CHUNK_SIZE
-
-  if (!unsupportedRangeDomains.has(hostname) && !noSliceDown && headHeaders['accept-ranges'] === 'bytes' && fileSize > chunkSize) {
-    return await downloadFileInChunks(url, options, fileSize, chunkSize, proxyUrl)
-  } else {
-    return await fetch(url, options, proxyUrl)
-  }
+  // 使用 hostname:port 作为域名标识，避免不同端口的服务互相影响
+  const defaultPort = protocol === 'https:' ? '443' : '80'
+  const hostKey = `${hostname}:${port || defaultPort}`
+
+  // 直接尝试分片下载，不检查 Accept-Ranges 和 fileSize
+  // 原因：
+  // 1. 有些服务器 HEAD 不返回 Accept-Ranges 但实际支持分片
+  // 2. 有些服务器 HEAD 返回 fileSize=0 但实际支持分片
+  // downloadFileInChunks 内部有完善的回退机制处理不支持的情况
+  const result = await downloadFileInChunks(url, options, proxyUrl, hostKey)
+  return result
 }
 
 async function fetch (url: string, options: http.RequestOptions, proxyUrl?: string): Promise<http.IncomingMessage> {
@@ -149,10 +156,10 @@ async function fetch (url: string, options: http.RequestOptions, proxyUrl?: stri
       req.off('error', noop)
     })
     // request timeout：只用于“拿到 response 之前”（连接/握手/首包）
-    .setTimeout(HTTP_REQUEST_TIMEOUT, () => {
+    .setTimeout(CONFIG.HTTP_REQUEST_TIMEOUT, () => {
       // 已经拿到 response 时，不要再用 request timeout 误伤（会导致 aborted/ECONNRESET）
       if (res) return
-      abortController.abort(new Error(`FileBox: Http request timeout (${HTTP_REQUEST_TIMEOUT})!`))
+      abortController.abort(new Error(`FileBox: Http request timeout (${CONFIG.HTTP_REQUEST_TIMEOUT})!`))
     })
     .end()
 
@@ -165,7 +172,6 @@ async function fetch (url: string, options: http.RequestOptions, proxyUrl?: stri
     res.on('error', noop)
     signal.throwIfAborted()
   } catch (e) {
-    // once(...) 被 signal abort 时通常会抛 AbortError；优先抛出 abort(reason) 的真实原因
     const reason = signal.reason as unknown
     const err = reason instanceof Error
       ? reason
@@ -193,21 +199,44 @@ async function fetch (url: string, options: http.RequestOptions, proxyUrl?: stri
       signal.removeEventListener('abort', onAbort)
       res!.off('error', noop)
     })
-    .setTimeout(HTTP_RESPONSE_TIMEOUT, () => {
-      abortController.abort(new Error(`FileBox: Http response timeout (${HTTP_RESPONSE_TIMEOUT})!`))
+    .setTimeout(CONFIG.HTTP_RESPONSE_TIMEOUT, () => {
+      abortController.abort(new Error(`FileBox: Http response timeout (${CONFIG.HTTP_RESPONSE_TIMEOUT})!`))
     })
   return res!
 }
 
+function createSkipTransform (skipBytes: number): Transform {
+  let skipped = 0
+  return new Transform({
+    transform (chunk, _encoding, callback) {
+      if (skipped < skipBytes) {
+        const remaining = skipBytes - skipped
+        if (chunk.length <= remaining) {
+          // 整个 chunk 都需要跳过
+          skipped += chunk.length
+          callback()
+          return
+        } else {
+          // 跳过部分 chunk
+          skipped = skipBytes
+          callback(null, chunk.subarray(remaining))
+          return
+        }
+      }
+      // 已经跳过足够的字节，直接传递
+      callback(null, chunk)
+    },
+  })
+}
+
 async function downloadFileInChunks (
   url: string,
   options: http.RequestOptions,
-  fileSize: number,
-  chunkSize = HTTP_CHUNK_SIZE,
-  proxyUrl?: string,
+  proxyUrl: string | undefined,
+  hostname: string,
 ): Promise<Readable> {
   const tmpFile = join(tmpdir(), `filebox-${randomUUID()}`)
-  const writeStream = createWriteStream(tmpFile)
+  let writeStream = createWriteStream(tmpFile)
   const writeAbortController = new AbortController()
   const signal = writeAbortController.signal
   const onWriteError = (err: unknown) => {
@@ -219,78 +248,174 @@ async function downloadFileInChunks (
     headers: {},
     ...options,
   }
-  let chunkSeq = 0
+  // 预期文件大小（初始为 null，从首次 206 响应中获取）
+  let expectedTotal: number | null = null
   let start = 0
-  let end = 0
   let downSize = 0
   let retries = 3
+  // 控制是否使用 Range 请求（根据域名黑名单初始化）
+  let useRange = !unsupportedRangeDomains.has(hostname)
+  let useChunked = false
+
+  do {
+    // 每次循环前检查文件实际大小，作为真实的下载进度
+    // 这样在重试时可以从实际写入的位置继续，避免数据重复
+    const fileStats = await stat(tmpFile).then(stats => stats.size).catch(() => 0)
+    if (fileStats !== downSize) {
+      // 文件实际大小与记录的不一致，使用实际大小
+      downSize = fileStats
+      start = fileStats
+    }
 
-  while (downSize < fileSize) {
-    end = Math.min(start + chunkSize, fileSize - 1)
-    const range = `bytes=${start}-${end}`
     const requestOptions = Object.assign({}, requestBaseOptions)
     assert(requestOptions.headers, 'Errors that should not happen: Invalid headers')
-    ;(requestOptions.headers as http.OutgoingHttpHeaders)['Range'] = range
+    const headers = requestOptions.headers as http.OutgoingHttpHeaders
+
+    // 根据 useRange flag 决定是否添加 Range header
+    if (useRange) {
+      const range = `bytes=${start}-`
+      headers['Range'] = range
+    } else {
+      delete headers['Range']
+    }
 
+    let res: http.IncomingMessage
     try {
-      const res = await fetch(url, requestOptions, proxyUrl)
+      res = await fetch(url, requestOptions, proxyUrl)
       if (res.statusCode === 416) {
-        unsupportedRangeDomains.add(new URL(url).hostname)
-        // 某些云服务商对分片下载的支持可能不规范，需要保留一个回退的方式
-        writeStream.destroy()
-        try {
-          await once(writeStream, 'close', { signal })
-        } catch {}
-        await rm(tmpFile, { force: true })
-        return await fetch(url, requestBaseOptions, proxyUrl)
+        // 416: Range Not Satisfiable，服务器不支持此范围或文件大小不匹配
+        throw new FallbackError('416 Range Not Satisfiable')
       }
       assert(allowStatusCode.includes(res.statusCode ?? 0), `Request failed with status code ${res.statusCode}`)
-      assert(Number(res.headers['content-length']) > 0, 'Server returned 0 bytes of data')
-      try {
-        const { total } = parseContentRange(res.headers['content-range'] ?? '')
-        if (total > 0 && total < fileSize) {
+      const contentLength = Number(res.headers['content-length']) || 0
+      assert(contentLength >= 0, `Server returned ${contentLength} bytes of data`)
+
+      // 206: 部分内容，继续分片下载
+      // 200: 完整内容，服务器不支持 range 或返回全部数据
+      if (res.statusCode === 206) {
+        // 206 响应必须包含有效的 Content-Range 头（RFC 7233）
+        const contentRange = res.headers['content-range']
+        if (!contentRange) {
+          // Content-Range 缺失，服务器不规范，回退到非分片下载
+          throw new FallbackError('Missing Content-Range header')
+        }
+
+        let end: number
+        let total: number
+        let actualStart: number
+        try {
+          const parsed = parseContentRange(contentRange)
+          actualStart = parsed.start
+          end = parsed.end
+          total = parsed.total
+        } catch (error) {
+          // Content-Range 格式错误，服务器不规范，回退到非分片下载
+          throw new FallbackError(`Invalid Content-Range: ${contentRange}`)
+        }
+
+        if (expectedTotal === null) {
+          // 首次获得文件总大小
           // 某些云服务商（如腾讯云）在 head 方法中返回的 size 是原图大小，但下载时返回的是压缩后的图片，会比原图小。
-          // 这种在首次下载时虽然请求了原图大小的范围，可能比缩略图大，但会一次性返回完整的原图，而不是报错 416，通过修正 fileSize 跳出循环即可。
-          fileSize = total
+          // 这种在首次下载时虽然请求了原图大小的范围，可能比缩略图大，但会一次性返回完整的原图，而不是报错 416，通过修正 expectedTotal 跳出循环即可。
+          expectedTotal = total
+        } else if (total !== expectedTotal) {
+          // 服务器返回的文件总大小出现了变化
+          throw new Error(`File size mismatch: expected ${expectedTotal}, but server returned ${total}`)
         }
-      } catch (error) {}
-      for await (const chunk of res) {
-        assert(Buffer.isBuffer(chunk))
-        downSize += chunk.length
-        if (!writeStream.write(chunk)) {
-          try {
-            await once(writeStream, 'drain', { signal })
-          } catch (e) {
-            const reason = signal.reason as unknown
-            throw reason instanceof Error ? reason : (e as Error)
+
+        // 标记使用了分片下载
+        useChunked = true
+
+        // 验证服务器返回的范围是否与请求匹配
+        if (actualStart !== start) {
+          if (actualStart > start) {
+            // 服务器跳过了部分数据，这是严重错误
+            throw new Error(`Range mismatch: requested start=${start}, but server returned start=${actualStart} (gap detected)`)
+          } else {
+            // actualStart < start: 服务器返回了重叠数据，需要跳过前面的字节
+            const skipBytes = start - actualStart
+            const skipTransform = createSkipTransform(skipBytes)
+            await pipeline(res, skipTransform, writeStream, { end: false, signal })
+            // 更新进度时使用我们请求的范围，而不是服务器返回的范围
+            downSize += end - actualStart + 1 - skipBytes
+            start = downSize
+            retries = 3 // 成功后重置重试次数
+            continue
           }
         }
+        // 使用 pipeline，但不关闭 writeStream（继续下载下一个分片）
+        await pipeline(res, writeStream, { end: false, signal })
+        // pipeline 成功后才更新下载进度
+        // end 是最后一个字节的索引，下次从 end+1 开始
+        downSize += end - start + 1
+        start = downSize
+      } else if (res.statusCode === 200) {
+        // 200: 服务器返回完整文件
+        if (useChunked || start > 0) {
+          // 之前以分片模式下载过数据
+          writeStream.destroy()
+          await rm(tmpFile, { force: true }).catch(() => {})
+          writeStream = createWriteStream(tmpFile)
+          writeStream.on('error', onWriteError)
+          start = 0
+          downSize = 0
+        }
+
+        // 处理完整文件响应
+        expectedTotal = contentLength
+        await pipeline(res, writeStream, { end: false, signal })
+        downSize = contentLength
+        break
+      } else {
+        throw new Error(`Unexpected status code: ${res.statusCode}`)
       }
-      res.destroy()
+      // 成功后重置重试次数
+      retries = 3
     } catch (error) {
+      if (error instanceof FallbackError) {
+        // 回退逻辑：记录域名、重置状态，在下次循环中以非 range 模式请求
+        unsupportedRangeDomains.add(hostname)
+
+        // 关闭当前写入流
+        writeStream.destroy()
+        await rm(tmpFile, { force: true }).catch(() => {})
+
+        // 检查是否已经是非 Range 模式，避免无限回退
+        if (!useRange) {
+          // 已经是非 Range 模式还失败，无法继续
+          throw new Error(`Download failed even in non-chunked mode: ${(error as Error).message}`)
+        }
+
+        writeStream = createWriteStream(tmpFile)
+        writeStream.once('error', onWriteError)
+
+        // 重置所有状态
+        expectedTotal = null
+        downSize = 0
+        start = 0
+        useChunked = false
+        useRange = false
+        retries = 3
+        continue
+      }
+
+      // 普通错误：重试
       const err = error instanceof Error ? error : new Error(String(error))
       if (--retries <= 0) {
         writeStream.destroy()
-        void rm(tmpFile, { force: true })
+        await rm(tmpFile, { force: true }).catch(() => {})
         throw new Error(`Download file with chunk failed! ${err.message}`, { cause: err })
       }
+      // 失败后等待一小段时间再重试
+      await setTimeout(100)
     }
-    chunkSeq++
-    start = downSize
-  }
+  } while (expectedTotal === null || downSize < expectedTotal)
 
-  writeStream.end()
-  try {
+  if (!writeStream.destroyed && !writeStream.writableFinished) {
+    writeStream.end()
     await once(writeStream, 'finish', { signal })
-  } catch (e) {
-    const reason = signal.reason as unknown
-    if (reason instanceof Error) {
-      throw reason
-    }
-    throw e
-  } finally {
-    writeStream.off('error', onWriteError)
   }
+  writeStream.off('error', onWriteError)
 
   const readStream = createReadStream(tmpFile)
   readStream.once('close', () => {

package/src/version.ts

@@ -1,4 +1,4 @@
 /**
  * This file was auto generated from scripts/generate-version.sh
  */
-export const VERSION: string = '1.7.20'
+export const VERSION: string = '1.8.1'