@juzi/file-box 1.7.19 → 1.8.0

package/src/misc.spec.ts

@@ -1,6 +1,8 @@
 #!/usr/bin/env -S node --no-warnings --loader ts-node/esm
 
 // tslint:disable:no-shadowed-variable
+import { createServer } from 'http'
+import type { AddressInfo } from 'net'
 import { test } from 'tstest'
 
 import {
@@ -27,14 +29,42 @@ test('dataUrl to base64', async t => {
 })
 
 test('httpHeadHeader', async t => {
-  const URL = 'https://github.com/huan/file-box/archive/v0.6.tar.gz'
-
-  const EXPECTED_HEADERS_KEY   = 'content-disposition'
-  const EXPECTED_HEADERS_VALUE = 'attachment; filename=file-box-0.6.tar.gz'
-
-  const headers = await httpHeadHeader(URL)
-
-  t.equal(headers[EXPECTED_HEADERS_KEY], EXPECTED_HEADERS_VALUE, 'should get the headers right')
+  /**
+   * 使用本地 server，避免依赖外网（CI/本地网络不稳定会导致 flaky）
+   * 同时覆盖：302 Location 为相对路径的跳转逻辑
+   */
+  const server = createServer((req, res) => {
+    if (req.url === '/redirect') {
+      res.writeHead(302, { Location: '/final' })
+      res.end()
+      return
+    }
+    if (req.url === '/final') {
+      res.writeHead(200, {
+        'Content-Disposition': 'attachment; filename=file-box-0.6.tar.gz',
+        'Content-Length': '0',
+      })
+      res.end()
+      return
+    }
+    res.writeHead(404)
+    res.end()
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  const headers = await httpHeadHeader(`${host}/redirect`)
+  t.equal(
+    headers['content-disposition'],
+    'attachment; filename=file-box-0.6.tar.gz',
+    'should get the headers right',
+  )
 })
 
 test('httpHeaderToFileName', async t => {
@@ -54,7 +84,33 @@ test('httpHeaderToFileName', async t => {
 })
 
 test('httpStream', async t => {
-  const URL = 'https://httpbin.org/headers'
+  const server = createServer((req, res) => {
+    const content = JSON.stringify({ headers: req.headers })
+
+    // Handle HEAD requests
+    if (req.method === 'HEAD') {
+      res.writeHead(200, {
+        'Content-Length': String(content.length),
+        'Content-Type': 'application/json',
+      })
+      res.end()
+      return
+    }
+
+    res.writeHead(200, {
+      'Content-Length': String(content.length),
+      'Content-Type': 'application/json',
+    })
+    res.end(content)
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
 
   const MOL_KEY = 'Mol'
   const MOL_VAL = '42'
@@ -62,19 +118,65 @@ test('httpStream', async t => {
   const headers = {} as { [idx: string]: string }
   headers[MOL_KEY] = MOL_VAL
 
-  const res = await httpStream(URL, headers)
+  const res = await httpStream(`${host}/headers`, headers)
 
   const buffer = await streamToBuffer(res)
   const obj = JSON.parse(buffer.toString())
-  t.equal(obj.headers[MOL_KEY], MOL_VAL, 'should send the header right')
+  // Node 会把 header name 规范成小写
+  t.equal(obj.headers[MOL_KEY.toLowerCase()], MOL_VAL, 'should send the header right')
 })
 
 test('httpStream in chunks', async (t) => {
-  const URL = 'https://media.w3.org/2010/05/sintel/trailer.mp4'
-  const res = await httpStream(URL)
-  let length = 0
-  for await (const chunk of res) {
-    length += chunk.length
-  }
-  t.equal(length, 4372373, 'should get data in chunks right')
+  const FILE_SIZE = 1024 * 1024 + 123 // > 默认 chunk size 触发分片逻辑
+  const content = Buffer.alloc(FILE_SIZE, 'A')
+
+  const server = createServer((req, res) => {
+    // HEAD：让 httpStream 判断是否支持 range + content-length
+    if (req.method === 'HEAD') {
+      res.writeHead(200, {
+        'Accept-Ranges': 'bytes',
+        'Content-Length': String(FILE_SIZE),
+      })
+      res.end()
+      return
+    }
+
+    const range = req.headers.range
+    if (range) {
+      const m = String(range).match(/bytes=(\d+)-(\d*)/)
+      if (!m) {
+        res.writeHead(416)
+        res.end()
+        return
+      }
+      const start = Number(m[1])
+      const end = m[2] ? Number(m[2]) : FILE_SIZE - 1
+      const chunk = content.subarray(start, end + 1)
+      res.writeHead(206, {
+        'Accept-Ranges': 'bytes',
+        'Content-Length': String(chunk.length),
+        'Content-Range': `bytes ${start}-${end}/${FILE_SIZE}`,
+      })
+      res.end(chunk)
+      return
+    }
+
+    res.writeHead(200, {
+      'Accept-Ranges': 'bytes',
+      'Content-Length': String(FILE_SIZE),
+    })
+    res.end(content)
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  const res = await httpStream(`${host}/file`)
+  const buffer = await streamToBuffer(res)
+  t.equal(buffer.length, FILE_SIZE, 'should get data in chunks right')
 })

package/src/misc.ts

@@ -2,20 +2,20 @@ import assert from 'assert'
 import { randomUUID } from 'crypto'
 import { once } from 'events'
 import { createReadStream, createWriteStream } from 'fs'
-import { rm } from 'fs/promises'
+import { rm, stat } from 'fs/promises'
 import http, { RequestOptions } from 'http'
 import https from 'https'
 import { HttpsProxyAgent } from 'https-proxy-agent'
 import { tmpdir } from 'os'
 import { join } from 'path'
 import type { Readable } from 'stream'
+import { Transform } from 'stream'
+import { pipeline } from 'stream/promises'
 import { URL } from 'url'
 
 import {
-  HTTP_CHUNK_SIZE,
   HTTP_REQUEST_TIMEOUT,
   HTTP_RESPONSE_TIMEOUT,
-  NO_SLICE_DOWN,
 } from './config.js'
 
 const protocolMap: {
@@ -25,6 +25,7 @@ const protocolMap: {
   'https:': { agent: https.globalAgent, request: https.request },
 }
 
+const noop = () => { }
 const unsupportedRangeDomains = new Set<string>()
 
 function getProtocol (protocol: string) {
@@ -53,8 +54,8 @@ export async function httpHeadHeader (url: string, headers: http.OutgoingHttpHea
     }
 
     const res = await fetch(url, {
-      method: 'HEAD',
       headers,
+      method: 'HEAD',
     }, proxyUrl)
     res.destroy()
 
@@ -71,7 +72,8 @@ export async function httpHeadHeader (url: string, headers: http.OutgoingHttpHea
       throw new Error('302 found but no location!')
     }
 
-    url = res.headers.location
+    // Location 可能是相对路径，需要以当前 url 作为 base 解析
+    url = new URL(res.headers.location, url).toString()
   }
 }
 
@@ -107,8 +109,14 @@ export async function httpStream (url: string, headers: http.OutgoingHttpHeaders
 
   const fileSize = Number(headHeaders['content-length'])
 
-  if (!unsupportedRangeDomains.has(hostname)! && !NO_SLICE_DOWN && headHeaders['accept-ranges'] === 'bytes' && fileSize > HTTP_CHUNK_SIZE) {
-    return await downloadFileInChunks(url, options, fileSize, HTTP_CHUNK_SIZE, proxyUrl)
+  // 运行时读取 env：方便测试/调用方动态调整
+  const noSliceDown = process.env['FILEBOX_NO_SLICE_DOWN'] === 'true'
+
+  // 检查服务器是否支持 range 请求
+  const supportsRange = headHeaders['accept-ranges'] === 'bytes'
+
+  if (!unsupportedRangeDomains.has(hostname) && !noSliceDown && supportsRange && fileSize > 0) {
+    return await downloadFileInChunks(url, options, proxyUrl)
   } else {
     return await fetch(url, options, proxyUrl)
   }
@@ -117,105 +125,283 @@ export async function httpStream (url: string, headers: http.OutgoingHttpHeaders
 async function fetch (url: string, options: http.RequestOptions, proxyUrl?: string): Promise<http.IncomingMessage> {
   const { protocol } = new URL(url)
   const { request, agent } = getProtocol(protocol)
+  const abortController = new AbortController()
+  const signal = abortController.signal
   const opts: http.RequestOptions = {
     agent,
     ...options,
+    signal,
   }
   setProxy(opts, proxyUrl)
+
   const req = request(url, opts)
-  req
-    .on('error', () => {
-      req.destroy()
+  let res: http.IncomingMessage | undefined
+
+  // 兜底：任何时候 req.error 都不会变成 uncaughtException
+  const onReqError = (err: unknown) => {
+    const error = err instanceof Error ? err : new Error(String(err))
+    // 统一用 abort 中止请求：signal 已挂在 req 上，并且我们会把 abort(reason) 桥接到 res.destroy(...)
+    abortController.abort(error)
+  }
+  req.on('error', noop)
+    .on('error', onReqError)
+    .once('close', () => {
+      // close 后禁用 request timeout，避免定时器晚到触发 destroy -> error 无人监听
+      try { req.setTimeout(0) } catch { }
+      req.off('error', onReqError)
+      req.off('error', noop)
     })
+    // request timeout：只用于“拿到 response 之前”（连接/握手/首包）
     .setTimeout(HTTP_REQUEST_TIMEOUT, () => {
-      req.emit('error', new Error(`FileBox: Http request timeout (${HTTP_REQUEST_TIMEOUT})!`))
+      // 已经拿到 response 时，不要再用 request timeout 误伤（会导致 aborted/ECONNRESET）
+      if (res) return
+      abortController.abort(new Error(`FileBox: Http request timeout (${HTTP_REQUEST_TIMEOUT})!`))
     })
     .end()
-  const responseEvents = await once(req, 'response')
-  const res = responseEvents[0] as http.IncomingMessage
-  res
-    .on('error', () => {
-      res.destroy()
+
+  try {
+    const responseEvent = await once(req, 'response', { signal })
+    res = responseEvent[0] as http.IncomingMessage
+    // response 到来后清掉 request timeout，避免误伤长下载导致 aborted/ECONNRESET
+    try { req.setTimeout(0) } catch {}
+    // 必须尽早挂，避免 “response 刚到就 error/abort” 的竞态导致 uncaughtException
+    res.on('error', noop)
+    signal.throwIfAborted()
+  } catch (e) {
+    // once(...) 被 signal abort 时通常会抛 AbortError；优先抛出 abort(reason) 的真实原因
+    const reason = signal.reason as unknown
+    const err = reason instanceof Error
+      ? reason
+      : (e instanceof Error ? e : new Error(String(e)))
+    // 失败时尽量主动清理，避免 socket 悬挂（destroy 重复调用是安全的）
+    try { res?.destroy(err) } catch {}
+    try { req.destroy(err) } catch {}
+    throw err
+  }
+
+  const onAbort = () => {
+    const reason = signal.reason as unknown
+    res?.destroy(reason instanceof Error ? reason : new Error(String(reason)))
+  }
+  signal.addEventListener('abort', onAbort, { once: true })
+  res!
+    .once('end', () => { try { res!.setTimeout(0) } catch { } })
+    .once('close', () => {
+      // close 时做清理/兜底判断（尽力而为）
+      try { res!.setTimeout(0) } catch { }
+      if (!res!.complete && !res!.destroyed) {
+        // 有些场景不会 emit 'aborted'，用 close + complete 兜底一次
+        res!.destroy(new Error('FileBox: Http response aborted!'))
+      }
+      signal.removeEventListener('abort', onAbort)
+      res!.off('error', noop)
     })
-  if (res.socket) {
-    res.setTimeout(HTTP_RESPONSE_TIMEOUT, () => {
-      res.emit('error', new Error(`FileBox: Http response timeout (${HTTP_RESPONSE_TIMEOUT})!`))
+    .setTimeout(HTTP_RESPONSE_TIMEOUT, () => {
+      abortController.abort(new Error(`FileBox: Http response timeout (${HTTP_RESPONSE_TIMEOUT})!`))
     })
-  }
-  return res
+  return res!
+}
+
+function createSkipTransform (skipBytes: number): Transform {
+  let skipped = 0
+  return new Transform({
+    transform (chunk, _encoding, callback) {
+      if (skipped < skipBytes) {
+        const remaining = skipBytes - skipped
+        if (chunk.length <= remaining) {
+          // 整个 chunk 都需要跳过
+          skipped += chunk.length
+          callback()
+          return
+        } else {
+          // 跳过部分 chunk
+          skipped = skipBytes
+          callback(null, chunk.subarray(remaining))
+          return
+        }
+      }
+      // 已经跳过足够的字节，直接传递
+      callback(null, chunk)
+    },
+  })
 }
 
 async function downloadFileInChunks (
   url: string,
   options: http.RequestOptions,
-  fileSize: number,
-  chunkSize = HTTP_CHUNK_SIZE,
   proxyUrl?: string,
 ): Promise<Readable> {
   const tmpFile = join(tmpdir(), `filebox-${randomUUID()}`)
   const writeStream = createWriteStream(tmpFile)
+  const writeAbortController = new AbortController()
+  const signal = writeAbortController.signal
+  const onWriteError = (err: unknown) => {
+    writeAbortController.abort(err instanceof Error ? err : new Error(String(err)))
+  }
+  writeStream.once('error', onWriteError)
   const allowStatusCode = [ 200, 206 ]
   const requestBaseOptions: http.RequestOptions = {
     headers: {},
     ...options,
   }
-  let chunkSeq = 0
+  // 预期文件大小（初始为 null，从首次 206 响应中获取）
+  let expectedTotal: number | null = null
   let start = 0
-  let end = 0
   let downSize = 0
   let retries = 3
+  // 标识是否需要回退到非分片下载
+  let shouldFallback = false
+
+  do {
+    // 每次循环前检查文件实际大小，作为真实的下载进度
+    // 这样在重试时可以从实际写入的位置继续，避免数据重复
+    try {
+      const fileStats = await stat(tmpFile)
+      const actualSize = fileStats.size
+      if (actualSize > downSize) {
+        // 文件实际大小比记录的大，说明之前有部分写入
+        downSize = actualSize
+        start = actualSize
+      }
+    } catch (error) {
+      // 文件不存在或无法访问，使用当前的 downSize
+    }
 
-  while (downSize < fileSize) {
-    end = Math.min(start + chunkSize, fileSize - 1)
-    const range = `bytes=${start}-${end}`
+    const range = `bytes=${start}-`
     const requestOptions = Object.assign({}, requestBaseOptions)
     assert(requestOptions.headers, 'Errors that should not happen: Invalid headers')
     ;(requestOptions.headers as http.OutgoingHttpHeaders)['Range'] = range
 
+    // 每次请求创建独立的 AbortController 来管理当前请求的生命周期
+    const requestAbortController = new AbortController()
+    requestOptions.signal = requestAbortController.signal
+
     try {
       const res = await fetch(url, requestOptions, proxyUrl)
       if (res.statusCode === 416) {
-        unsupportedRangeDomains.add(new URL(url).hostname)
         // 某些云服务商对分片下载的支持可能不规范，需要保留一个回退的方式
-        writeStream.close()
-        await rm(tmpFile, { force: true })
-        return await fetch(url, requestBaseOptions, proxyUrl)
+        shouldFallback = true
+        break
       }
       assert(allowStatusCode.includes(res.statusCode ?? 0), `Request failed with status code ${res.statusCode}`)
-      assert(Number(res.headers['content-length']) > 0, 'Server returned 0 bytes of data')
-      try {
-        const { total } = parseContentRange(res.headers['content-range'] ?? '')
-        if (total > 0 && total < fileSize) {
+      const contentLength = Number(res.headers['content-length'])
+      assert(contentLength > 0, 'Server returned 0 bytes of data')
+
+      // 206: 部分内容，继续分片下载
+      // 200: 完整内容，服务器不支持 range 或返回全部数据
+      if (res.statusCode === 206) {
+        // 206 响应必须包含有效的 Content-Range 头（RFC 7233）
+        const contentRange = res.headers['content-range']
+        if (!contentRange) {
+          // Content-Range 缺失，服务器不规范，回退到非分片下载
+          shouldFallback = true
+          break
+        }
+
+        let end: number
+        let total: number
+        let actualStart: number
+        try {
+          const parsed = parseContentRange(contentRange)
+          actualStart = parsed.start
+          end = parsed.end
+          total = parsed.total
+        } catch (error) {
+          // Content-Range 格式错误，服务器不规范，回退到非分片下载
+          shouldFallback = true
+          break
+        }
+
+        if (expectedTotal === null) {
+          // 首次获得文件总大小
           // 某些云服务商（如腾讯云）在 head 方法中返回的 size 是原图大小，但下载时返回的是压缩后的图片，会比原图小。
-          // 这种在首次下载时虽然请求了原图大小的范围，可能比缩略图大，但会一次性返回完整的原图，而不是报错 416，通过修正 fileSize 跳出循环即可。
-          fileSize = total
+          // 这种在首次下载时虽然请求了原图大小的范围，可能比缩略图大，但会一次性返回完整的原图，而不是报错 416，通过修正 expectedTotal 跳出循环即可。
+          expectedTotal = total
+        } else if (total !== expectedTotal) {
+          // 服务器返回的文件总大小出现了变化
+          throw new Error(`File size mismatch: expected ${expectedTotal}, but server returned ${total}`)
+        }
+
+        // 验证服务器返回的范围是否与请求匹配
+        if (actualStart !== start) {
+          if (actualStart > start) {
+            // 服务器跳过了部分数据，这是严重错误
+            throw new Error(`Range mismatch: requested start=${start}, but server returned start=${actualStart} (gap detected)`)
+          } else {
+            // actualStart < start: 服务器返回了重叠数据，需要跳过前面的字节
+            const skipBytes = start - actualStart
+            const skipTransform = createSkipTransform(skipBytes)
+            await pipeline(res, skipTransform, writeStream, { end: false, signal })
+            // 更新进度时使用我们请求的范围，而不是服务器返回的范围
+            downSize += end - actualStart + 1 - skipBytes
+            start = downSize
+            retries = 3 // 成功后重置重试次数
+            continue
+          }
+        }
+        // 使用 pipeline，但不关闭 writeStream（继续下载下一个分片）
+        await pipeline(res, writeStream, { end: false, signal })
+        // pipeline 成功后才更新下载进度
+        // end 是最后一个字节的索引，下次从 end+1 开始
+        downSize += end - start + 1
+        start = downSize
+      } else {
+        // 200: 服务器返回完整文件，不支持 range
+        if (start > 0) {
+          // 中途收到 200，服务器停止支持 range，标记并回退到普通下载
+          shouldFallback = true
+          break
         }
-      } catch (error) {}
-      for await (const chunk of res) {
-        assert(Buffer.isBuffer(chunk))
-        downSize += chunk.length
-        writeStream.write(chunk)
+        // 首次请求返回 200，正常处理
+        await pipeline(res, writeStream, { signal })
+        downSize = contentLength
+        break
       }
-      res.destroy()
+      // 成功后重置重试次数
+      retries = 3
     } catch (error) {
-      const err = error as Error
+      const err = error instanceof Error ? error : new Error(String(error))
       if (--retries <= 0) {
-        writeStream.close()
-        void rm(tmpFile, { force: true })
+        writeStream.destroy()
+        await rm(tmpFile, { force: true })
         throw new Error(`Download file with chunk failed! ${err.message}`, { cause: err })
       }
+      // 失败后等待一小段时间再重试
+      await new Promise(resolve => setTimeout(resolve, 100))
+    } finally {
+      // 确保请求被清理（成功时也需要 abort 以释放资源）
+      requestAbortController.abort()
+    }
+  } while (expectedTotal === null || downSize < expectedTotal)
+
+  // 统一处理回退到非分片下载的情况
+  if (shouldFallback) {
+    unsupportedRangeDomains.add(new URL(url).hostname)
+    writeStream.destroy()
+    try {
+      await once(writeStream, 'close', { signal })
+    } catch {}
+    await rm(tmpFile, { force: true })
+    return await fetch(url, requestBaseOptions, proxyUrl)
+  }
+
+  writeStream.end()
+  try {
+    await once(writeStream, 'finish', { signal })
+  } catch (e) {
+    const reason = signal.reason as unknown
+    if (reason instanceof Error) {
+      throw reason
     }
-    chunkSeq++
-    start = downSize
+    throw e
+  } finally {
+    writeStream.off('error', onWriteError)
   }
-  writeStream.close()
 
   const readStream = createReadStream(tmpFile)
-  readStream
-    .once('end', () => readStream.close())
-    .once('close', () => {
-      void rm(tmpFile, { force: true })
-    })
+  readStream.once('close', () => {
+    rm(tmpFile, { force: true }).catch(() => {})
+  })
   return readStream
 }
 
@@ -234,15 +420,14 @@ function setProxy (options: RequestOptions, proxyUrl?: string): void {
   }
 }
 
-
 function parseContentRange (contentRange: string): { start: number, end: number, total: number } {
   const matches = contentRange.match(/bytes (\d+)-(\d+)\/(\d+)/)
   if (!matches) {
     throw new Error('Invalid content range')
   }
   return {
-    start: Number(matches[1]),
     end: Number(matches[2]),
+    start: Number(matches[1]),
     total: Number(matches[3]),
   }
 }

package/src/version.ts

@@ -1,4 +1,4 @@
 /**
  * This file was auto generated from scripts/generate-version.sh
  */
-export const VERSION: string = '1.7.19'
+export const VERSION: string = '1.8.0'