@juzi/file-box 1.7.18 → 1.7.20

package/src/file-box.spec.ts

@@ -2,13 +2,15 @@
 
 import 'reflect-metadata'
 
-import assert                       from 'assert'
-import { PassThrough, Readable }    from 'stream'
-import { test, sinon }              from 'tstest'
-import { instanceToClass }          from 'clone-class'
+import assert from 'assert'
+import { instanceToClass } from 'clone-class'
+import { createServer } from 'http'
+import type { AddressInfo } from 'net'
+import { PassThrough, Readable } from 'stream'
+import { sinon, test } from 'tstest'
 
-import { FileBox }      from './file-box.js'
-import { FileBoxType }  from './file-box.type.js'
+import { FileBox } from './file-box.js'
+import { FileBoxType } from './file-box.type.js'
 
 const requiredMetadataKey = Symbol('required')
 
@@ -113,7 +115,32 @@ test('syncRemote()', async t => {
 
   }
 
-  const URL = 'http://httpbin.org/response-headers?Content-Disposition=attachment;%20filename%3d%22test.txt%22&filename=test.txt'
+  /**
+   * 使用本地 server，避免依赖外网（httpbin 不稳定/可能被墙/会返回 html）
+   */
+  const server = createServer((req, res) => {
+    if (req.method === 'HEAD' && req.url?.startsWith('/response-headers')) {
+      res.writeHead(200, {
+        'Content-Disposition': 'attachment; filename="test.txt"',
+        'Content-Length': '159',
+        'Content-Type': 'application/json',
+      })
+      res.end()
+      return
+    }
+    res.writeHead(404)
+    res.end()
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  const URL = `${host}/response-headers?Content-Disposition=attachment;%20filename%3d%22test.txt%22&filename=test.txt`
 
   const EXPECTED_NAME_FROM_URL    = 'response-headers'
   const EXPECTED_TYPE_FROM_URL    = 'application/unknown'

package/src/file-box.ts

@@ -385,10 +385,10 @@ class FileBox implements Pipeable, FileBoxInterface {
 
       case FileBoxType.Url:
         fileBox = this.fromUrl(obj.url, {
+          headers: obj.headers,
           md5: obj.md5,
           name: obj.name,
           size: obj.size,
-          headers: obj.headers,
         })
         break
 
@@ -510,12 +510,12 @@ class FileBox implements Pipeable, FileBoxInterface {
   private readonly qrCode?    : string
   private readonly uuid?      : string
 
-  get url() {
+  get url () {
     return this.remoteUrl
   }
 
   private proxyUrl?: string
-  useProxyUrl(url?: string) {
+  useProxyUrl (url?: string) {
     this.proxyUrl = url
   }
 
@@ -618,7 +618,7 @@ class FileBox implements Pipeable, FileBoxInterface {
 
   async ready (): Promise<void> {
     let tryCount = 0
-    while (1) {
+    while (true) {
       try {
         switch (this.type) {
           case FileBoxType.Url:
@@ -636,7 +636,7 @@ class FileBox implements Pipeable, FileBoxInterface {
         }
         break
       } catch (e) {
-        tryCount ++
+        tryCount++
         if (tryCount >= READY_RETRY) {
           throw e
         }

package/src/misc.spec.ts

@@ -1,6 +1,8 @@
 #!/usr/bin/env -S node --no-warnings --loader ts-node/esm
 
 // tslint:disable:no-shadowed-variable
+import { createServer } from 'http'
+import type { AddressInfo } from 'net'
 import { test } from 'tstest'
 
 import {
@@ -27,14 +29,42 @@ test('dataUrl to base64', async t => {
 })
 
 test('httpHeadHeader', async t => {
-  const URL = 'https://github.com/huan/file-box/archive/v0.6.tar.gz'
-
-  const EXPECTED_HEADERS_KEY   = 'content-disposition'
-  const EXPECTED_HEADERS_VALUE = 'attachment; filename=file-box-0.6.tar.gz'
-
-  const headers = await httpHeadHeader(URL)
-
-  t.equal(headers[EXPECTED_HEADERS_KEY], EXPECTED_HEADERS_VALUE, 'should get the headers right')
+  /**
+   * 使用本地 server，避免依赖外网（CI/本地网络不稳定会导致 flaky）
+   * 同时覆盖：302 Location 为相对路径的跳转逻辑
+   */
+  const server = createServer((req, res) => {
+    if (req.url === '/redirect') {
+      res.writeHead(302, { Location: '/final' })
+      res.end()
+      return
+    }
+    if (req.url === '/final') {
+      res.writeHead(200, {
+        'Content-Disposition': 'attachment; filename=file-box-0.6.tar.gz',
+        'Content-Length': '0',
+      })
+      res.end()
+      return
+    }
+    res.writeHead(404)
+    res.end()
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  const headers = await httpHeadHeader(`${host}/redirect`)
+  t.equal(
+    headers['content-disposition'],
+    'attachment; filename=file-box-0.6.tar.gz',
+    'should get the headers right',
+  )
 })
 
 test('httpHeaderToFileName', async t => {
@@ -54,7 +84,18 @@ test('httpHeaderToFileName', async t => {
 })
 
 test('httpStream', async t => {
-  const URL = 'https://httpbin.org/headers'
+  const server = createServer((req, res) => {
+    res.writeHead(200, { 'Content-Type': 'application/json' })
+    res.end(JSON.stringify({ headers: req.headers }))
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
 
   const MOL_KEY = 'Mol'
   const MOL_VAL = '42'
@@ -62,19 +103,72 @@ test('httpStream', async t => {
   const headers = {} as { [idx: string]: string }
   headers[MOL_KEY] = MOL_VAL
 
-  const res = await httpStream(URL, headers)
+  const res = await httpStream(`${host}/headers`, headers)
 
   const buffer = await streamToBuffer(res)
   const obj = JSON.parse(buffer.toString())
-  t.equal(obj.headers[MOL_KEY], MOL_VAL, 'should send the header right')
+  // Node 会把 header name 规范成小写
+  t.equal(obj.headers[MOL_KEY.toLowerCase()], MOL_VAL, 'should send the header right')
 })
 
 test('httpStream in chunks', async (t) => {
-  const URL = 'https://media.w3.org/2010/05/sintel/trailer.mp4'
-  const res = await httpStream(URL)
-  let length = 0
-  for await (const chunk of res) {
-    length += chunk.length
+  const FILE_SIZE = 1024 * 1024 + 123 // > 默认 chunk size 触发分片逻辑
+  const content = Buffer.alloc(FILE_SIZE, 'A')
+
+  const server = createServer((req, res) => {
+    // HEAD：让 httpStream 判断是否支持 range + content-length
+    if (req.method === 'HEAD') {
+      res.writeHead(200, {
+        'Accept-Ranges': 'bytes',
+        'Content-Length': String(FILE_SIZE),
+      })
+      res.end()
+      return
+    }
+
+    const range = req.headers.range
+    if (range) {
+      const m = String(range).match(/bytes=(\d+)-(\d+)/)
+      if (!m) {
+        res.writeHead(416)
+        res.end()
+        return
+      }
+      const start = Number(m[1])
+      const end = Number(m[2])
+      const chunk = content.subarray(start, end + 1)
+      res.writeHead(206, {
+        'Accept-Ranges': 'bytes',
+        'Content-Length': String(chunk.length),
+        'Content-Range': `bytes ${start}-${end}/${FILE_SIZE}`,
+      })
+      res.end(chunk)
+      return
+    }
+
+    res.writeHead(200, {
+      'Accept-Ranges': 'bytes',
+      'Content-Length': String(FILE_SIZE),
+    })
+    res.end(content)
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  const originalChunkSize = process.env['FILEBOX_HTTP_CHUNK_SIZE']
+  process.env['FILEBOX_HTTP_CHUNK_SIZE'] = String(256 * 1024)
+  try {
+    const res = await httpStream(`${host}/file`)
+    const buffer = await streamToBuffer(res)
+    t.equal(buffer.length, FILE_SIZE, 'should get data in chunks right')
+  } finally {
+    if (originalChunkSize) process.env['FILEBOX_HTTP_CHUNK_SIZE'] = originalChunkSize
+    else delete process.env['FILEBOX_HTTP_CHUNK_SIZE']
   }
-  t.equal(length, 4372373, 'should get data in chunks right')
 })

package/src/misc.ts

@@ -15,7 +15,6 @@ import {
   HTTP_CHUNK_SIZE,
   HTTP_REQUEST_TIMEOUT,
   HTTP_RESPONSE_TIMEOUT,
-  NO_SLICE_DOWN,
 } from './config.js'
 
 const protocolMap: {
@@ -25,6 +24,9 @@ const protocolMap: {
   'https:': { agent: https.globalAgent, request: https.request },
 }
 
+const noop = () => { }
+const unsupportedRangeDomains = new Set<string>()
+
 function getProtocol (protocol: string) {
   assert(protocolMap[protocol], new Error('unknown protocol: ' + protocol))
   return protocolMap[protocol]!
@@ -51,8 +53,8 @@ export async function httpHeadHeader (url: string, headers: http.OutgoingHttpHea
     }
 
     const res = await fetch(url, {
-      method: 'HEAD',
       headers,
+      method: 'HEAD',
     }, proxyUrl)
     res.destroy()
 
@@ -69,7 +71,8 @@ export async function httpHeadHeader (url: string, headers: http.OutgoingHttpHea
       throw new Error('302 found but no location!')
     }
 
-    url = res.headers.location
+    // Location 可能是相对路径，需要以当前 url 作为 base 解析
+    url = new URL(res.headers.location, url).toString()
   }
 }
 
@@ -94,9 +97,9 @@ export async function httpStream (url: string, headers: http.OutgoingHttpHeaders
   const headHeaders = await httpHeadHeader(url, headers, proxyUrl)
   if (headHeaders.location) {
     url = headHeaders.location
-    const { protocol } = new URL(url)
-    getProtocol(protocol)
   }
+  const { protocol, hostname } = new URL(url)
+  getProtocol(protocol)
 
   const options: http.RequestOptions = {
     headers: { ...headers },
@@ -105,8 +108,12 @@ export async function httpStream (url: string, headers: http.OutgoingHttpHeaders
 
   const fileSize = Number(headHeaders['content-length'])
 
-  if (!NO_SLICE_DOWN && headHeaders['accept-ranges'] === 'bytes' && fileSize > HTTP_CHUNK_SIZE) {
-    return await downloadFileInChunks(url, options, fileSize, HTTP_CHUNK_SIZE, proxyUrl)
+  // 运行时读取 env：方便测试/调用方动态调整
+  const noSliceDown = process.env['FILEBOX_NO_SLICE_DOWN'] === 'true'
+  const chunkSize = Number(process.env['FILEBOX_HTTP_CHUNK_SIZE']) || HTTP_CHUNK_SIZE
+
+  if (!unsupportedRangeDomains.has(hostname) && !noSliceDown && headHeaders['accept-ranges'] === 'bytes' && fileSize > chunkSize) {
+    return await downloadFileInChunks(url, options, fileSize, chunkSize, proxyUrl)
   } else {
     return await fetch(url, options, proxyUrl)
   }
@@ -115,32 +122,81 @@ export async function httpStream (url: string, headers: http.OutgoingHttpHeaders
 async function fetch (url: string, options: http.RequestOptions, proxyUrl?: string): Promise<http.IncomingMessage> {
   const { protocol } = new URL(url)
   const { request, agent } = getProtocol(protocol)
+  const abortController = new AbortController()
+  const signal = abortController.signal
   const opts: http.RequestOptions = {
     agent,
     ...options,
+    signal,
   }
   setProxy(opts, proxyUrl)
+
   const req = request(url, opts)
-  req
-    .on('error', () => {
-      req.destroy()
+  let res: http.IncomingMessage | undefined
+
+  // 兜底：任何时候 req.error 都不会变成 uncaughtException
+  const onReqError = (err: unknown) => {
+    const error = err instanceof Error ? err : new Error(String(err))
+    // 统一用 abort 中止请求：signal 已挂在 req 上，并且我们会把 abort(reason) 桥接到 res.destroy(...)
+    abortController.abort(error)
+  }
+  req.on('error', noop)
+    .on('error', onReqError)
+    .once('close', () => {
+      // close 后禁用 request timeout，避免定时器晚到触发 destroy -> error 无人监听
+      try { req.setTimeout(0) } catch { }
+      req.off('error', onReqError)
+      req.off('error', noop)
     })
+    // request timeout：只用于“拿到 response 之前”（连接/握手/首包）
     .setTimeout(HTTP_REQUEST_TIMEOUT, () => {
-      req.emit('error', new Error(`FileBox: Http request timeout (${HTTP_REQUEST_TIMEOUT})!`))
+      // 已经拿到 response 时，不要再用 request timeout 误伤（会导致 aborted/ECONNRESET）
+      if (res) return
+      abortController.abort(new Error(`FileBox: Http request timeout (${HTTP_REQUEST_TIMEOUT})!`))
     })
     .end()
-  const responseEvents = await once(req, 'response')
-  const res = responseEvents[0] as http.IncomingMessage
-  res
-    .on('error', () => {
-      res.destroy()
+
+  try {
+    const responseEvent = await once(req, 'response', { signal })
+    res = responseEvent[0] as http.IncomingMessage
+    // response 到来后清掉 request timeout，避免误伤长下载导致 aborted/ECONNRESET
+    try { req.setTimeout(0) } catch {}
+    // 必须尽早挂，避免 “response 刚到就 error/abort” 的竞态导致 uncaughtException
+    res.on('error', noop)
+    signal.throwIfAborted()
+  } catch (e) {
+    // once(...) 被 signal abort 时通常会抛 AbortError；优先抛出 abort(reason) 的真实原因
+    const reason = signal.reason as unknown
+    const err = reason instanceof Error
+      ? reason
+      : (e instanceof Error ? e : new Error(String(e)))
+    // 失败时尽量主动清理，避免 socket 悬挂（destroy 重复调用是安全的）
+    try { res?.destroy(err) } catch {}
+    try { req.destroy(err) } catch {}
+    throw err
+  }
+
+  const onAbort = () => {
+    const reason = signal.reason as unknown
+    res?.destroy(reason instanceof Error ? reason : new Error(String(reason)))
+  }
+  signal.addEventListener('abort', onAbort, { once: true })
+  res!
+    .once('end', () => { try { res!.setTimeout(0) } catch { } })
+    .once('close', () => {
+      // close 时做清理/兜底判断（尽力而为）
+      try { res!.setTimeout(0) } catch { }
+      if (!res!.complete && !res!.destroyed) {
+        // 有些场景不会 emit 'aborted'，用 close + complete 兜底一次
+        res!.destroy(new Error('FileBox: Http response aborted!'))
+      }
+      signal.removeEventListener('abort', onAbort)
+      res!.off('error', noop)
     })
-  if (res.socket) {
-    res.setTimeout(HTTP_RESPONSE_TIMEOUT, () => {
-      res.emit('error', new Error(`FileBox: Http response timeout (${HTTP_RESPONSE_TIMEOUT})!`))
+    .setTimeout(HTTP_RESPONSE_TIMEOUT, () => {
+      abortController.abort(new Error(`FileBox: Http response timeout (${HTTP_RESPONSE_TIMEOUT})!`))
     })
-  }
-  return res
+  return res!
 }
 
 async function downloadFileInChunks (
@@ -152,6 +208,12 @@ async function downloadFileInChunks (
 ): Promise<Readable> {
   const tmpFile = join(tmpdir(), `filebox-${randomUUID()}`)
   const writeStream = createWriteStream(tmpFile)
+  const writeAbortController = new AbortController()
+  const signal = writeAbortController.signal
+  const onWriteError = (err: unknown) => {
+    writeAbortController.abort(err instanceof Error ? err : new Error(String(err)))
+  }
+  writeStream.once('error', onWriteError)
   const allowStatusCode = [ 200, 206 ]
   const requestBaseOptions: http.RequestOptions = {
     headers: {},
@@ -168,37 +230,72 @@ async function downloadFileInChunks (
     const range = `bytes=${start}-${end}`
     const requestOptions = Object.assign({}, requestBaseOptions)
     assert(requestOptions.headers, 'Errors that should not happen: Invalid headers')
-    requestOptions.headers['Range'] = range
+    ;(requestOptions.headers as http.OutgoingHttpHeaders)['Range'] = range
 
     try {
       const res = await fetch(url, requestOptions, proxyUrl)
+      if (res.statusCode === 416) {
+        unsupportedRangeDomains.add(new URL(url).hostname)
+        // 某些云服务商对分片下载的支持可能不规范，需要保留一个回退的方式
+        writeStream.destroy()
+        try {
+          await once(writeStream, 'close', { signal })
+        } catch {}
+        await rm(tmpFile, { force: true })
+        return await fetch(url, requestBaseOptions, proxyUrl)
+      }
       assert(allowStatusCode.includes(res.statusCode ?? 0), `Request failed with status code ${res.statusCode}`)
       assert(Number(res.headers['content-length']) > 0, 'Server returned 0 bytes of data')
+      try {
+        const { total } = parseContentRange(res.headers['content-range'] ?? '')
+        if (total > 0 && total < fileSize) {
+          // 某些云服务商（如腾讯云）在 head 方法中返回的 size 是原图大小，但下载时返回的是压缩后的图片，会比原图小。
+          // 这种在首次下载时虽然请求了原图大小的范围，可能比缩略图大，但会一次性返回完整的原图，而不是报错 416，通过修正 fileSize 跳出循环即可。
+          fileSize = total
+        }
+      } catch (error) {}
       for await (const chunk of res) {
         assert(Buffer.isBuffer(chunk))
         downSize += chunk.length
-        writeStream.write(chunk)
+        if (!writeStream.write(chunk)) {
+          try {
+            await once(writeStream, 'drain', { signal })
+          } catch (e) {
+            const reason = signal.reason as unknown
+            throw reason instanceof Error ? reason : (e as Error)
+          }
+        }
       }
       res.destroy()
     } catch (error) {
-      const err = error as Error
+      const err = error instanceof Error ? error : new Error(String(error))
       if (--retries <= 0) {
+        writeStream.destroy()
         void rm(tmpFile, { force: true })
-        writeStream.close()
         throw new Error(`Download file with chunk failed! ${err.message}`, { cause: err })
       }
     }
     chunkSeq++
     start = downSize
   }
-  writeStream.close()
+
+  writeStream.end()
+  try {
+    await once(writeStream, 'finish', { signal })
+  } catch (e) {
+    const reason = signal.reason as unknown
+    if (reason instanceof Error) {
+      throw reason
+    }
+    throw e
+  } finally {
+    writeStream.off('error', onWriteError)
+  }
 
   const readStream = createReadStream(tmpFile)
-  readStream
-    .once('end', () => readStream.close())
-    .once('close', () => {
-      void rm(tmpFile, { force: true })
-    })
+  readStream.once('close', () => {
+    rm(tmpFile, { force: true }).catch(() => {})
+  })
   return readStream
 }
 
@@ -216,3 +313,15 @@ function setProxy (options: RequestOptions, proxyUrl?: string): void {
     options.agent = agent
   }
 }
+
+function parseContentRange (contentRange: string): { start: number, end: number, total: number } {
+  const matches = contentRange.match(/bytes (\d+)-(\d+)\/(\d+)/)
+  if (!matches) {
+    throw new Error('Invalid content range')
+  }
+  return {
+    end: Number(matches[2]),
+    start: Number(matches[1]),
+    total: Number(matches[3]),
+  }
+}

package/src/version.ts

@@ -1,4 +1,4 @@
 /**
  * This file was auto generated from scripts/generate-version.sh
  */
-export const VERSION: string = '1.7.18'
+export const VERSION: string = '1.7.20'