@juzi/file-box 1.8.2 → 1.8.5

package/src/misc.spec.ts

@@ -12,12 +12,16 @@ import {
   httpHeadHeader,
   httpStream,
   streamToBuffer,
+  __clearUnsupportedRangeDomains,
+  __addUnsupportedRangeDomain,
 } from './misc.js'
 
 // 设置短超时用于测试
 CONFIG.HTTP_REQUEST_TIMEOUT = 1000
 CONFIG.HTTP_RESPONSE_TIMEOUT = 1000
 
+const TC_QQ_DOWNLOAD_URL = 'http://wxapp.tc.qq.com/270/20304/stodownload?m=775ba582fe1d27e158806a4c10230a45&filekey=30350201010421301f0202010e040253480410775ba582fe1d27e158806a4c10230a450203017a31040d00000004627466730000000132&hy=SH&storeid=2685965c90008afdf000000000000010e00004f50534801c33031571b54757&bizid=1023'
+
 test('dataUrl to base64', async t => {
   const base64 = [
     'R0lGODlhEAAQAMQAAORHHOVSKudfOulrSOp3WOyDZu6QdvCchPGolfO0o/XBs/fNwfjZ0frl',
@@ -72,6 +76,17 @@ test('httpHeadHeader', async t => {
   )
 })
 
+test('httpHeadHeader with tc qq download url', async t => {
+  const requestTimeout = CONFIG.HTTP_REQUEST_TIMEOUT
+  CONFIG.HTTP_REQUEST_TIMEOUT = 10000
+  t.teardown(() => { CONFIG.HTTP_REQUEST_TIMEOUT = requestTimeout })
+
+  const headers = await httpHeadHeader(TC_QQ_DOWNLOAD_URL)
+  t.equal(headers['accept-ranges'], 'bytes', 'should support byte range')
+  t.equal(headers['content-type'], 'image/jpg', 'should get content type')
+  t.ok(Number(headers['content-length']) > 0, 'should get content length')
+})
+
 test('httpHeaderToFileName', async t => {
   const HEADERS_QUOTATION_MARK: any = {
     'content-disposition': 'attachment; filename="db-0.0.19.zip"',
@@ -187,3 +202,170 @@ test('httpStream in chunks', async (t) => {
   const buffer = await streamToBuffer(res)
   t.equal(buffer.length, FILE_SIZE, 'should get data in chunks right')
 })
+
+test('httpStream: HEAD Accept-Ranges=none 时不发 Range 请求(A2)', async (t) => {
+  __clearUnsupportedRangeDomains()
+
+  const TRUE_DATA = Buffer.from('TRUE-DATA-A2', 'utf8')
+  let getCallCount = 0
+  let getHadRangeHeader: boolean | undefined
+
+  const server = createServer((req, res) => {
+    if (req.method === 'HEAD') {
+      res.writeHead(200, {
+        'Accept-Ranges': 'none',
+        'Content-Length': String(TRUE_DATA.length),
+      })
+      res.end()
+      return
+    }
+    getCallCount += 1
+    getHadRangeHeader = 'range' in req.headers
+    res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+    res.end(TRUE_DATA)
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  const stream = await httpStream(`${host}/file`)
+  const buffer = await streamToBuffer(stream)
+
+  t.equal(getCallCount, 1, 'GET 应只被调用 1 次')
+  t.equal(getHadRangeHeader, false, 'GET 请求不应携带 Range header')
+  t.equal(buffer.toString('utf8'), TRUE_DATA.toString('utf8'), '应拿到真实数据')
+})
+
+test('httpStream: HEAD Accept-Ranges=none 端到端持久化,第二次请求直接跳过 Range(A2 持久化)', async (t) => {
+  __clearUnsupportedRangeDomains()
+
+  const TRUE_DATA = Buffer.from('E2E-A2-PERSISTENCE', 'utf8')
+  let getCallCount = 0
+  const getRangeHeaderByCall: (string | undefined)[] = []
+
+  const server = createServer((req, res) => {
+    if (req.method === 'HEAD') {
+      res.writeHead(200, {
+        'Accept-Ranges': 'none',
+        'Content-Length': String(TRUE_DATA.length),
+      })
+      res.end()
+      return
+    }
+    getCallCount += 1
+    const rangeHeader = req.headers.range
+    getRangeHeaderByCall.push(typeof rangeHeader === 'string' ? rangeHeader : undefined)
+    res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+    res.end(TRUE_DATA)
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  // 第一次请求:HEAD 里 Accept-Ranges=none,A2 把 host 加入黑名单
+  const stream1 = await httpStream(`${host}/file`)
+  await streamToBuffer(stream1)
+
+  // 第二次请求:同 host,预期 A2 黑名单命中,GET 不带 Range
+  const stream2 = await httpStream(`${host}/file`)
+  const buffer2 = await streamToBuffer(stream2)
+
+  t.equal(getCallCount, 2, 'GET 应被调用 2 次(每次请求各 1 次,无 Range 重试)')
+  t.equal(getRangeHeaderByCall[0], undefined, '第 1 次 GET 不带 Range(本次 HEAD 已宣告 none)')
+  t.equal(getRangeHeaderByCall[1], undefined, '第 2 次 GET 不带 Range(黑名单命中)')
+  t.equal(buffer2.toString('utf8'), TRUE_DATA.toString('utf8'), '第二次应拿到真实数据')
+})
+
+test('httpStream: 带 Range 却收到 200 时回退重发不带 Range(B1 - CMSV6 场景)', async (t) => {
+  __clearUnsupportedRangeDomains()
+
+  const FAKE_DATA = Buffer.from('FAKE-DATA-FROM-WRONG-STREAM', 'utf8')
+  const TRUE_DATA = Buffer.alloc(FAKE_DATA.length, 'T') // 长度相同,内容不同
+  let getCallCount = 0
+  const getRangeHeaderByCall: (string | undefined)[] = []
+
+  const server = createServer((req, res) => {
+    if (req.method === 'HEAD') {
+      // 注意:不返回 Accept-Ranges,模拟 CMSV6
+      res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+      res.end()
+      return
+    }
+    getCallCount += 1
+    const rangeHeader = req.headers.range
+    getRangeHeaderByCall.push(typeof rangeHeader === 'string' ? rangeHeader : undefined)
+
+    // CMSV6 行为:无论是否带 Range,都返回 200 + 正确 Content-Length
+    // 但内容随 Range 存在与否而不同
+    res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+    if (rangeHeader) {
+      res.end(FAKE_DATA)
+    } else {
+      res.end(TRUE_DATA)
+    }
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  const stream = await httpStream(`${host}/file`)
+  const buffer = await streamToBuffer(stream)
+
+  t.equal(getCallCount, 2, 'GET 应被调用 2 次(第一次带 Range 触发 B1,第二次回退)')
+  t.ok(getRangeHeaderByCall[0], '第 1 次 GET 应携带 Range header')
+  t.equal(getRangeHeaderByCall[1], undefined, '第 2 次 GET 不应携带 Range header')
+  t.equal(buffer.toString('utf8'), TRUE_DATA.toString('utf8'), '最终数据应为 TRUE_DATA(回退后拿到的)')
+})
+
+test('httpStream: 黑名单登记的 host 后续请求直接跳过 Range(B1 黑名单持久化)', async (t) => {
+  __clearUnsupportedRangeDomains()
+
+  const TRUE_DATA = Buffer.from('SECOND-DOWNLOAD-AFTER-BLACKLIST', 'utf8')
+  let getCallCount = 0
+  let getHadRangeHeader: boolean | undefined
+
+  const server = createServer((req, res) => {
+    if (req.method === 'HEAD') {
+      res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+      res.end()
+      return
+    }
+    getCallCount += 1
+    getHadRangeHeader = 'range' in req.headers
+    res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+    res.end(TRUE_DATA)
+  })
+
+  const port = await new Promise<number>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(addr.port)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  // 测试前手工 seed hostKey 进黑名单,模拟"此前已因 B1 加入过"
+  __addUnsupportedRangeDomain(`127.0.0.1:${port}`)
+
+  const stream = await httpStream(`http://127.0.0.1:${port}/file`)
+  const buffer = await streamToBuffer(stream)
+
+  t.equal(getCallCount, 1, '黑名单命中后 GET 只调用 1 次')
+  t.equal(getHadRangeHeader, false, '黑名单命中后 GET 不带 Range header')
+  t.equal(buffer.toString('utf8'), TRUE_DATA.toString('utf8'), '应拿到真实数据')
+})

package/src/misc.ts

@@ -24,8 +24,27 @@ const protocolMap: {
 }
 
 const noop = () => { }
+
+// 运行期黑名单:记录已知不支持 Range 请求的 host(格式 `hostname:port`)
+// 用 Set 天然按插入序保序,命中上限时淘汰最早插入的条目(FIFO)
+// 上限防止长运行进程(如网关)内存无界增长
+const UNSUPPORTED_RANGE_DOMAINS_MAX = 1024
 const unsupportedRangeDomains = new Set<string>()
 
+function addUnsupportedRangeDomain (hostKey: string): void {
+  if (unsupportedRangeDomains.has(hostKey)) {
+    return
+  }
+  unsupportedRangeDomains.add(hostKey)
+  if (unsupportedRangeDomains.size > UNSUPPORTED_RANGE_DOMAINS_MAX) {
+    // Set.values() 按插入顺序迭代,删除最早一条即可
+    const oldest = unsupportedRangeDomains.values().next().value
+    if (oldest !== undefined) {
+      unsupportedRangeDomains.delete(oldest)
+    }
+  }
+}
+
 // 自定义 Error：标记需要回退到非分片下载
 class FallbackError extends Error {
 
@@ -36,6 +55,16 @@ class FallbackError extends Error {
 
 }
 
+// 仅测试用:清空运行期黑名单(模块级 Set 跨 test 会污染)
+export function __clearUnsupportedRangeDomains (): void {
+  unsupportedRangeDomains.clear()
+}
+
+// 仅测试用:手工登记一个 host 到黑名单(用于验证后续请求直接跳过 Range)
+export function __addUnsupportedRangeDomain (hostKey: string): void {
+  addUnsupportedRangeDomain(hostKey)
+}
+
 function getProtocol (protocol: string) {
   assert(protocolMap[protocol], new Error('unknown protocol: ' + protocol))
   return protocolMap[protocol]!
@@ -61,10 +90,7 @@ export async function httpHeadHeader (url: string, headers: http.OutgoingHttpHea
       throw new Error(`ttl expired! too many(>${REDIRECT_TTL}) 302 redirection.`)
     }
 
-    const res = await fetch(url, {
-      headers,
-      method: 'HEAD',
-    }, proxyUrl)
+    const res = await fetchHead(url, headers, proxyUrl)
     res.destroy()
 
     if (!/^3/.test(String(res.statusCode))) {
@@ -117,11 +143,20 @@ export async function httpStream (url: string, headers: http.OutgoingHttpHeaders
   const defaultPort = protocol === 'https:' ? '443' : '80'
   const hostKey = `${hostname}:${port || defaultPort}`
 
-  // 直接尝试分片下载，不检查 Accept-Ranges 和 fileSize
+  // A2：若 HEAD 明确声明 Accept-Ranges: none，记录到运行期黑名单
+  // 以便 downloadFileInChunks 本次请求就直接以非 Range 模式发起
+  // Accept-Ranges header 可能是 string | string[]，归一化后匹配 'none'
+  const acceptRangesRaw = headHeaders['accept-ranges']
+  const acceptRanges = Array.isArray(acceptRangesRaw) ? acceptRangesRaw[0] : acceptRangesRaw
+  if (typeof acceptRanges === 'string' && acceptRanges.trim().toLowerCase() === 'none') {
+    addUnsupportedRangeDomain(hostKey)
+  }
+
+  // 直接尝试分片下载，不检查 fileSize
   // 原因：
   // 1. 有些服务器 HEAD 不返回 Accept-Ranges 但实际支持分片
   // 2. 有些服务器 HEAD 返回 fileSize=0 但实际支持分片
-  // downloadFileInChunks 内部有完善的回退机制处理不支持的情况
+  // downloadFileInChunks 内部有完善的回退机制处理不支持的情况（见 B1）
   const result = await downloadFileInChunks(url, options, proxyUrl, hostKey)
   return result
 }
@@ -205,6 +240,68 @@ async function fetch (url: string, options: http.RequestOptions, proxyUrl?: stri
   return res!
 }
 
+async function fetchHead (url: string, headers: http.OutgoingHttpHeaders = {}, proxyUrl?: string): Promise<http.IncomingMessage> {
+  try {
+    return await fetch(url, {
+      headers,
+      method: 'HEAD',
+    }, proxyUrl)
+  } catch (error) {
+    if (!shouldFallbackHeadToRangeGet(error)) {
+      throw error
+    }
+
+    try {
+      return await fetchRangeHeader(url, headers, proxyUrl)
+    } catch {
+      throw error
+    }
+  }
+}
+
+function shouldFallbackHeadToRangeGet (error: unknown): boolean {
+  const code = typeof error === 'object' && error && 'code' in error
+    ? String((error as NodeJS.ErrnoException).code)
+    : ''
+  const message = error instanceof Error ? error.message : String(error)
+
+  return code.startsWith('HPE_') || message.includes('Parse Error')
+}
+
+async function fetchRangeHeader (url: string, headers: http.OutgoingHttpHeaders = {}, proxyUrl?: string): Promise<http.IncomingMessage> {
+  const res = await fetch(url, {
+    headers: createRangeProbeHeaders(headers),
+    method: 'GET',
+  }, proxyUrl)
+  normalizeRangeProbeHeaders(res.headers)
+  return res
+}
+
+function createRangeProbeHeaders (headers: http.OutgoingHttpHeaders): http.OutgoingHttpHeaders {
+  const rangeHeaders: http.OutgoingHttpHeaders = {}
+
+  for (const [ key, value ] of Object.entries(headers)) {
+    if (key.toLowerCase() === 'range') {
+      continue
+    }
+    rangeHeaders[key] = value
+  }
+
+  rangeHeaders['Range'] = 'bytes=0-0'
+  return rangeHeaders
+}
+
+function normalizeRangeProbeHeaders (headers: http.IncomingHttpHeaders): void {
+  const contentRange = Array.isArray(headers['content-range'])
+    ? headers['content-range'][0]
+    : headers['content-range']
+  const matches = contentRange?.match(/bytes \d+-\d+\/(\d+)/)
+
+  if (matches?.[1]) {
+    headers['content-length'] = matches[1]
+  }
+}
+
 function createSkipTransform (skipBytes: number): Transform {
   let skipped = 0
   return new Transform({
@@ -233,7 +330,7 @@ async function downloadFileInChunks (
   url: string,
   options: http.RequestOptions,
   proxyUrl: string | undefined,
-  hostname: string,
+  hostKey: string,
 ): Promise<Readable> {
   const tmpFile = join(tmpdir(), `filebox-${randomUUID()}`)
   let writeStream = createWriteStream(tmpFile)
@@ -254,8 +351,7 @@ async function downloadFileInChunks (
   let downSize = 0
   let retries = 3
   // 控制是否使用 Range 请求（根据域名黑名单初始化）
-  let useRange = !unsupportedRangeDomains.has(hostname)
-  let useChunked = false
+  let useRange = !unsupportedRangeDomains.has(hostKey)
 
   do {
     // 每次循环前检查文件实际大小，作为真实的下载进度
@@ -323,9 +419,6 @@ async function downloadFileInChunks (
           throw new Error(`File size mismatch: expected ${expectedTotal}, but server returned ${total}`)
         }
 
-        // 标记使用了分片下载
-        useChunked = true
-
         // 验证服务器返回的范围是否与请求匹配
         if (actualStart !== start) {
           if (actualStart > start) {
@@ -350,18 +443,16 @@ async function downloadFileInChunks (
         downSize += end - start + 1
         start = downSize
       } else if (res.statusCode === 200) {
-        // 200: 服务器返回完整文件
-        if (useChunked || start > 0) {
-          // 之前以分片模式下载过数据
-          writeStream.destroy()
-          await rm(tmpFile, { force: true }).catch(() => {})
-          writeStream = createWriteStream(tmpFile)
-          writeStream.on('error', onWriteError)
-          start = 0
-          downSize = 0
+        if (useRange) {
+          // B1：发了 Range 却收到 200 —— 服务器未实现 Range
+          // 响应体是"对带 Range 请求的回答"，不可信（见 CMSV6 场景）
+          // 交给 FallbackError 的 catch 分支统一处理：销毁流、删 tmp、加入黑名单、重发
+          throw new FallbackError('Server returned 200 for Range request')
         }
-
-        // 处理完整文件响应
+        // 200: 服务器返回完整文件（本次未带 Range）
+        // B1 保证带 Range 收 200 一定经 FallbackError 回退,catch 里已重置
+        // expectedTotal/downSize/start/useRange,进入此分支时一定是
+        // 全新的非 Range 请求,无须再清理已写数据
         expectedTotal = contentLength
         await pipeline(res, writeStream, { end: false, signal })
         downSize = contentLength
@@ -374,7 +465,7 @@ async function downloadFileInChunks (
     } catch (error) {
       if (error instanceof FallbackError) {
         // 回退逻辑：记录域名、重置状态，在下次循环中以非 range 模式请求
-        unsupportedRangeDomains.add(hostname)
+        addUnsupportedRangeDomain(hostKey)
 
         // 关闭当前写入流
         writeStream.destroy()
@@ -393,7 +484,6 @@ async function downloadFileInChunks (
         expectedTotal = null
         downSize = 0
         start = 0
-        useChunked = false
         useRange = false
         retries = 3
         continue

package/src/version.ts

@@ -1,4 +1,4 @@
 /**
  * This file was auto generated from scripts/generate-version.sh
  */
-export const VERSION: string = '1.8.2'
+export const VERSION: string = '1.8.5'