@juzi/file-box 1.8.4 → 1.8.6

package/src/misc.spec.ts

@@ -12,6 +12,8 @@ import {
   httpHeadHeader,
   httpStream,
   streamToBuffer,
+  __clearUnsupportedRangeDomains,
+  __addUnsupportedRangeDomain,
 } from './misc.js'
 
 // 设置短超时用于测试
@@ -200,3 +202,217 @@ test('httpStream in chunks', async (t) => {
   const buffer = await streamToBuffer(res)
   t.equal(buffer.length, FILE_SIZE, 'should get data in chunks right')
 })
+
+test('httpStream: HEAD Accept-Ranges=none 时不发 Range 请求(A2)', async (t) => {
+  __clearUnsupportedRangeDomains()
+
+  const TRUE_DATA = Buffer.from('TRUE-DATA-A2', 'utf8')
+  let getCallCount = 0
+  let getHadRangeHeader: boolean | undefined
+
+  const server = createServer((req, res) => {
+    if (req.method === 'HEAD') {
+      res.writeHead(200, {
+        'Accept-Ranges': 'none',
+        'Content-Length': String(TRUE_DATA.length),
+      })
+      res.end()
+      return
+    }
+    getCallCount += 1
+    getHadRangeHeader = 'range' in req.headers
+    res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+    res.end(TRUE_DATA)
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  const stream = await httpStream(`${host}/file`)
+  const buffer = await streamToBuffer(stream)
+
+  t.equal(getCallCount, 1, 'GET 应只被调用 1 次')
+  t.equal(getHadRangeHeader, false, 'GET 请求不应携带 Range header')
+  t.equal(buffer.toString('utf8'), TRUE_DATA.toString('utf8'), '应拿到真实数据')
+})
+
+test('httpStream: HEAD Accept-Ranges=none 端到端持久化,第二次请求直接跳过 Range(A2 持久化)', async (t) => {
+  __clearUnsupportedRangeDomains()
+
+  const TRUE_DATA = Buffer.from('E2E-A2-PERSISTENCE', 'utf8')
+  let getCallCount = 0
+  const getRangeHeaderByCall: (string | undefined)[] = []
+
+  const server = createServer((req, res) => {
+    if (req.method === 'HEAD') {
+      res.writeHead(200, {
+        'Accept-Ranges': 'none',
+        'Content-Length': String(TRUE_DATA.length),
+      })
+      res.end()
+      return
+    }
+    getCallCount += 1
+    const rangeHeader = req.headers.range
+    getRangeHeaderByCall.push(typeof rangeHeader === 'string' ? rangeHeader : undefined)
+    res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+    res.end(TRUE_DATA)
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  // 第一次请求:HEAD 里 Accept-Ranges=none,A2 把 host 加入黑名单
+  const stream1 = await httpStream(`${host}/file`)
+  await streamToBuffer(stream1)
+
+  // 第二次请求:同 host,预期 A2 黑名单命中,GET 不带 Range
+  const stream2 = await httpStream(`${host}/file`)
+  const buffer2 = await streamToBuffer(stream2)
+
+  t.equal(getCallCount, 2, 'GET 应被调用 2 次(每次请求各 1 次,无 Range 重试)')
+  t.equal(getRangeHeaderByCall[0], undefined, '第 1 次 GET 不带 Range(本次 HEAD 已宣告 none)')
+  t.equal(getRangeHeaderByCall[1], undefined, '第 2 次 GET 不带 Range(黑名单命中)')
+  t.equal(buffer2.toString('utf8'), TRUE_DATA.toString('utf8'), '第二次应拿到真实数据')
+})
+
+test('httpStream: 带 Range 却收到 200 时回退重发不带 Range(B1 - CMSV6 场景)', async (t) => {
+  __clearUnsupportedRangeDomains()
+
+  const FAKE_DATA = Buffer.from('FAKE-DATA-FROM-WRONG-STREAM', 'utf8')
+  const TRUE_DATA = Buffer.alloc(FAKE_DATA.length, 'T') // 长度相同,内容不同
+  let getCallCount = 0
+  const getRangeHeaderByCall: (string | undefined)[] = []
+
+  const server = createServer((req, res) => {
+    if (req.method === 'HEAD') {
+      // 注意:不返回 Accept-Ranges,模拟 CMSV6
+      res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+      res.end()
+      return
+    }
+    getCallCount += 1
+    const rangeHeader = req.headers.range
+    getRangeHeaderByCall.push(typeof rangeHeader === 'string' ? rangeHeader : undefined)
+
+    // CMSV6 行为:无论是否带 Range,都返回 200 + 正确 Content-Length
+    // 但内容随 Range 存在与否而不同
+    res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+    if (rangeHeader) {
+      res.end(FAKE_DATA)
+    } else {
+      res.end(TRUE_DATA)
+    }
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  const stream = await httpStream(`${host}/file`)
+  const buffer = await streamToBuffer(stream)
+
+  t.equal(getCallCount, 2, 'GET 应被调用 2 次(第一次带 Range 触发 B1,第二次回退)')
+  t.ok(getRangeHeaderByCall[0], '第 1 次 GET 应携带 Range header')
+  t.equal(getRangeHeaderByCall[1], undefined, '第 2 次 GET 不应携带 Range header')
+  t.equal(buffer.toString('utf8'), TRUE_DATA.toString('utf8'), '最终数据应为 TRUE_DATA(回退后拿到的)')
+})
+
+test('httpStream: HEAD 返回 4xx 且 Range GET 返回 400 时回退到非 Range 模式(B2 - HEAD 不支持场景)', async (t) => {
+  __clearUnsupportedRangeDomains()
+
+  const TRUE_DATA = Buffer.from('TRUE-DATA-HEAD-UNSUPPORTED', 'utf8')
+  let getCallCount = 0
+  const getRangeHeaderByCall: (string | undefined)[] = []
+
+  const server = createServer((req, res) => {
+    if (req.method === 'HEAD') {
+      // 模拟服务器不支持 HEAD 方法，返回 425
+      res.writeHead(425)
+      res.end()
+      return
+    }
+    getCallCount += 1
+    const rangeHeader = req.headers.range
+    getRangeHeaderByCall.push(typeof rangeHeader === 'string' ? rangeHeader : undefined)
+
+    if (rangeHeader) {
+      // 服务器也不支持 Range，返回 400
+      res.writeHead(400)
+      res.end('Bad Request')
+      return
+    }
+
+    // 不带 Range 的 GET 正常返回
+    res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+    res.end(TRUE_DATA)
+  })
+
+  const host = await new Promise<string>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(`http://127.0.0.1:${addr.port}`)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  const stream = await httpStream(`${host}/file`)
+  const buffer = await streamToBuffer(stream)
+
+  t.equal(getCallCount, 2, 'GET 应被调用 2 次(第一次带 Range 收到 400 触发回退,第二次不带 Range)')
+  t.ok(getRangeHeaderByCall[0], '第 1 次 GET 应携带 Range header')
+  t.equal(getRangeHeaderByCall[1], undefined, '第 2 次 GET 不应携带 Range header')
+  t.equal(buffer.toString('utf8'), TRUE_DATA.toString('utf8'), '最终数据应为 TRUE_DATA')
+})
+
+test('httpStream: 黑名单登记的 host 后续请求直接跳过 Range(B1 黑名单持久化)', async (t) => {
+  __clearUnsupportedRangeDomains()
+
+  const TRUE_DATA = Buffer.from('SECOND-DOWNLOAD-AFTER-BLACKLIST', 'utf8')
+  let getCallCount = 0
+  let getHadRangeHeader: boolean | undefined
+
+  const server = createServer((req, res) => {
+    if (req.method === 'HEAD') {
+      res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+      res.end()
+      return
+    }
+    getCallCount += 1
+    getHadRangeHeader = 'range' in req.headers
+    res.writeHead(200, { 'Content-Length': String(TRUE_DATA.length) })
+    res.end(TRUE_DATA)
+  })
+
+  const port = await new Promise<number>((resolve) => {
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address() as AddressInfo
+      resolve(addr.port)
+    })
+  })
+  t.teardown(() => { server.close() })
+
+  // 测试前手工 seed hostKey 进黑名单,模拟"此前已因 B1 加入过"
+  __addUnsupportedRangeDomain(`127.0.0.1:${port}`)
+
+  const stream = await httpStream(`http://127.0.0.1:${port}/file`)
+  const buffer = await streamToBuffer(stream)
+
+  t.equal(getCallCount, 1, '黑名单命中后 GET 只调用 1 次')
+  t.equal(getHadRangeHeader, false, '黑名单命中后 GET 不带 Range header')
+  t.equal(buffer.toString('utf8'), TRUE_DATA.toString('utf8'), '应拿到真实数据')
+})

package/src/misc.ts

@@ -24,8 +24,27 @@ const protocolMap: {
 }
 
 const noop = () => { }
+
+// 运行期黑名单:记录已知不支持 Range 请求的 host(格式 `hostname:port`)
+// 用 Set 天然按插入序保序,命中上限时淘汰最早插入的条目(FIFO)
+// 上限防止长运行进程(如网关)内存无界增长
+const UNSUPPORTED_RANGE_DOMAINS_MAX = 1024
 const unsupportedRangeDomains = new Set<string>()
 
+function addUnsupportedRangeDomain (hostKey: string): void {
+  if (unsupportedRangeDomains.has(hostKey)) {
+    return
+  }
+  unsupportedRangeDomains.add(hostKey)
+  if (unsupportedRangeDomains.size > UNSUPPORTED_RANGE_DOMAINS_MAX) {
+    // Set.values() 按插入顺序迭代,删除最早一条即可
+    const oldest = unsupportedRangeDomains.values().next().value
+    if (oldest !== undefined) {
+      unsupportedRangeDomains.delete(oldest)
+    }
+  }
+}
+
 // 自定义 Error：标记需要回退到非分片下载
 class FallbackError extends Error {
 
@@ -36,6 +55,16 @@ class FallbackError extends Error {
 
 }
 
+// 仅测试用:清空运行期黑名单(模块级 Set 跨 test 会污染)
+export function __clearUnsupportedRangeDomains (): void {
+  unsupportedRangeDomains.clear()
+}
+
+// 仅测试用:手工登记一个 host 到黑名单(用于验证后续请求直接跳过 Range)
+export function __addUnsupportedRangeDomain (hostKey: string): void {
+  addUnsupportedRangeDomain(hostKey)
+}
+
 function getProtocol (protocol: string) {
   assert(protocolMap[protocol], new Error('unknown protocol: ' + protocol))
   return protocolMap[protocol]!
@@ -114,11 +143,20 @@ export async function httpStream (url: string, headers: http.OutgoingHttpHeaders
   const defaultPort = protocol === 'https:' ? '443' : '80'
   const hostKey = `${hostname}:${port || defaultPort}`
 
-  // 直接尝试分片下载，不检查 Accept-Ranges 和 fileSize
+  // A2：若 HEAD 明确声明 Accept-Ranges: none，记录到运行期黑名单
+  // 以便 downloadFileInChunks 本次请求就直接以非 Range 模式发起
+  // Accept-Ranges header 可能是 string | string[]，归一化后匹配 'none'
+  const acceptRangesRaw = headHeaders['accept-ranges']
+  const acceptRanges = Array.isArray(acceptRangesRaw) ? acceptRangesRaw[0] : acceptRangesRaw
+  if (typeof acceptRanges === 'string' && acceptRanges.trim().toLowerCase() === 'none') {
+    addUnsupportedRangeDomain(hostKey)
+  }
+
+  // 直接尝试分片下载，不检查 fileSize
   // 原因：
   // 1. 有些服务器 HEAD 不返回 Accept-Ranges 但实际支持分片
   // 2. 有些服务器 HEAD 返回 fileSize=0 但实际支持分片
-  // downloadFileInChunks 内部有完善的回退机制处理不支持的情况
+  // downloadFileInChunks 内部有完善的回退机制处理不支持的情况（见 B1）
   const result = await downloadFileInChunks(url, options, proxyUrl, hostKey)
   return result
 }
@@ -292,7 +330,7 @@ async function downloadFileInChunks (
   url: string,
   options: http.RequestOptions,
   proxyUrl: string | undefined,
-  hostname: string,
+  hostKey: string,
 ): Promise<Readable> {
   const tmpFile = join(tmpdir(), `filebox-${randomUUID()}`)
   let writeStream = createWriteStream(tmpFile)
@@ -313,8 +351,7 @@ async function downloadFileInChunks (
   let downSize = 0
   let retries = 3
   // 控制是否使用 Range 请求（根据域名黑名单初始化）
-  let useRange = !unsupportedRangeDomains.has(hostname)
-  let useChunked = false
+  let useRange = !unsupportedRangeDomains.has(hostKey)
 
   do {
     // 每次循环前检查文件实际大小，作为真实的下载进度
@@ -345,6 +382,12 @@ async function downloadFileInChunks (
         // 416: Range Not Satisfiable，服务器不支持此范围或文件大小不匹配
         throw new FallbackError('416 Range Not Satisfiable')
       }
+      // B2：带 Range 请求收到 4xx 时，视为服务器不支持 Range，触发回退
+      // 典型场景：HEAD 返回非标准状态码（如 425），httpHeadHeader 无法判断是否支持 Range，
+      // 随后带 Range 的 GET 收到 400/403/405 等
+      if (useRange && !allowStatusCode.includes(res.statusCode ?? 0) && res.statusCode && res.statusCode >= 400 && res.statusCode < 500) {
+        throw new FallbackError(`Server returned ${res.statusCode} for Range request`)
+      }
       assert(allowStatusCode.includes(res.statusCode ?? 0), `Request failed with status code ${res.statusCode}`)
       const contentLength = Number(res.headers['content-length']) || 0
       assert(contentLength >= 0, `Server returned ${contentLength} bytes of data`)
@@ -382,9 +425,6 @@ async function downloadFileInChunks (
           throw new Error(`File size mismatch: expected ${expectedTotal}, but server returned ${total}`)
         }
 
-        // 标记使用了分片下载
-        useChunked = true
-
         // 验证服务器返回的范围是否与请求匹配
         if (actualStart !== start) {
           if (actualStart > start) {
@@ -409,18 +449,16 @@ async function downloadFileInChunks (
         downSize += end - start + 1
         start = downSize
       } else if (res.statusCode === 200) {
-        // 200: 服务器返回完整文件
-        if (useChunked || start > 0) {
-          // 之前以分片模式下载过数据
-          writeStream.destroy()
-          await rm(tmpFile, { force: true }).catch(() => {})
-          writeStream = createWriteStream(tmpFile)
-          writeStream.on('error', onWriteError)
-          start = 0
-          downSize = 0
+        if (useRange) {
+          // B1：发了 Range 却收到 200 —— 服务器未实现 Range
+          // 响应体是"对带 Range 请求的回答"，不可信（见 CMSV6 场景）
+          // 交给 FallbackError 的 catch 分支统一处理：销毁流、删 tmp、加入黑名单、重发
+          throw new FallbackError('Server returned 200 for Range request')
         }
-
-        // 处理完整文件响应
+        // 200: 服务器返回完整文件（本次未带 Range）
+        // B1 保证带 Range 收 200 一定经 FallbackError 回退,catch 里已重置
+        // expectedTotal/downSize/start/useRange,进入此分支时一定是
+        // 全新的非 Range 请求,无须再清理已写数据
         expectedTotal = contentLength
         await pipeline(res, writeStream, { end: false, signal })
         downSize = contentLength
@@ -433,7 +471,7 @@ async function downloadFileInChunks (
     } catch (error) {
       if (error instanceof FallbackError) {
         // 回退逻辑：记录域名、重置状态，在下次循环中以非 range 模式请求
-        unsupportedRangeDomains.add(hostname)
+        addUnsupportedRangeDomain(hostKey)
 
         // 关闭当前写入流
         writeStream.destroy()
@@ -452,7 +490,6 @@ async function downloadFileInChunks (
         expectedTotal = null
         downSize = 0
         start = 0
-        useChunked = false
         useRange = false
         retries = 3
         continue

package/src/version.ts

@@ -1,4 +1,4 @@
 /**
  * This file was auto generated from scripts/generate-version.sh
  */
-export const VERSION: string = '1.8.4'
+export const VERSION: string = '1.8.6'