@juzi/file-box 1.7.4 → 1.7.5

package/src/misc.ts

@@ -1,10 +1,28 @@
-import http    from 'http'
-import https   from 'https'
+import assert from 'assert'
+import { randomUUID } from 'crypto'
+import { once } from 'events'
+import { createReadStream, createWriteStream } from 'fs'
+import { rm } from 'fs/promises'
+import http from 'http'
+import https from 'https'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import type { Readable } from 'stream'
 import { URL } from 'url'
 
-import { PassThrough, pipeline, Readable } from 'stream'
+import { HTTP_CHUNK_SIZE, HTTP_REQUEST_TIMEOUT, HTTP_RESPONSE_TIMEOUT, NO_SLICE_DOWN } from './config.js'
 
-import { HTTP_CHUNK_SIZE, HTTP_TIMEOUT } from './config.js'
+const protocolMap: {
+  [key: string]: { agent: http.Agent; request: typeof http.request }
+} = {
+  'http:': { agent: http.globalAgent, request: http.request },
+  'https:': { agent: https.globalAgent, request: https.request },
+}
+
+function getProtocol (protocol: string) {
+  assert(protocolMap[protocol], new Error('unknown protocol: ' + protocol))
+  return protocolMap[protocol]!
+}
 
 export function dataUrlToBase64 (dataUrl: string): string {
   const dataList = dataUrl.split(',')
@@ -18,7 +36,6 @@ export function dataUrlToBase64 (dataUrl: string): string {
  * @credit https://stackoverflow.com/a/43632171/1123955
  */
 export async function httpHeadHeader (url: string): Promise<http.IncomingHttpHeaders> {
-
   const originUrl = url
   let REDIRECT_TTL = 7
 
@@ -27,7 +44,10 @@ export async function httpHeadHeader (url: string): Promise<http.IncomingHttpHea
       throw new Error(`ttl expired! too many(>${REDIRECT_TTL}) 302 redirection.`)
     }
 
-    const res = await _headHeader(url)
+    const res = await fetch(url, {
+      method: 'HEAD',
+    })
+    res.destroy()
 
     if (!/^3/.test(String(res.statusCode))) {
       if (originUrl !== url) {
@@ -44,44 +64,9 @@ export async function httpHeadHeader (url: string): Promise<http.IncomingHttpHea
 
     url = res.headers.location
   }
-
-  async function _headHeader (destUrl: string): Promise<http.IncomingMessage> {
-    const parsedUrl = new URL(destUrl)
-    const options = {
-      method   : 'HEAD',
-      // method   : 'GET',
-    }
-
-    let request: typeof http.request
-
-    if (parsedUrl.protocol === 'https:') {
-      request = https.request
-    } else if (parsedUrl.protocol === 'http:') {
-      request = http.request
-    } else {
-      throw new Error('unknown protocol: ' + parsedUrl.protocol)
-    }
-
-    return new Promise<http.IncomingMessage>((resolve, reject) => {
-      let res: undefined | http.IncomingMessage
-      const req = request(parsedUrl, options, (response) => {
-        res = response
-        resolve(res)
-      })
-        .on('error', reject)
-        .setTimeout(HTTP_TIMEOUT, () => {
-          const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`)
-          req.emit('error', e)
-          req.destroy()
-        })
-        .end()
-    })
-  }
 }
 
-export function httpHeaderToFileName (
-  headers: http.IncomingHttpHeaders,
-): null | string {
+export function httpHeaderToFileName (headers: http.IncomingHttpHeaders): null | string {
   const contentDisposition = headers['content-disposition']
 
   if (!contentDisposition) {
@@ -98,224 +83,117 @@ export function httpHeaderToFileName (
   return null
 }
 
-export async function httpStream (
-  url     : string,
-  headers : http.OutgoingHttpHeaders = {},
-): Promise<Readable> {
+export async function httpStream (url: string, headers: http.OutgoingHttpHeaders = {}): Promise<Readable> {
   const headHeaders = await httpHeadHeader(url)
   if (headHeaders.location) {
     url = headHeaders.location
+    const { protocol } = new URL(url)
+    getProtocol(protocol)
   }
 
-  const parsedUrl = new URL(url)
-
-  const protocol = parsedUrl.protocol
-
-  const options: http.RequestOptions = {}
-
-  let get: typeof https.get
-
-  if (!protocol) {
-    throw new Error('protocol is empty')
-  }
-
-  if (protocol.match(/^https:/i)) {
-    get           = https.get
-    options.agent = https.globalAgent
-  } else if (protocol.match(/^http:/i)) {
-    get           = http.get
-    options.agent = http.globalAgent
-  } else {
-    throw new Error('protocol unknown: ' + protocol)
-  }
-
-  options.headers = {
-    ...headers,
+  const options: http.RequestOptions = {
+    headers: { ...headers },
+    method: 'GET',
   }
 
-  const fileSize    = Number(headHeaders['content-length'])
+  const fileSize = Number(headHeaders['content-length'])
 
-  if (headHeaders['accept-ranges'] === 'bytes' && fileSize > HTTP_CHUNK_SIZE) {
-    return await downloadFileInChunks(get, url, options, fileSize, HTTP_CHUNK_SIZE)
+  if (!NO_SLICE_DOWN && headHeaders['accept-ranges'] === 'bytes' && fileSize > HTTP_CHUNK_SIZE) {
+    return await downloadFileInChunks(url, options, fileSize, HTTP_CHUNK_SIZE)
   } else {
-    return await downloadFile(get, url, options)
+    return await fetch(url, options)
   }
 }
 
-async function downloadFile (
-  get: typeof https.get,
-  url: string,
-  options: http.RequestOptions,
-): Promise<Readable> {
-  return new Promise<Readable>((resolve, reject) => {
-    let res: http.IncomingMessage | null = null
-    const req = get(url, options, (response) => {
-      res = response
-      resolve(res)
+async function fetch (url: string, options: http.RequestOptions): Promise<http.IncomingMessage> {
+  const { protocol } = new URL(url)
+  const { request, agent } = getProtocol(protocol)
+  const opts: http.RequestOptions = {
+    agent,
+    ...options,
+  }
+  const req = request(url, opts).end()
+  req
+    .on('error', () => {
+      req.destroy()
     })
-
-    req
-      .on('error', reject)
-      .setTimeout(HTTP_TIMEOUT, () => {
-        const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`)
-        if (res) {
-          res.emit('error', e)
-        }
-        req.emit('error', e)
-        req.destroy()
-      })
-      .end()
-  })
+    .setTimeout(HTTP_REQUEST_TIMEOUT, () => {
+      req.emit('error', new Error(`FileBox: Http request timeout (${HTTP_REQUEST_TIMEOUT})!`))
+    })
+  const responseEvents = await once(req, 'response')
+  const res = responseEvents[0] as http.IncomingMessage
+  res
+    .on('error', () => {
+      res.destroy()
+    })
+    .setTimeout(HTTP_RESPONSE_TIMEOUT, () => {
+      res.emit('error', new Error(`FileBox: Http response timeout (${HTTP_RESPONSE_TIMEOUT})!`))
+    })
+  return res
 }
 
 async function downloadFileInChunks (
-  get: typeof https.get,
   url: string,
   options: http.RequestOptions,
   fileSize: number,
   chunkSize = HTTP_CHUNK_SIZE,
 ): Promise<Readable> {
-  const ac = new AbortController()
-  const stream = new PassThrough()
-
-  const abortAc = () => {
-    if (!ac.signal.aborted) {
-      ac.abort()
-    }
+  const tmpFile = join(tmpdir(), `filebox-${randomUUID()}`)
+  const writeStream = createWriteStream(tmpFile)
+  const allowStatusCode = [ 200, 206 ]
+  const requestBaseOptions: http.RequestOptions = {
+    headers: {},
+    ...options,
   }
-
-  stream
-    .once('close', abortAc)
-    .once('error', abortAc)
-
-  const chunksCount = Math.ceil(fileSize / chunkSize)
-  let chunksDownloaded = 0
-  let dataTotalSize = 0
-
-  const doDownloadChunk = async function (i: number, retries: number) {
-    const start = i * chunkSize
-    const end = Math.min((i + 1) * chunkSize - 1, fileSize - 1)
+  let chunkSeq = 0
+  let start = 0
+  let end = 0
+  let downSize = 0
+  let retries = 3
+
+  while (downSize < fileSize) {
+    end = Math.min(start + chunkSize, fileSize - 1)
     const range = `bytes=${start}-${end}`
-
-    // console.info('doDownloadChunk() range:', range)
-
-    if (ac.signal.aborted) {
-      stream.destroy(new Error('Signal aborted.'))
-      return
-    }
-
-    const requestOptions: http.RequestOptions = {
-      ...options,
-      signal: ac.signal,
-      timeout: HTTP_TIMEOUT,
-    }
-    if (!requestOptions.headers) {
-      requestOptions.headers = {}
-    }
+    const requestOptions = Object.assign({}, requestBaseOptions)
+    assert(requestOptions.headers, 'Errors that should not happen: Invalid headers')
     requestOptions.headers['Range'] = range
 
     try {
-      const chunk = await downloadChunk(get, url, requestOptions, retries)
-      if (chunk.errored) {
-        throw new Error('chunk stream error')
-      }
-      if (chunk.closed) {
-        throw new Error('chunk stream closed')
+      const res = await fetch(url, requestOptions)
+      assert(allowStatusCode.includes(res.statusCode ?? 0), `Request failed with status code ${res.statusCode}`)
+      assert(Number(res.headers['content-length']) > 0, 'Server returned 0 bytes of data')
+      for await (const chunk of res) {
+        assert(Buffer.isBuffer(chunk))
+        downSize += chunk.length
+        writeStream.write(chunk)
       }
-      if (chunk.destroyed) {
-        throw new Error('chunk stream destroyed')
-      }
-      const buf = await streamToBuffer(chunk)
-      stream.push(buf)
-      chunksDownloaded++
-      dataTotalSize += buf.length
-
-      if (chunksDownloaded === chunksCount || dataTotalSize >= fileSize) {
-        stream.push(null)
-      }
-    } catch (err) {
-      if (retries === 0) {
-        stream.emit('error', err)
-      } else {
-        await doDownloadChunk(i, retries - 1)
+      res.destroy()
+    } catch (error) {
+      const err = error as Error
+      if (--retries <= 0) {
+        void rm(tmpFile, { force: true })
+        writeStream.close()
+        throw new Error(`Download file with chunk failed! ${err.message}`, { cause: err })
       }
     }
+    chunkSeq++
+    start = downSize
   }
+  writeStream.close()
 
-  const doDownloadAllChunks = async function () {
-    for (let i = 0; i < chunksCount; i++) {
-      if (ac.signal.aborted) {
-        return
-      }
-      await doDownloadChunk(i, 3)
-    }
-  }
-
-  void doDownloadAllChunks().catch((e) => {
-    stream.emit('error', e)
-  })
-
-  return stream
-}
-
-async function downloadChunk (
-  get: typeof https.get,
-  url: string,
-  requestOptions: http.RequestOptions,
-  retries: number,
-): Promise<Readable> {
-  return new Promise<Readable>((resolve, reject) => {
-    const doRequest = (attempt: number) => {
-      let resolved = false
-      const req = get(url, requestOptions, (res) => {
-        const statusCode = res.statusCode ?? 0
-        // console.info('downloadChunk(%d) statusCode: %d rsp.headers: %o', attempt, statusCode, res.headers)
-
-        if (statusCode < 200 || statusCode >= 300) {
-          if (attempt < retries) {
-            void doRequest(attempt + 1)
-          } else {
-            reject(new Error(`Request failed with status code ${res.statusCode}`))
-          }
-          return
-        }
-
-        const stream = pipeline(res, new PassThrough(), () => {})
-        resolve(stream)
-        resolved = true
-      })
-
-      req
-        .once('error', (err) => {
-          if (resolved) {
-            return
-          }
-          // console.error('downloadChunk(%d) req error:', attempt, err)
-          if (attempt < retries) {
-            void doRequest(attempt + 1)
-          } else {
-            reject(err)
-          }
-        })
-        .setTimeout(HTTP_TIMEOUT, () => {
-          const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`)
-          req.emit('error', e)
-          req.destroy()
-        })
-        .end()
-    }
-    void doRequest(0)
-  })
+  const readStream = createReadStream(tmpFile)
+  readStream
+    .once('end', () => readStream.close())
+    .once('close', () => {
+      void rm(tmpFile, { force: true })
+    })
+  return readStream
 }
 
 export async function streamToBuffer (stream: Readable): Promise<Buffer> {
-  return new Promise<Buffer>((resolve, reject) => {
-    const bufferList: Buffer[] = []
-    stream.once('error', reject)
-    stream.once('end', () => {
-      const fullBuffer = Buffer.concat(bufferList)
-      resolve(fullBuffer)
-    })
-    stream.on('data', (buffer) => bufferList.push(buffer))
-  })
+  const chunks: Buffer[] = []
+  for await (const chunk of stream) {
+    chunks.push(chunk)
+  }
+  return Buffer.concat(chunks)
 }

package/src/version.ts

@@ -1,4 +1,4 @@
 /**
  * This file was auto generated from scripts/generate-version.sh
  */
-export const VERSION: string = '1.7.4'
+export const VERSION: string = '1.7.5'