@juzi/file-box 1.7.1 → 1.7.3

package/src/misc.ts

@@ -1,9 +1,10 @@
 import http    from 'http'
 import https   from 'https'
 import { URL } from 'url'
-import type stream   from 'stream'
 
-import { HTTP_TIMEOUT } from './config.js'
+import { PassThrough, pipeline, Readable } from 'stream'
+
+import { HTTP_CHUNK_SIZE, HTTP_TIMEOUT } from './config.js'
 
 export function dataUrlToBase64 (dataUrl: string): string {
   const dataList = dataUrl.split(',')
@@ -96,10 +97,10 @@ export function httpHeaderToFileName (
 export async function httpStream (
   url     : string,
   headers : http.OutgoingHttpHeaders = {},
-): Promise<http.IncomingMessage> {
+): Promise<Readable> {
   const parsedUrl = new URL(url)
 
-  const protocol  = parsedUrl.protocol
+  const protocol = parsedUrl.protocol
 
   const options: http.RequestOptions = {}
 
@@ -123,12 +124,29 @@ export async function httpStream (
     ...headers,
   }
 
-  return new Promise<http.IncomingMessage>((resolve, reject) => {
+  const headHeaders = await httpHeadHeader(url)
+  const fileSize    = Number(headHeaders['content-length'])
+
+  if (headHeaders['accept-ranges'] === 'bytes' && fileSize > HTTP_CHUNK_SIZE) {
+    return await downloadFileInChunks(get, url, options, fileSize, HTTP_CHUNK_SIZE)
+  } else {
+    return await downloadFile(get, url, options)
+  }
+}
+
+async function downloadFile (
+  get: typeof https.get,
+  url: string,
+  options: http.RequestOptions,
+): Promise<Readable> {
+  return new Promise<Readable>((resolve, reject) => {
     let res: http.IncomingMessage | null = null
-    const req = get(parsedUrl, options, (response) => {
+    const req = get(url, options, (response) => {
       res = response
       resolve(res)
     })
+
+    req
       .on('error', reject)
       .setTimeout(HTTP_TIMEOUT, () => {
         const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`)
@@ -142,9 +160,147 @@ export async function httpStream (
   })
 }
 
-export async function streamToBuffer (
-  stream: stream.Readable,
-): Promise<Buffer> {
+async function downloadFileInChunks (
+  get: typeof https.get,
+  url: string,
+  options: http.RequestOptions,
+  fileSize: number,
+  chunkSize = HTTP_CHUNK_SIZE,
+): Promise<Readable> {
+  const ac = new AbortController()
+  const stream = new PassThrough()
+
+  const abortAc = () => {
+    if (!ac.signal.aborted) {
+      ac.abort()
+    }
+  }
+
+  stream
+    .once('close', abortAc)
+    .once('error', abortAc)
+
+  const chunksCount = Math.ceil(fileSize / chunkSize)
+  let chunksDownloaded = 0
+  let dataTotalSize = 0
+
+  const doDownloadChunk = async function (i: number, retries: number) {
+    const start = i * chunkSize
+    const end = Math.min((i + 1) * chunkSize - 1, fileSize - 1)
+    const range = `bytes=${start}-${end}`
+
+    // console.info('doDownloadChunk() range:', range)
+
+    if (ac.signal.aborted) {
+      stream.destroy(new Error('Signal aborted.'))
+      return
+    }
+
+    const requestOptions: http.RequestOptions = {
+      ...options,
+      signal: ac.signal,
+      timeout: HTTP_TIMEOUT,
+    }
+    if (!requestOptions.headers) {
+      requestOptions.headers = {}
+    }
+    requestOptions.headers['Range'] = range
+
+    try {
+      const chunk = await downloadChunk(get, url, requestOptions, retries)
+      if (chunk.errored) {
+        throw new Error('chunk stream error')
+      }
+      if (chunk.closed) {
+        throw new Error('chunk stream closed')
+      }
+      if (chunk.destroyed) {
+        throw new Error('chunk stream destroyed')
+      }
+      const buf = await streamToBuffer(chunk)
+      stream.push(buf)
+      chunksDownloaded++
+      dataTotalSize += buf.length
+
+      if (chunksDownloaded === chunksCount || dataTotalSize >= fileSize) {
+        stream.push(null)
+      }
+    } catch (err) {
+      if (retries === 0) {
+        stream.emit('error', err)
+      } else {
+        await doDownloadChunk(i, retries - 1)
+      }
+    }
+  }
+
+  const doDownloadAllChunks = async function () {
+    for (let i = 0; i < chunksCount; i++) {
+      if (ac.signal.aborted) {
+        return
+      }
+      await doDownloadChunk(i, 3)
+    }
+  }
+
+  void doDownloadAllChunks().catch((e) => {
+    stream.emit('error', e)
+  })
+
+  return stream
+}
+
+async function downloadChunk (
+  get: typeof https.get,
+  url: string,
+  requestOptions: http.RequestOptions,
+  retries: number,
+): Promise<Readable> {
+  return new Promise<Readable>((resolve, reject) => {
+    const doRequest = (attempt: number) => {
+      let resolved = false
+      const req = get(url, requestOptions, (res) => {
+        const statusCode = res.statusCode ?? 0
+        // console.info('downloadChunk(%d) statusCode: %d rsp.headers: %o', attempt, statusCode, res.headers)
+
+        if (statusCode < 200 || statusCode >= 300) {
+          if (attempt < retries) {
+            void doRequest(attempt + 1)
+          } else {
+            reject(new Error(`Request failed with status code ${res.statusCode}`))
+          }
+          return
+        }
+
+        const stream = pipeline(res, new PassThrough(), () => {})
+        resolve(stream)
+        resolved = true
+      })
+
+      req
+        .once('error', (err) => {
+          if (resolved) {
+            return
+          }
+          // console.error('downloadChunk(%d) req error:', attempt, err)
+          if (attempt < retries) {
+            void doRequest(attempt + 1)
+          } else {
+            reject(err)
+          }
+        })
+        .setTimeout(HTTP_TIMEOUT, () => {
+          const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`)
+          req.emit('error', e)
+          req.destroy()
+        })
+        .end()
+    }
+    void doRequest(0)
+  })
+}
+
+export async function streamToBuffer (stream: Readable): Promise<Buffer> {
   return new Promise<Buffer>((resolve, reject) => {
     const bufferList: Buffer[] = []
     stream.once('error', reject)
@@ -152,6 +308,6 @@ export async function streamToBuffer (
       const fullBuffer = Buffer.concat(bufferList)
       resolve(fullBuffer)
     })
-    stream.on('data', buffer => bufferList.push(buffer))
+    stream.on('data', (buffer) => bufferList.push(buffer))
   })
 }

package/src/version.ts

@@ -1,4 +1,4 @@
 /**
  * This file was auto generated from scripts/generate-version.sh
  */
-export const VERSION: string = '1.7.1'
+export const VERSION: string = '1.7.3'