@nxtedition/lib 19.0.42 → 19.0.44

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/lib",
-  "version": "19.0.42",
+  "version": "19.0.44",
   "license": "MIT",
   "author": "Robert Nagy <robert.nagy@boffins.se>",
   "type": "module",

package/s3.js

@@ -1,20 +1,15 @@
 import crypto from 'node:crypto'
 import stream from 'node:stream'
-import assert from 'node:assert'
+import tp from 'node:timers/promises'
 import AWS from '@aws-sdk/client-s3'
 import PQueue from 'p-queue'
 
 const CONTENT_MD5_EXPR = /^[A-F0-9]{32}$/i
 const CONTENT_LENGTH_EXPR = /^\d+$/i
 
-export async function upload({
-  client: s3,
-  signal,
-  logger,
-  partSize = 16e6,
-  queueSize = 2,
-  params,
-}) {
+const queue = new PQueue({ concurrency: 16 })
+
+export async function upload({ client: s3, signal, logger, partSize = 16e6, params }) {
   if (s3 == null) {
     throw new Error('Invalid client')
   }
@@ -23,10 +18,6 @@ export async function upload({
     throw new Error('Invalid partSize')
   }
 
-  if (!Number.isFinite(queueSize) || queueSize <= 0 || queueSize > 32) {
-    throw new Error('Invalid queueSize')
-  }
-
   if (params == null || typeof params !== 'object') {
     throw new Error('Invalid params')
   }
@@ -41,11 +32,8 @@ export async function upload({
     throw new Error(`Invalid ContentLength: ${ContentLength}`)
   }
 
-  const queue = new PQueue({ concurrency: queueSize })
   const promises = []
 
-  assert(queue.concurrency > 0 && queue.concurrency <= 32)
-
   const ac = new AbortController()
   const onAbort = () => {
     ac.abort()
@@ -102,32 +90,40 @@ export async function upload({
         queue
           .add(
             async () => {
-              logger?.debug({ number, size }, 'part upload started')
-              try {
-                const { ETag } = await s3.send(
-                  new AWS.UploadPartCommand({
-                    Bucket,
-                    Key,
-                    UploadId: uploadId,
-                    ContentMD5: hasher.digest('base64'),
-                    ContentLength: size,
-                    PartNumber: number,
-                    Body: new stream.Readable({
-                      read() {
-                        for (const chunk of chunks.splice(0)) {
-                          this.push(chunk)
-                        }
-                        this.push(null)
-                      },
+              for (let retryCount = 0; true; retryCount++) {
+                logger?.debug({ number, size }, 'part upload started')
+                try {
+                  const { ETag } = await s3.send(
+                    new AWS.UploadPartCommand({
+                      Bucket,
+                      Key,
+                      UploadId: uploadId,
+                      ContentMD5: hasher.digest('base64'),
+                      ContentLength: size,
+                      PartNumber: number,
+                      Body: new stream.Readable({
+                        read() {
+                          for (const chunk of chunks.splice(0)) {
+                            this.push(chunk)
+                          }
+                          this.push(null)
+                        },
+                      }),
                     }),
-                  }),
-                )
-                logger?.debug({ number, size, etag: ETag }, 'part upload completed')
-                return { part: { ETag, PartNumber: number } }
-              } catch (err) {
-                uploader.ac.abort(err)
-                logger?.warn({ err }, 'part upload failed')
-                return { error: err }
+                  )
+                  logger?.debug({ number, size, etag: ETag }, 'part upload completed')
+                  return { part: { ETag, PartNumber: number } }
+                } catch (err) {
+                  logger?.warn({ err }, 'part upload failed')
+
+                  if (retryCount < 3) {
+                    logger?.warn({ retryCount }, 'part upload retry')
+                    await tp.setTimeout(1e3, undefined, { signal: uploader.signal })
+                  } else {
+                    uploader.ac.abort(err)
+                    return { error: err }
+                  }
+                }
               }
             },
             { signal: uploader.signal },