@nxtedition/lib 19.0.34 → 19.0.37

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/lib",
-  "version": "19.0.34",
+  "version": "19.0.37",
   "license": "MIT",
   "author": "Robert Nagy <robert.nagy@boffins.se>",
   "type": "module",

package/s3.js

@@ -1,116 +1,17 @@
 import crypto from 'node:crypto'
 import stream from 'node:stream'
-import path from 'node:path'
-import os from 'node:os'
-import fs from 'node:fs'
-import assert from 'node:assert'
 import AWS from '@aws-sdk/client-s3'
 import PQueue from 'p-queue'
 
 const CONTENT_MD5_EXPR = /^[A-F0-9]{32}$/i
 const CONTENT_LENGTH_EXPR = /^\d+$/i
 
-const noop = (arg0) => {}
-
-class PartUploader {
-  #number
-  #path
-  #size
-  #writable
-  #callback
-  #hasher
-  #signal
-  #logger
-
-  constructor({ dir, number, signal, logger }) {
-    this.#writable = null
-    this.#callback = noop
-    this.#hasher = crypto.createHash('md5')
-    this.#size = 0
-    this.#signal = signal
-    this.#number = number
-    this.#path = path.join(dir, `${this.#number}.part`)
-    this.#logger = logger?.child({ part: this.#number, path: this.#path })
-  }
-
-  get size() {
-    return this.#size
-  }
-
-  async write(chunk) {
-    if (!this.#writable) {
-      this.#writable = fs
-        .createWriteStream(this.#path, { signal: this.#signal })
-        .on('drain', () => {
-          this.#callback(null)
-          this.#callback = noop
-        })
-        .on('error', (err) => {
-          this.#callback(err)
-          this.callback = noop
-        })
-
-      this.#logger?.debug('created part')
-    }
-
-    if (this.#writable.errored) {
-      throw this.#writable.errored
-    }
-
-    this.#size += chunk.byteLength
-    this.#hasher.update(chunk)
-
-    if (!this.#writable.write(chunk)) {
-      await new Promise((resolve, reject) => {
-        this.#callback = (err) => (err ? reject(err) : resolve(null))
-      })
-      this.#signal.throwIfAborted()
-    }
-  }
-
-  async end(s3, params) {
-    try {
-      if (!this.#writable) {
-        throw new Error('No data to send')
-      }
-
-      if (this.#writable.errored) {
-        throw this.#writable.errored
-      }
-
-      this.#writable.end()
-      await stream.promises.finished(this.#writable)
-
-      assert(this.#writable.bytesWritten === this.#size, 'Expected size to match bytesWritten')
-
-      this.#logger?.debug('uploading part')
-      const { ETag } = await s3.send(
-        new AWS.UploadPartCommand({
-          ...params,
-          ContentMD5: this.#hasher.digest('base64'),
-          ContentLength: this.#size,
-          PartNumber: this.#number,
-          Body: fs.createReadStream(this.#path, { signal: this.#signal }),
-        }),
-      )
-      this.#logger?.debug({ etag: ETag }, 'uploaded part')
-
-      return { ETag, PartNumber: this.#number }
-    } finally {
-      await fs.promises.unlink(this.#writable.path)
-      this.#logger?.debug('deleted part')
-    }
-  }
-}
-
 export async function upload({
   client: s3,
-  signal: outerSignal,
+  signal,
   logger,
-  tmpdir = os.tmpdir(),
-  partSize = 64e6,
-  queueSize = 4,
-  leavePartsOnError = false,
+  partSize = 16e6,
+  queueSize = 2,
   params,
 }) {
   if (s3 == null) {
@@ -139,77 +40,122 @@ export async function upload({
     throw new Error(`Invalid ContentLength: ${ContentLength}`)
   }
 
-  outerSignal?.throwIfAborted()
-
   const queue = new PQueue({ concurrency: queueSize })
   const promises = []
-  const ac = new AbortController()
-  const signal = ac.signal
-
-  const abort = () => ac.abort()
-  outerSignal?.addEventListener('abort', abort)
 
   let uploadId
-  let uploadDir
   try {
-    uploadDir = await fs.promises.mkdtemp(path.join(tmpdir, 's3-upload-'))
-    logger?.debug({ uploadDir }, 'created upload directory')
-    signal.throwIfAborted()
-
     const multipartUploadOutput = await s3.send(
-      new AWS.CreateMultipartUploadCommand({
-        Bucket,
-        Key,
-      }),
+      new AWS.CreateMultipartUploadCommand({ Bucket, Key }),
+      { abortSignal: signal },
     )
     uploadId = multipartUploadOutput.UploadId
     logger = logger?.child({ uploadId })
     logger?.debug('created multipart upload')
-    signal.throwIfAborted()
 
     const uploader = {
       size: 0,
       hasher: crypto.createHash('md5'),
-      part: new PartUploader({ dir: uploadDir, number: 1, signal, logger }),
-      number: 1,
+      part: {
+        /** @type {Array<Buffer>} **/ chunks: [],
+        hasher: crypto.createHash('md5'),
+        size: 0,
+        number: 1,
+      },
     }
 
     const maybeFlush = (minSize) => {
-      if (uploader.part.size && (minSize == null || uploader.part.size >= minSize)) {
-        const part = uploader.part
-        uploader.part = new PartUploader({
-          dir: uploadDir,
-          number: ++uploader.number,
-          logger,
-          signal,
-        })
-
-        const promise = queue.add(() => part.end(s3, { Bucket, Key, UploadId: uploadId }))
-        promise.catch((err) => {
-          ac.abort(err)
-        })
-        promises.push(promise)
+      const { part } = uploader
+
+      if (!part.size) {
+        return
+      }
+
+      if (minSize != null && part.size < minSize) {
+        return
       }
+
+      const chunks = part.chunks
+      const number = part.number
+      const size = part.size
+      const hasher = part.hasher
+
+      part.chunks = []
+      part.number += 1
+      part.size = 0
+      part.hasher = crypto.createHash('md5')
+
+      promises.push(
+        queue
+          .add(
+            async ({ signal }) => {
+              logger?.debug({ number, size }, 'part upload started')
+              try {
+                const { ETag } = await s3.send(
+                  new AWS.UploadPartCommand({
+                    Bucket,
+                    Key,
+                    UploadId: uploadId,
+                    ContentMD5: hasher.digest('base64'),
+                    ContentLength: size,
+                    PartNumber: number,
+                    Body: new stream.Readable({
+                      read() {
+                        for (const chunk of chunks.splice(0)) {
+                          this.push(chunk)
+                        }
+                        this.push(null)
+                      },
+                    }),
+                  }),
+                  { abortSignal: signal },
+                )
+                logger?.debug({ number, size, etag: ETag }, 'part upload completed')
+                return { part: { ETag, PartNumber: number } }
+              } catch (err) {
+                logger?.debug({ err }, 'part upload failed')
+                return { error: err }
+              }
+            },
+            { signal },
+          )
+          .catch((err) => ({ error: err })),
+      )
+
+      return queue.onEmpty()
     }
 
     for await (const chunk of Body) {
-      signal.throwIfAborted()
+      signal?.throwIfAborted()
 
       uploader.hasher.update(chunk)
       uploader.size += chunk.byteLength
 
-      const thenable = uploader.part.write(chunk)
+      uploader.part.hasher.update(chunk)
+      uploader.part.chunks.push(chunk)
+      uploader.part.size += chunk.byteLength
+
+      const thenable = maybeFlush(partSize)
       if (thenable) {
         await thenable
-        signal.throwIfAborted()
       }
-
-      maybeFlush(partSize)
     }
-    maybeFlush()
+    await maybeFlush()
+
+    const parts = []
+    const errors = []
+    for (const { part, error } of await Promise.all(promises)) {
+      if (error) {
+        errors.push(error)
+      } else if (part) {
+        parts.push(part)
+      }
+    }
+    signal?.throwIfAborted()
 
-    const parts = await Promise.all(promises)
-    signal.throwIfAborted()
+    if (errors.length > 0) {
+      throw new AggregateError(errors, 'upload failed')
+    }
 
     if (parts.length === 0) {
       throw new Error('upload empty')
@@ -222,7 +168,9 @@ export async function upload({
         UploadId: uploadId,
         MultipartUpload: { Parts: parts },
       }),
+      { abortSignal: signal },
     )
+    signal?.throwIfAborted()
 
     const result = {
       size: uploader.size,
@@ -231,10 +179,6 @@ export async function upload({
       parts,
     }
 
-    logger?.debug(result, 'completed multipart upload')
-
-    signal.throwIfAborted()
-
     const size = ContentLength != null ? Number(ContentLength) : null
     const hash = ContentMD5
 
@@ -246,19 +190,15 @@ export async function upload({
       throw new Error(`Expected hash ${hash} but got ${result.hash}`)
     }
 
+    logger?.debug(result, 'completed multipart upload')
+
     return result
   } catch (err) {
-    ac.abort(err)
+    logger?.error({ err }, 'failed multipart upload')
 
-    if (uploadId && !leavePartsOnError) {
+    if (uploadId) {
       try {
-        await s3.send(
-          new AWS.AbortMultipartUploadCommand({
-            Bucket,
-            Key,
-            UploadId: uploadId,
-          }),
-        )
+        await s3.send(new AWS.AbortMultipartUploadCommand({ Bucket, Key, UploadId: uploadId }))
         logger?.warn('aborted multipart upload')
       } catch (er) {
         throw new AggregateError([err, er])
@@ -266,10 +206,5 @@ export async function upload({
     }
 
     throw err
-  } finally {
-    outerSignal?.removeEventListener('abort', abort)
-    if (uploadDir) {
-      await fs.promises.rmdir(uploadDir, { recursive: true })
-    }
   }
 }