@nxtedition/lib 19.0.28 → 19.0.29

package/errors.js

@@ -47,7 +47,7 @@ export function serializeError(error) {
   }
 
   if (typeof error === 'string') {
-    return serializeError({ message: error })
+    return [serializeError({ message: error })]
   }
 
   if (Buffer.isBuffer(error)) {
@@ -56,7 +56,7 @@ export function serializeError(error) {
 
   if (Array.isArray(error)) {
     const errors = error.map(serializeError).filter(Boolean)
-    return errors.length === 0 ? null : errors.length === 1 ? errors[0] : errors
+    return errors
   }
 
   if (Object.prototype.hasOwnProperty.call(error, kSeen)) {
@@ -105,26 +105,28 @@ export function serializeError(error) {
     signalCode = SIGNALS[signalCode] ?? signalCode
   }
 
-  errors = Array.isArray(errors) ? errors.map(serializeError) : undefined
+  errors = Array.isArray(errors) ? errors.map(serializeError).filter(Boolean) : undefined
   cause = cause ? serializeError(cause) : undefined
 
   delete error[kSeen]
 
-  return JSON.parse(
-    JSON.stringify({
-      ...properties,
-      message,
-      type,
-      code,
-      exitCode,
-      signalCode,
-      statusCode,
-      headers,
-      data,
-      cause,
-      errors,
-    }),
-  )
+  return [
+    JSON.parse(
+      JSON.stringify({
+        ...properties,
+        message,
+        type,
+        code,
+        exitCode,
+        signalCode,
+        statusCode,
+        headers,
+        data,
+        cause,
+        errors,
+      }),
+    ),
+  ]
 }
 
 // TODO (fix): Recursion guard?

package/merge-ranges.js

@@ -29,6 +29,10 @@ export default function mergeRanges(ranges) {
     const range = ranges[n]
     const top = stack[stack.length - 1]
 
+    if (range.length !== 2 || !Number.isFinite(range[0]) || !Number.isFinite(range[1])) {
+      continue
+    }
+
     if (top[1] < range[0]) {
       // No overlap, push range onto stack
       stack.push([range[0], range[1]])

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/lib",
-  "version": "19.0.28",
+  "version": "19.0.29",
   "license": "MIT",
   "author": "Robert Nagy <robert.nagy@boffins.se>",
   "type": "module",
@@ -16,6 +16,7 @@
     "elasticsearch.js",
     "merge-ranges.js",
     "http.js",
+    "s3.js",
     "deepstream.js",
     "logger.js",
     "mime.js",
@@ -35,7 +36,7 @@
     "prepare": "husky install",
     "prepublishOnly": "pinst --disable",
     "postpublish": "pinst --enable",
-    "test": "node --test __tests__ && npx tap test"
+    "test": "node --test"
   },
   "lint-staged": {
     "*.{js,jsx,md,ts}": [
@@ -74,6 +75,7 @@
     "/__tests__"
   ],
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.540.0",
     "@elastic/elasticsearch": "^8.12.2",
     "@elastic/transport": "^8.4.1",
     "@nxtedition/nxt-undici": "^2.0.41",
@@ -89,6 +91,7 @@
     "nconf": "^0.12.1",
     "nested-error-stacks": "^2.1.1",
     "object-hash": "^3.0.0",
+    "p-queue": "^8.0.1",
     "pino": "^8.19.0",
     "pino-std-serializers": "^6.2.2",
     "qs": "^6.12.0",
@@ -116,8 +119,7 @@
     "pinst": "^3.0.0",
     "prettier": "^3.2.5",
     "rxjs": "^7.5.6",
-    "send": "^0.18.0",
-    "tap": "^18.7.1"
+    "send": "^0.18.0"
   },
   "peerDependencies": {
     "@elastic/elasticsearch": "^8.6.0",

package/s3.js

@@ -0,0 +1,258 @@
+import crypto from 'node:crypto'
+import stream from 'node:stream'
+import path from 'node:path'
+import os from 'node:os'
+import fs from 'node:fs'
+import assert from 'node:assert'
+import AWS from '@aws-sdk/client-s3'
+import PQueue from 'p-queue'
+
+const CONTENT_MD5_EXPR = /^[A-F0-9]{32}$/i
+const CONTENT_LENGTH_EXPR = /^\d+$/i
+
+const noop = (arg0) => {}
+
+class PartUploader {
+  #number
+  #path
+  #size
+  #writable
+  #callback
+  #hasher
+  #signal
+
+  constructor(dir, number, signal) {
+    this.#writable = null
+    this.#callback = noop
+    this.#hasher = crypto.createHash('md5')
+    this.#size = 0
+    this.#signal = signal
+    this.#number = number
+    this.#path = path.join(dir, `${this.#number}.part`)
+  }
+
+  get size() {
+    return this.#size
+  }
+
+  async write(chunk) {
+    this.#writable ??= fs
+      .createWriteStream(this.#path, { signal: this.#signal })
+      .on('drain', () => {
+        this.#callback(null)
+        this.#callback = noop
+      })
+      .on('error', (err) => {
+        this.#callback(err)
+        this.callback = noop
+      })
+
+    if (this.#writable.errored) {
+      throw this.#writable.errored
+    }
+
+    this.#size += chunk.byteLength
+    this.#hasher.update(chunk)
+
+    if (!this.#writable.write(chunk)) {
+      await new Promise((resolve, reject) => {
+        this.#callback = (err) => (err ? reject(err) : resolve(null))
+      })
+      this.#signal.throwIfAborted()
+    }
+  }
+
+  async end(s3, params) {
+    try {
+      if (!this.#writable) {
+        throw new Error('No data to send')
+      }
+
+      if (this.#writable.errored) {
+        throw this.#writable.errored
+      }
+
+      this.#writable.end()
+      await stream.promises.finished(this.#writable)
+
+      assert(this.#writable.bytesWritten === this.#size, 'Expected size to match bytesWritten')
+
+      const { ETag } = await s3.send(
+        new AWS.UploadPartCommand({
+          ...params,
+          ContentMD5: this.#hasher.digest('base64'),
+          ContentLength: this.#size,
+          PartNumber: this.#number,
+          Body: fs.createReadStream(this.#path, { signal: this.#signal }),
+        }),
+      )
+
+      return { part: { ETag, PartNumber: this.#number } }
+    } catch (err) {
+      return { error: err }
+    } finally {
+      await fs.promises.unlink(this.#writable.path)
+    }
+  }
+}
+
+export async function upload(
+  {
+    client: s3,
+    signal: outerSignal,
+    tmpdir = os.tmpdir(),
+    partSize = 64e6,
+    queueSize = 4,
+    leavePartsOnError = false,
+  },
+  { Body, Key, Bucket, ContentMD5, ContentLength },
+) {
+  if (s3 == null) {
+    throw new Error('Invalid client')
+  }
+
+  if (!Number.isFinite(partSize) || partSize <= 0) {
+    throw new Error('Invalid partSize')
+  }
+
+  if (!Number.isFinite(queueSize) || queueSize <= 0) {
+    throw new Error('Invalid queueSize')
+  }
+
+  if (ContentMD5 != null && !CONTENT_MD5_EXPR.test(ContentMD5)) {
+    throw new Error(`Invalid ContentMD5: ${ContentMD5}`)
+  }
+
+  if (ContentLength != null && !CONTENT_LENGTH_EXPR.test(ContentLength)) {
+    throw new Error(`Invalid ContentLength: ${ContentLength}`)
+  }
+
+  const dir = await fs.promises.mkdtemp(path.join(tmpdir, 's3-upload-'))
+  outerSignal?.throwIfAborted()
+
+  await fs.promises.stat(dir)
+  outerSignal?.throwIfAborted()
+
+  const queue = new PQueue({ concurrency: queueSize })
+  const promises = []
+  const ac = new AbortController()
+  const signal = ac.signal
+
+  const abort = () => ac.abort()
+  outerSignal?.addEventListener('abort', abort)
+
+  let uploadId
+  try {
+    const multipartUploadOutput = await s3.send(
+      new AWS.CreateMultipartUploadCommand({
+        Bucket,
+        Key,
+      }),
+    )
+    uploadId = multipartUploadOutput.UploadId
+    signal.throwIfAborted()
+
+    const uploader = {
+      size: 0,
+      hasher: crypto.createHash('md5'),
+      part: new PartUploader(dir, 1, signal),
+      number: 1,
+    }
+
+    const maybeFlush = (minSize) => {
+      if (uploader.part.size && (minSize == null || uploader.part.size >= minSize)) {
+        const part = uploader.part
+        uploader.part = new PartUploader(dir, ++uploader.number, signal)
+
+        const promise = queue.add(() => part.end(s3, { Bucket, Key, UploadId: uploadId }))
+        promises.push(promise)
+      }
+    }
+
+    for await (const chunk of Body) {
+      signal.throwIfAborted()
+
+      uploader.hasher.update(chunk)
+      uploader.size += chunk.byteLength
+
+      const thenable = uploader.part.write(chunk)
+      if (thenable) {
+        await thenable
+        signal.throwIfAborted()
+      }
+
+      maybeFlush(partSize)
+    }
+    maybeFlush()
+
+    const parts = []
+    const errors = []
+    for (const { part, error } of await Promise.all(promises)) {
+      if (error) {
+        errors.push(error)
+      } else {
+        parts.push(part)
+      }
+    }
+    signal.throwIfAborted()
+
+    if (errors.length > 0) {
+      throw new AggregateError(errors, 'upload failed')
+    }
+
+    if (parts.length === 0) {
+      throw new Error('upload empty')
+    }
+
+    const uploadOutput = await s3.send(
+      new AWS.CompleteMultipartUploadCommand({
+        Bucket,
+        Key,
+        UploadId: uploadId,
+        MultipartUpload: { Parts: parts },
+      }),
+    )
+    signal.throwIfAborted()
+
+    const result = {
+      size: uploader.size,
+      hash: uploader.hasher.digest('hex'),
+      output: uploadOutput,
+      parts,
+    }
+
+    const size = ContentLength != null ? Number(ContentLength) : null
+    const hash = ContentMD5
+
+    if (size != null && size !== result.size) {
+      throw new Error(`Expected size ${size} but got ${result.size}`)
+    }
+
+    if (hash != null && hash !== result.hash) {
+      throw new Error(`Expected hash ${hash} but got ${result.hash}`)
+    }
+
+    return result
+  } catch (err) {
+    ac.abort(err)
+
+    if (uploadId && !leavePartsOnError) {
+      try {
+        await s3.send(
+          new AWS.AbortMultipartUploadCommand({
+            Bucket,
+            Key,
+            UploadId: uploadId,
+          }),
+        )
+      } catch (er) {
+        throw new AggregateError([err, er])
+      }
+    }
+
+    throw err
+  } finally {
+    outerSignal?.removeEventListener('abort', abort)
+    await fs.promises.rmdir(dir, { recursive: true })
+  }
+}

package/util/template/javascript.js

@@ -26,7 +26,6 @@ class TimerEntry {
 
   dispose() {
     clearTimeout(this.timer)
-
     this.timer = null
   }
 }
@@ -232,6 +231,7 @@ export default function ({ ds, proxify, compiler }) {
       this._subscription = null
       this._args = kEmpty
       this._wrap = null
+      this._suspended = false
 
       if (rxjs.isObservable(args)) {
         this._subscription = args.subscribe({
@@ -262,32 +262,40 @@ export default function ({ ds, proxify, compiler }) {
       throw kSuspend
     }
 
-    fetch(url, init, suspend = true) {
-      return this._getFetch(url, init, suspend)
+    fetch(resource, options, suspend = true) {
+      return this._getFetch(resource, options, suspend)
     }
 
     observe(observable, suspend = true) {
       return this._getObservable(observable, suspend)
     }
 
-    wait(promise, suspend = true) {
-      return this._getWait(promise, suspend)
+    then(promise, suspend = true) {
+      return this._getPromise(promise, suspend)
     }
 
     ds(id, state, suspend = true) {
       return this._getRecord(id, state, suspend)
     }
 
-    _ds(key, postfix, state, suspend = true) {
-      return !key || typeof key !== 'string'
-        ? null
-        : this._getRecord(postfix ? key + postfix : key, state, suspend)
+    timer(dueTime, dueValue = dueTime, suspend = true) {
+      return this._getTimer(dueTime, dueValue, suspend)
     }
 
     asset(id, type, state, suspend = true) {
       return this._getHasRawAssetType(id, type, state, suspend)
     }
 
+    hash(value) {
+      return objectHash(value)
+    }
+
+    _ds(key, postfix, state, suspend) {
+      return !key || typeof key !== 'string'
+        ? null
+        : this._getRecord(postfix ? key + postfix : key, state, suspend)
+    }
+
     _asset(id, type, state, suspend) {
       if (!type || typeof type !== 'string') {
         throw new Error(`invalid argument: type (${type})`)
@@ -298,14 +306,6 @@ export default function ({ ds, proxify, compiler }) {
         : this._getHasRawAssetType(id, type, state, suspend)
     }
 
-    timer(dueTime, dueValue, suspend = true) {
-      return this._getTimer(dueTime, dueValue, suspend)
-    }
-
-    hash(value) {
-      return objectHash(value)
-    }
-
     _destroy() {
       this._destroyed = true
       this._subscription?.unsubscribe()
@@ -341,7 +341,12 @@ export default function ({ ds, proxify, compiler }) {
       compiler.current = self
 
       try {
+        assert(self._suspended === false)
         const value = self._script.runInContext(self._context)
+        if (self._suspended) {
+          return
+        }
+
         if (value !== self._value) {
           self._value = value
           self._observer.next(value)
@@ -363,6 +368,7 @@ export default function ({ ds, proxify, compiler }) {
         self._context.$ = null
         self._context.nxt = null
 
+        self._suspended = false
         self._disposing = true
 
         if (self._entries) {
@@ -419,6 +425,7 @@ export default function ({ ds, proxify, compiler }) {
       }
 
       if (!entry.status) {
+        this._suspended = true
         if (suspend ?? true) {
           throw kSuspend
         } else {
@@ -445,6 +452,7 @@ export default function ({ ds, proxify, compiler }) {
       }
 
       if (entry.value === kEmpty) {
+        this._suspended = true
         if (suspend ?? true) {
           throw kSuspend
         } else {
@@ -455,7 +463,7 @@ export default function ({ ds, proxify, compiler }) {
       return entry.value
     }
 
-    _getWait(promise, suspend) {
+    _getPromise(promise, suspend) {
       if (typeof promise?.then !== 'function') {
         throw new Error(`invalid argument: Promise (${promise})`)
       }
@@ -471,6 +479,7 @@ export default function ({ ds, proxify, compiler }) {
       }
 
       if (entry.value === kEmpty) {
+        this._suspended = true
         if (suspend ?? true) {
           throw kSuspend
         } else {
@@ -489,12 +498,12 @@ export default function ({ ds, proxify, compiler }) {
       const timeout = dueTime - Date.now()
 
       if (Number.isFinite(dueTime) && timeout > 0) {
+        this._suspended = true
         this._getEntry(key, TimerEntry, timeout)
-
         if (suspend ?? true) {
           throw kSuspend
         } else {
-          return null
+          return dueValue
         }
       }
 
@@ -521,10 +530,11 @@ export default function ({ ds, proxify, compiler }) {
       const entry = this._getEntry(key, RecordEntry, ds)
 
       if (entry.record.state < state) {
+        this._suspended = true
         if (suspend ?? true) {
           throw kSuspend
         } else {
-          return null
+          return entry.record.data
         }
       }
 
@@ -540,11 +550,7 @@ export default function ({ ds, proxify, compiler }) {
         throw new Error(`invalid argument: type (${type})`)
       }
 
-      const data = this._getRecord(
-        id + ':asset.rawTypes?',
-        state ?? ds.record.PROVIDER,
-        suspend ?? true,
-      )
+      const data = this._getRecord(id + ':asset.rawTypes?', state ?? ds.record.PROVIDER, suspend)
       return data && Array.isArray(data.value) && data.value.includes(type) ? id : null
     }
   }