@nxtedition/lib 19.1.1 → 19.1.3

package/couch.js

@@ -6,9 +6,10 @@ import { makeWeakCache } from './weakCache.js'
 import tp from 'timers/promises'
 import { defaultDelay as delay } from './http.js'
 import urljoin from 'url-join'
-import undici from 'undici'
+import undici, { util as undiciUtil } from 'undici'
 import { AbortError } from './errors.js'
 import split2 from 'split2'
+import { parse as parseContentType } from 'content-type'
 
 // https://github.com/fastify/fastify/blob/main/lib/reqIdGenFactory.js
 // 2,147,483,647 (2^31 − 1) stands for max SMI value (an internal optimization of V8).
@@ -886,22 +887,15 @@ function _normalizeAllDocsOptions({
   }
 }
 
-class Handler extends undici.DecoratorHandler {
-  #str
-  #decoder
-  #onAbort
+class SignalHandler extends undici.DecoratorHandler {
   #signal
-  #statusCode
-  #statusText
-  #opts
   #handler
+  #abort
 
-  constructor({ opts, signal, handler }) {
+  constructor({ signal, handler }) {
     super(handler)
-
-    this.#handler = handler
-    this.#opts = opts
     this.#signal = signal
+    this.#handler = handler
   }
 
   onConnect(abort) {
@@ -909,50 +903,94 @@ class Handler extends undici.DecoratorHandler {
       abort(this.#signal.reason)
     } else {
       if (this.#signal) {
-        this.#onAbort = () => abort(this.#signal.reason)
-        this.#signal.addEventListener('abort', this.#onAbort)
+        this.#abort = () => abort(this.#signal.reason)
+        this.#signal.addEventListener('abort', this.#abort)
       }
-
-      this.#str = ''
-      this.#decoder = null
       this.#handler.onConnect?.(abort)
     }
   }
 
-  onHeaders(statusCode, rawHeaders, resume, statusText) {
+  onComplete(...args) {
+    this.#signal?.removeEventListener('abort', this.#abort)
+    this.#handler.onComplete?.(...args)
+  }
+
+  onError(...args) {
+    this.#signal?.removeEventListener('abort', this.#abort)
+    this.#handler.onError?.(...args)
+  }
+}
+
+class ErrorHandler extends undici.DecoratorHandler {
+  #str
+  #decoder
+  #statusCode
+  #statusText
+  #contentType
+  #opts
+  #handler
+  #error
+  #headers
+
+  constructor({ opts, handler }) {
+    super(handler)
+
+    this.#handler = handler
+    this.#opts = opts
+  }
+
+  onConnect(...args) {
+    this.#str = ''
+    this.#decoder = null
+    this.#handler.onConnect?.(...args)
+  }
+
+  onHeaders(statusCode, headers, resume, statusText) {
     this.#statusCode = statusCode
     this.#statusText = statusText
 
-    if (this.#statusCode >= 200 && this.#statusCode < 300) {
-      this.#handler.onHeaders?.(statusCode, rawHeaders, resume, statusText)
-    } else {
+    if (this.#statusCode < 200 || this.#statusCode > 300) {
+      this.#headers = Array.isArray(headers) ? undiciUtil.parseHeaders(headers) : headers
+
+      this.#error = new Error(this.#statusText ?? this.#statusCode)
+
+      this.#contentType = parseContentType(this.#headers['content-type'] ?? '')?.type
+      if (this.#contentType !== 'application/json' && this.#contentType !== 'plain/text') {
+        throw this.#error
+      }
+
       this.#decoder = new TextDecoder()
+    } else {
+      return this.#handler.onHeaders?.(statusCode, headers, resume, statusText)
     }
   }
 
   onData(data) {
-    if (this.#decoder) {
-      this.#str += this.#decoder.decode(data, { stream: true })
+    if (this.#error) {
+      this.#str += this.#decoder?.decode(data, { stream: true }) ?? ''
     } else {
       return this.#handler?.onData(data)
     }
   }
 
-  onComplete(trailers) {
-    if (this.#decoder) {
-      let body
-      let reason
-      let error
-      try {
-        body = JSON.parse(this.#str)
-        reason = body.reason
-        error = body.error
-      } catch {
-        // Do nothing...
-      }
+  onComplete(...args) {
+    if (this.#error) {
+      this.#str += this.#decoder?.decode(undefined, { stream: false }) ?? ''
+
+      if (this.#contentType === 'application/json') {
+        let body
+        let reason
+        let error
+        try {
+          body = JSON.parse(this.#str)
+          reason = body.reason
+          error = body.error
+        } catch {
+          body = this.#str
+          // Do nothing...
+        }
 
-      this.onFinally(
-        Object.assign(new Error(this.#statusText ?? reason), {
+        throw Object.assign(this.#error, {
           reason,
           error,
           data: {
@@ -962,26 +1000,23 @@ class Handler extends undici.DecoratorHandler {
               statusCode: this.#statusCode,
             },
           },
-        }),
-      )
-    } else {
-      this.onFinally(null, trailers)
-    }
-  }
-
-  onError(err) {
-    this.onFinally(err)
-  }
-
-  onFinally(err, val) {
-    if (this.#signal) {
-      this.#signal.removeEventListener('abort', this.#onAbort)
-    }
+        })
+      } else if (this.#contentType === 'plain/text') {
+        throw Object.assign(this.#error, {
+          data: {
+            ureq: this.#opts,
+            ures: {
+              body: this.#str,
+              statusCode: this.#statusCode,
+              headers: this.#headers,
+            },
+          },
+        })
+      }
 
-    if (err) {
-      this.#handler.onError?.(err)
+      assert(false)
     } else {
-      this.#handler.onComplete?.(val)
+      this.#handler.onComplete?.(...args)
     }
   }
 }
@@ -992,10 +1027,12 @@ class StreamOutput extends stream.Readable {
   #abort
   #state
   #decoder
+  #batched
   #didPush = false
 
-  constructor({ highWaterMark }) {
+  constructor({ batched = false, highWaterMark = batched ? 2 : 128 }) {
     super({ objectMode: true, highWaterMark })
+    this.#batched = batched
   }
 
   _read() {
@@ -1033,6 +1070,7 @@ class StreamOutput extends stream.Readable {
     lines[0] = this.#str + lines[0]
     this.#str = lines.pop() ?? ''
 
+    const rows = this.#batched ? [] : null
     for (const line of lines) {
       if (this.#state === 0) {
         if (line.endsWith('[')) {
@@ -1044,12 +1082,22 @@ class StreamOutput extends stream.Readable {
         if (line.startsWith(']')) {
           this.#state = 2
         } else {
-          this.push(JSON.parse(line.slice(0, line.lastIndexOf('}') + 1)))
-          this.#didPush = true
+          const row = JSON.parse(line.slice(0, line.lastIndexOf('}') + 1))
+          if (rows) {
+            rows.push(row)
+          } else {
+            this.push(row)
+            this.#didPush = true
+          }
         }
       }
     }
 
+    if (rows) {
+      this.push(rows)
+      this.#didPush = true
+    }
+
     return this.readableLength < this.readableHighWaterMark
   }
 
@@ -1121,6 +1169,8 @@ export function request(
     method,
     headers: {
       'content-type': body != null && typeof body === 'object' ? 'application/json' : 'plain/text',
+      'user-agent': globalThis.userAgent,
+      'request-id': genReqId(),
       accept: 'application/json',
       ...headers,
     },
@@ -1128,11 +1178,12 @@ export function request(
   }
 
   dispatcher = dispatcher.compose(
-    (dispatch) => (opts, handler) => dispatch(opts, new Handler({ opts, signal, handler })),
+    (dispatch) => (opts, handler) => dispatch(opts, new ErrorHandler({ opts, handler })),
+    (dispatch) => (opts, handler) => dispatch(opts, new SignalHandler({ signal, handler })),
   )
 
   if (stream) {
-    const handler = new StreamOutput({ highWaterMark: stream.highWaterMark ?? 128 })
+    const handler = new StreamOutput(stream)
     dispatcher.dispatch(opts, handler)
     return handler
   } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/lib",
-  "version": "19.1.1",
+  "version": "19.1.3",
   "license": "MIT",
   "author": "Robert Nagy <robert.nagy@boffins.se>",
   "type": "module",
@@ -79,6 +79,7 @@
     "@elastic/elasticsearch": "^8.13.1",
     "@elastic/transport": "^8.5.1",
     "@nxtedition/nxt-undici": "^2.0.46",
+    "content-type": "^1.0.5",
     "date-fns": "^3.6.0",
     "fast-querystring": "^1.1.1",
     "hasha": "^6.0.0",

package/s3.js

@@ -4,11 +4,24 @@ import tp from 'node:timers/promises'
 import AWS from '@aws-sdk/client-s3'
 import PQueue from 'p-queue'
 
-const CONTENT_MD5_EXPR = /^[A-F0-9]{32}$/i
-const CONTENT_LENGTH_EXPR = /^\d+$/i
-
 const queue = new PQueue({ concurrency: 8 })
 
+/**
+ * Uploads a file to S3 using multipart upload.
+ *
+ * @param {Object} options - The options for the upload.
+ * @param {AWS.S3} options.client - The S3 client.
+ * @param {AbortSignal} options.signal - The signal to abort the upload.
+ * @param {Object} options.logger - The logger to use.
+ * @param {number} [options.partSize=16e6] - The size of each part in the multipart upload.
+ * @param {Object} options.params - The parameters for the upload.
+ * @param {Buffer|NodeJS.ReadStream} options.params.Body - The data to upload.
+ * @param {string} options.params.Key - The key of the object.
+ * @param {string} options.params.Bucket - The name of the bucket.
+ * @param {string} [options.params.ContentMD5] - The MD5 hash of the object as base64 string.
+ * @param {number} [options.params.ContentLength] - The length of the object.
+ * @returns {Promise<Object>} The result of the upload.
+ */
 export async function upload({ client: s3, signal, logger, partSize = 16e6, params }) {
   if (s3 == null) {
     throw new Error('Invalid client')
@@ -24,13 +37,7 @@ export async function upload({ client: s3, signal, logger, partSize = 16e6, para
 
   const { Body, Key, Bucket, ContentMD5, ContentLength } = params ?? {}
 
-  if (ContentMD5 != null && !CONTENT_MD5_EXPR.test(ContentMD5)) {
-    throw new Error(`Invalid ContentMD5: ${ContentMD5}`)
-  }
-
-  if (ContentLength != null && !CONTENT_LENGTH_EXPR.test(ContentLength)) {
-    throw new Error(`Invalid ContentLength: ${ContentLength}`)
-  }
+  // TODO (fix): Valdate ContentMD & ContentLength
 
   const promises = []
 
@@ -190,7 +197,7 @@ export async function upload({ client: s3, signal, logger, partSize = 16e6, para
     }
 
     const size = ContentLength != null ? Number(ContentLength) : null
-    const hash = ContentMD5
+    const hash = ContentMD5 ? Buffer.from(ContentMD5, 'base64').toString('hex') : null
 
     if (size != null && size !== result.size) {
       throw new Error(`Expected size ${size} but got ${result.size}`)