@nxtedition/lib 26.8.8 → 27.0.0

package/s3.js

@@ -1,339 +0,0 @@
-import crypto from 'node:crypto'
-import tp from 'node:timers/promises'
-import AWS from '@aws-sdk/client-s3'
-import PQueue from 'p-queue'
-import { NodeHttpHandler } from '@smithy/node-http-handler'
-import { request as undiciRequest, parseHeaders } from '@nxtedition/nxt-undici'
-
-const QUEUE = new PQueue({ concurrency: 16 })
-const MD5_HEX_EXPR = /^[A-F0-9]{32}$/i
-
-/**
- * @import { Dispatcher } from 'undici'
- * @import { S3ClientConfig, CreateMultipartUploadRequest } from '@aws-sdk/client-s3'
- */
-
-export class S3Client extends AWS.S3Client {
-  /**
-   * @param {S3ClientConfig & { dispatcher?: Dispatcher }} config
-   */
-  constructor(config) {
-    const { dispatcher, ...options } = config
-    super({
-      requestHandler: new UndiciRequestHandler(dispatcher),
-      ...options,
-    })
-  }
-}
-
-class UndiciRequestHandler extends NodeHttpHandler {
-  #dispatcher
-
-  /**
-   * @param {Dispatcher=} dispatcher
-   */
-  constructor(dispatcher) {
-    super()
-    this.#dispatcher = dispatcher
-  }
-
-  /**
-   * @param {import('@smithy/protocol-http').HttpRequest} request
-   * @param {import('@smithy/types').HttpHandlerOptions=} options
-   */
-  async handle(request, options) {
-    const abortSignal = options?.abortSignal
-
-    const { protocol, hostname, port, path, ...requestOptions } = request
-    // NOTE: Expect & Transfer-Encoding header is not supported by undici
-    const {
-      expect,
-      'transfer-encoding': transferEncoding,
-      ...headers
-    } = parseHeaders(request.headers)
-    const typedMethod = /** @type {import('undici').Dispatcher.HttpMethod} */ (request.method)
-
-    const url = `${request.protocol}//${request.hostname}${request.port ? `:${request.port}` : ''}${request.path}`
-
-    const response = await undiciRequest(url, {
-      ...requestOptions,
-      method: typedMethod,
-      signal: abortSignal,
-      dispatcher: this.#dispatcher,
-      headers,
-      body: request.body,
-      // nxt-undici
-      error: false,
-      dns: false,
-      retry: false,
-      follow: false,
-    })
-
-    return {
-      response: {
-        ...response,
-        headers: getTransformedHeaders(response.headers),
-      },
-    }
-  }
-}
-
-/**
- * Uploads a file to S3 using multipart upload.
- *
- * @param {Object} options - The options for the upload.
- * @param {AWS.S3} options.client - The S3 client.
- * @param {AbortSignal} [options.signal] - The signal to abort the upload.
- * @param {Object} [options.logger] - The logger to use.
- * @param {number} [options.partSize=16e6] - The size of each part in the multipart upload.
- * @param {PQueue} [options.queue] - The queue to use for part uploads.
- * @param {CreateMultipartUploadRequest & {
- *   Body: stream.Readable,
- *   ContentMD5?: string,
- *   ContentLength?: number,
- * }} options.params - The parameters for the upload.
- * @returns {Promise<Object>} The result of the upload.
- */
-export async function upload({
-  client: s3,
-  queue = QUEUE,
-  signal,
-  logger,
-  partSize = 5e6,
-  params,
-}) {
-  if (s3 == null) {
-    throw new Error('Invalid client')
-  }
-
-  if (!Number.isFinite(partSize) || partSize <= 0) {
-    throw new Error('Invalid partSize')
-  }
-
-  if (params == null || typeof params !== 'object') {
-    throw new Error('Invalid params')
-  }
-
-  const { Body, Key, Bucket, ContentMD5, ContentLength, ...createMultipartParams } = params
-
-  const size = ContentLength != null ? Number(ContentLength) : null
-
-  if (size != null && (!Number.isFinite(size) || size < 0)) {
-    throw new Error('Invalid params.ContentLength')
-  }
-
-  const hash = ContentMD5 != null ? Buffer.from(ContentMD5, 'base64').toString('hex') : null
-  if (hash != null && !MD5_HEX_EXPR.test(hash)) {
-    throw new Error('Invalid params.ContentMD5')
-  }
-
-  const promises = []
-
-  const ac = new AbortController()
-  const onAbort = () => {
-    ac.abort()
-  }
-  signal?.addEventListener('abort', onAbort)
-
-  let uploadId
-  try {
-    const uploader = {
-      size: 0,
-      hasher: crypto.createHash('md5'),
-      ac,
-      signal: ac.signal,
-      part: {
-        /** @type {Array<Buffer>} **/ chunks: [],
-        hasher: crypto.createHash('md5'),
-        size: 0,
-        number: 1,
-      },
-    }
-
-    const multipartUploadOutput = await s3.send(
-      new AWS.CreateMultipartUploadCommand({ ...createMultipartParams, Bucket, Key }),
-      { abortSignal: uploader.signal },
-    )
-    uploader.signal.throwIfAborted()
-
-    uploadId = multipartUploadOutput.UploadId
-    logger = logger?.child({ uploadId })
-    logger?.debug('multipart upload created')
-
-    const maybeFlush = (minSize) => {
-      const { part } = uploader
-
-      if (!part.size) {
-        return
-      }
-
-      if (minSize != null && part.size < minSize) {
-        return
-      }
-
-      const chunks = part.chunks
-      const number = part.number
-      const size = part.size
-      const hash = part.hasher.digest('hex')
-
-      part.chunks = []
-      part.number += 1
-      part.size = 0
-      part.hasher = crypto.createHash('md5')
-
-      promises.push(
-        queue
-          .add(
-            async () => {
-              for (let retryCount = 0; true; retryCount++) {
-                logger?.debug({ number, size }, 'part upload started')
-                try {
-                  const { ETag } = await s3.send(
-                    new AWS.UploadPartCommand({
-                      Bucket,
-                      Key,
-                      UploadId: uploadId,
-                      ContentMD5: Buffer.from(hash, 'hex').toString('base64'),
-                      ContentLength: size,
-                      PartNumber: number,
-                      Body: Buffer.concat(chunks),
-                    }),
-                  )
-                  logger?.debug({ number, size, etag: ETag }, 'part upload completed')
-                  return { part: { ETag, PartNumber: number } }
-                } catch (err) {
-                  logger?.warn({ err }, 'part upload failed')
-
-                  if (retryCount < 16 && err.name !== 'NotImplemented') {
-                    logger?.debug({ retryCount }, 'part upload retry')
-                    await tp.setTimeout(retryCount * retryCount * 1e3, undefined, {
-                      signal: uploader.signal,
-                    })
-                  } else {
-                    uploader.ac.abort(err)
-                    return { error: err }
-                  }
-                }
-              }
-            },
-            { signal: uploader.signal },
-          )
-          .catch((err) => ({ error: err })),
-      )
-
-      return queue.size > 0 ? queue.onEmpty() : null
-    }
-
-    for await (const chunk of Body) {
-      uploader.signal.throwIfAborted()
-
-      uploader.hasher.update(chunk)
-      uploader.size += chunk.byteLength
-
-      uploader.part.hasher.update(chunk)
-      uploader.part.chunks.push(chunk)
-      uploader.part.size += chunk.byteLength
-
-      const thenable = maybeFlush(partSize)
-      if (thenable) {
-        await thenable
-        uploader.signal.throwIfAborted()
-      }
-    }
-    await maybeFlush()
-    uploader.signal.throwIfAborted()
-
-    const parts = []
-    const errors = []
-    for (const { part, error } of await Promise.all(promises)) {
-      if (error) {
-        errors.push(error)
-      } else if (part) {
-        parts.push(part)
-      }
-    }
-    uploader.signal.throwIfAborted()
-
-    if (errors.length > 0) {
-      throw new AggregateError(errors, 'multipart upload failed')
-    }
-
-    if (parts.length === 0) {
-      throw new Error('upload empty')
-    }
-
-    const uploadOutput = await s3.send(
-      new AWS.CompleteMultipartUploadCommand({
-        Bucket,
-        Key,
-        UploadId: uploadId,
-        MultipartUpload: { Parts: parts },
-      }),
-    )
-    uploader.signal.throwIfAborted()
-
-    const result = {
-      size: uploader.size,
-      hash: uploader.hasher.digest('hex'),
-      output: uploadOutput,
-      parts,
-    }
-
-    if (size != null && size !== result.size) {
-      throw new Error(`Expected size ${size} but got ${result.size}`)
-    }
-
-    if (hash != null && hash !== result.hash) {
-      throw new Error(`Expected hash ${hash} but got ${result.hash}`)
-    }
-
-    logger?.debug(result, 'multipart upload completed')
-
-    return result
-  } catch (err) {
-    logger?.error({ err }, 'multipart upload failed')
-
-    if (uploadId) {
-      try {
-        await s3.send(new AWS.AbortMultipartUploadCommand({ Bucket, Key, UploadId: uploadId }))
-      } catch (er) {
-        throw new AggregateError([err, er])
-      }
-    }
-
-    throw err
-  } finally {
-    signal?.removeEventListener('abort', onAbort)
-  }
-}
-
-/**
- * @see https://github.com/smithy-lang/smithy-typescript/blob/main/packages/node-http-handler/src/get-transformed-headers.ts
- *
- * @param {import('http2').IncomingHttpHeaders} headers
- * @returns {import('@smithy/types').HeaderBag}
- */
-function getTransformedHeaders(headers) {
-  /**
-   * @type {import('@smithy/types').HeaderBag}
-   */
-  const transformedHeaders = {}
-
-  const isRangedResponse = headers['content-range'] != null
-
-  for (const name of Object.keys(headers)) {
-    // Google Cloud S3 returns checksums for the full object
-    // even with range requests, which causes checksum mismatches.
-    // I assume the checksum headers shouldn't be sent with ranged responses.
-    if (isRangedResponse && /^x-amz-checksum-/i.test(name)) {
-      continue
-    }
-
-    const headerValues = headers[name]
-    transformedHeaders[name] = Array.isArray(headerValues)
-      ? headerValues.join(',')
-      : (headerValues ?? '')
-  }
-
-  return transformedHeaders
-}

package/timeline.js

@@ -1,115 +0,0 @@
-import fp from 'lodash/fp.js'
-
-const STOP_TIME = 5
-const PRELOAD_TIME = 5
-
-// Events must have start/end/source/data, and should be sorted by start
-export function eventsToTimeline(events) {
-  const timeline = []
-  events = [...events]
-  let active = []
-
-  while (active.length || events.length) {
-    const activeEnd = Math.min(...active.map((x) => x.end))
-    if (active.length && (!events.length || activeEnd < events.at(0).start)) {
-      // current has ended
-      const prev = active.at(-1)
-      active = active.filter((e) => e.end > activeEnd)
-      if (active.length) {
-        // reverting to previously active
-        timeline.push({
-          ...fp.omit(['start', 'end'], active.at(-1)),
-          data: Object.assign({}, ...active.map((x) => x.data)),
-          time: activeEnd,
-        })
-      } else {
-        // stop active
-        if (activeEnd < Infinity) {
-          timeline.push({ ...fp.omit(['start', 'end', 'data'], prev), time: activeEnd })
-        }
-        // clear
-        const clearTime = activeEnd + STOP_TIME
-        if (!events.length || clearTime < events.at(0).start) {
-          timeline.push({ ...fp.omit(['start', 'end', 'source', 'data'], prev), time: clearTime })
-        }
-      }
-    } else {
-      // start next
-      const nextStart = events.at(0).start
-      let next
-      do {
-        next = events.shift()
-        active.push(next)
-      } while (events.length && nextStart === events.at(0).start)
-
-      timeline.push({
-        ...fp.omit(['start', 'end'], next),
-        data: Object.assign({}, ...active.map((x) => x.data)),
-        time: next.start,
-      })
-    }
-  }
-  return timeline
-}
-
-export function timelineToCommands(timeline) {
-  const commands = []
-  let current = { time: -Infinity }
-  for (let i = 0; i < timeline.length; i++) {
-    const next = timeline[i]
-    const canPreload =
-      next.source && next.data && current.source == null && current.time < next.time
-    if (canPreload) {
-      const loadTime = Math.max(current.time, next.time - PRELOAD_TIME)
-      commands.push(
-        {
-          command: 'load',
-          ...next,
-          time: loadTime,
-        },
-        {
-          command: 'play',
-          ...fp.omit(['source', 'data'], next),
-        },
-      )
-    } else if (next.source) {
-      if (next.source !== current.source || !current.data) {
-        commands.push({
-          command: 'play',
-          ...next,
-        })
-      } else if (!fp.isEqual(current.data, next.data)) {
-        commands.push({
-          command: next.data ? 'update' : 'stop',
-          ...fp.omit(['source'], next),
-        })
-      }
-    } else {
-      commands.push({
-        command: 'clear',
-        ...next,
-      })
-    }
-    current = next
-  }
-
-  return commands
-}
-
-export function pickLayer(source, min = 11, max = 99999) {
-  if (!source) {
-    return min
-  }
-
-  let hash = 0
-
-  for (let i = 0; i < source.length; i++) {
-    const chr = source.charCodeAt(i)
-    hash = (hash << 5) - hash + chr
-    hash |= 0
-  }
-
-  hash = hash < 0 ? -hash : hash
-
-  return min + (hash % (max - min + 1))
-}

package/util/template/index-common.d.ts

@@ -1,67 +0,0 @@
-import type { FirstValueFromConfig } from 'rxjs/internal/firstValueFrom'
-import type * as rx from 'rxjs'
-import type { DeepstreamClient } from '@nxtedition/deepstream.io-client-js'
-import type { Logger } from '../../logger.js'
-
-export interface MakeTemplateCompilerParams {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  ds?: DeepstreamClient<any, any>
-  proxify?: unknown
-  logger?: Logger
-  platform?: unknown
-}
-
-export declare function makeTemplateCompiler(params: MakeTemplateCompilerParams): TemplateCompiler
-
-export interface TemplateCompiler {
-  current: null
-
-  resolveTemplate: <ReturnValue, Context extends Record<string, unknown>>(
-    template: Nxtpression<ReturnValue, Context> | string,
-    args$?: Context | rx.Observable<Context>,
-    options?: FirstValueFromConfig<ReturnValue>,
-  ) => Promise<unknown>
-
-  onResolveTemplate: <ReturnValue, Context extends Record<string, unknown>>(
-    template: Nxtpression<ReturnValue, Context> | string,
-    args$?: Context | rx.Observable<Context>,
-  ) => rx.Observable<unknown>
-
-  compileTemplate: <ReturnValue, Context extends Record<string, unknown>>(
-    template: Nxtpression<ReturnValue, Context> | string,
-  ) => null | ((args$?: Context | rx.Observable<Context>) => rx.Observable<unknown>)
-
-  isTemplate: (value: unknown) => value is Nxtpression
-}
-
-export type Nxtpression<ReturnValue = unknown, Context extends object = object> =
-  | {
-      /**
-       * TS-HACK: this property doesn't really exist on the nxtpression string,
-       * it is only here to make sure the generic Context won't get stripped.
-       */
-      __context: Context
-
-      /**
-       * TS-HACK: this property doesn't really exist on the nxtpression string,
-       * it is only here to make sure the generic Context won't get stripped.
-       */
-      __returnValue: ReturnValue
-    }
-  | string
-  | ReturnValue
-
-export type ResolveNxtpressionDeep<T> =
-  ExtractReturn<T> extends never
-    ? T extends Array<infer U>
-      ? ResolveNxtpressionDeep<U>
-      : T extends object
-        ? {
-            [K in keyof T]: ResolveNxtpressionDeep<T[K]>
-          }
-        : T
-    : ExtractReturn<T>
-type TypeWithReturn<X> = {
-  __returnValue: X
-}
-type ExtractReturn<Type> = Type extends TypeWithReturn<infer X> ? X : never