@radio-garden/ditojs-server 2.85.2-0.5067ad799

package/src/schema/types/_asset.js

@@ -0,0 +1,31 @@
+export const asset = {
+  type: 'object',
+  properties: {
+    key: {
+      type: 'string',
+      required: true
+    },
+    name: {
+      type: 'string',
+      required: true
+    },
+    type: {
+      type: 'string',
+      required: true
+    },
+    size: {
+      type: 'integer',
+      required: true
+    },
+    url: {
+      type: 'string',
+      format: 'uri'
+    },
+    width: {
+      type: 'integer'
+    },
+    height: {
+      type: 'integer'
+    }
+  }
+}

package/src/schema/types/_color.js

@@ -0,0 +1,4 @@
+export const color = {
+  type: 'string',
+  format: 'hexcolor'
+}

package/src/schema/types/index.js

@@ -0,0 +1,2 @@
+export * from './_asset.js'
+export * from './_color.js'

package/src/services/Service.js

@@ -0,0 +1,35 @@
+import { camelize, hyphenate } from '@ditojs/utils'
+
+export class Service {
+  initialized = false
+  #loggerName
+
+  constructor(app, name) {
+    this.app = app
+    this.name = camelize(
+      (name || this.constructor.name).match(/^(.*?)(?:Service|)$/)[1]
+    )
+    this.#loggerName = hyphenate(this.name)
+    this.config = null
+  }
+
+  setup(config) {
+    this.config = config
+    this.app.on('before:start', () => this.start())
+    this.app.on('after:stop', () => this.stop())
+  }
+
+  // @overridable
+  async initialize() {}
+
+  // @overridable
+  async start() {}
+
+  // @overridable
+  async stop() {}
+
+  get logger() {
+    const logger = this.app.requestLocals.logger ?? this.app.logger
+    return logger.child({ name: this.#loggerName })
+  }
+}

package/src/services/index.js

@@ -0,0 +1 @@
+export * from './Service.js'

package/src/storage/AssetFile.js

@@ -0,0 +1,81 @@
+import path from 'path'
+import mime from 'mime-types'
+import { v4 as uuidv4 } from 'uuid'
+import { dataUriToBuffer } from 'data-uri-to-buffer'
+import { isString } from '@ditojs/utils'
+
+const SYMBOL_STORAGE = Symbol('storage')
+const SYMBOL_DATA = Symbol('data')
+
+export class AssetFile {
+  constructor({ name, data, type, width, height }) {
+    this.key = AssetFile.getUniqueKey(name)
+    this.name = name
+    // Set `type` before `data`, so it can be used as default in `set data`
+    this.type = type
+    this.width = width
+    this.height = height
+    this.data = data
+  }
+
+  get storage() {
+    return this[SYMBOL_STORAGE] || null
+  }
+
+  get data() {
+    return this[SYMBOL_DATA] || null
+  }
+
+  set data(data) {
+    if (isString(data)) {
+      if (data.startsWith('data:')) {
+        const { type, buffer } = dataUriToBuffer(data)
+        data = Buffer.from(buffer)
+        this.type ||= type || mime.lookup(this.name)
+      } else {
+        data = Buffer.from(data)
+        this.type ||= mime.lookup(this.name) || 'text/plain'
+      }
+    } else {
+      // Buffer & co.
+      data = Buffer.isBuffer(data) ? data : Buffer.from(data)
+      this.type ||= (
+        data.type || // See Storage._readFile()
+        mime.lookup(this.name) ||
+        'application/octet-stream'
+      )
+    }
+    this.size = Buffer.byteLength(data)
+    setHiddenProperty(this, SYMBOL_DATA, data)
+  }
+
+  get path() {
+    return this.storage?.getFilePath(this)
+  }
+
+  async read() {
+    return this.storage?.readFile(this) || null
+  }
+
+  static convert(object, storage) {
+    Object.setPrototypeOf(object, AssetFile.prototype)
+    setHiddenProperty(object, SYMBOL_STORAGE, storage)
+  }
+
+  static create(options) {
+    return new AssetFile(options)
+  }
+
+  static getUniqueKey(name) {
+    return `${uuidv4()}${path.extname(name).toLowerCase()}`
+  }
+}
+
+function setHiddenProperty(object, key, value) {
+  Object.defineProperty(object, key, {
+    configurable: true,
+    enumerable: false,
+    writable: true,
+    value
+  })
+}

package/src/storage/DiskStorage.js

@@ -0,0 +1,114 @@
+import fs from 'fs/promises'
+import path from 'path'
+import multer from '@koa/multer'
+import { mapConcurrently } from '@ditojs/utils'
+import { Storage } from './Storage.js'
+
+export class DiskStorage extends Storage {
+  static type = 'disk'
+
+  setup() {
+    if (!this.path) {
+      throw new Error(`Missing configuration (path) for storage ${this.name}`)
+    }
+    this.storage = multer.diskStorage({
+      destination: (req, storageFile, cb) => {
+        // Add `storageFile.key` property to internal storage file object.
+        storageFile.key = this.getUniqueKey(storageFile.originalname)
+        const dir = this._getPath(this._getNestedFolder(storageFile.key))
+        fs.mkdir(dir, { recursive: true })
+          .then(() => cb(null, dir))
+          .catch(cb)
+      },
+
+      filename: (req, storageFile, cb) => {
+        // Use added `storageFile.key` property for multer's `filename` also.
+        cb(null, storageFile.key)
+      }
+    })
+  }
+
+  // @override
+  _getFilePath(file) {
+    return this._getPath(this._getNestedFolder(file.key), file.key)
+  }
+
+  // @override
+  _getFileUrl(file) {
+    return this._getUrl(this._getNestedFolder(file.key, true), file.key)
+  }
+
+  // @override
+  async _addFile(file, data) {
+    const filePath = this._getFilePath(file)
+    const dir = path.dirname(filePath)
+    await fs.mkdir(dir, { recursive: true })
+    await fs.writeFile(filePath, data)
+  }
+
+  // @override
+  async _removeFile(file) {
+    const filePath = this._getFilePath(file)
+    await fs.unlink(filePath)
+    const removeIfEmpty = async dir => {
+      if ((await fs.readdir(dir)).length === 0) {
+        try {
+          await fs.rmdir(dir)
+        } catch (err) {
+          // The directory may already have been deleted by another async call,
+          // fail silently here in this case.
+          if (err.code !== 'ENOENT') {
+            throw err
+          }
+        }
+      }
+    }
+    // Clean up nested folders created with first two chars of `file.key` also:
+    const dir = path.dirname(filePath)
+    const parentDir = path.dirname(dir)
+    await removeIfEmpty(dir)
+    await removeIfEmpty(parentDir)
+  }
+
+  // @override
+  async _readFile(file) {
+    return fs.readFile(this._getFilePath(file))
+  }
+
+  // @override
+  async _listKeys() {
+    const readDir = (...parts) =>
+      fs.readdir(this._getPath(...parts), { withFileTypes: true })
+
+    const files = []
+    const list1 = await readDir()
+    await mapConcurrently(
+      list1,
+      async level1 => {
+        if (level1.isDirectory() && level1.name.length === 1) {
+          const list2 = await readDir(level1.name)
+          await mapConcurrently(
+            list2,
+            async level2 => {
+              if (level2.isDirectory() && level2.name.length === 1) {
+                const nestedFolder = this._getPath(level1.name, level2.name)
+                for (const file of await fs.readdir(nestedFolder)) {
+                  if (!file.startsWith('.')) {
+                    files.push(file)
+                  }
+                }
+              }
+            }
+          )
+        }
+      }
+    )
+    return files
+  }
+
+  _getNestedFolder(key, posix = false) {
+    // Store files in nested folders created with the first two chars of the
+    // key, for faster access & management with large amounts of files.
+    return (posix ? path.posix : path).join(key[0], key[1])
+  }
+}

package/src/storage/S3Storage.js

@@ -0,0 +1,169 @@
+import multerS3 from 'multer-s3'
+import { fileTypeFromBuffer } from 'file-type'
+import { Storage } from './Storage.js'
+import { PassThrough } from 'stream'
+import consumers from 'stream/consumers'
+import { readMediaAttributes } from 'leather'
+
+export class S3Storage extends Storage {
+  static type = 's3'
+
+  s3 = null
+  acl = null
+  bucket = null
+
+  async setup() {
+    const {
+      name,
+      s3,
+      acl,
+      bucket,
+      ...options
+    } = this.config
+
+    // "@aws-sdk/client-s3" is a peer-dependency, and importing it costly,
+    // so we do it lazily.
+    const { S3 } = await import('@aws-sdk/client-s3')
+    this.s3 = new S3(s3)
+    this.acl = acl
+    this.bucket = bucket
+
+    this.storage = multerS3({
+      s3: this.s3,
+      acl,
+      bucket,
+      ...options,
+
+      key: (req, file, cb) => {
+        cb(null, this.getUniqueKey(file.originalname))
+      },
+
+      contentType: (req, file, cb) => {
+        const { mimetype, stream } = file
+        if (mimetype) {
+          // 1. Trust file.mimetype if provided.
+          cb(null, mimetype)
+        } else {
+          let data = null
+
+          const done = type => {
+            stream.off('data', onData)
+            const outStream = new PassThrough()
+            outStream.write(data)
+            stream.pipe(outStream)
+            cb(null, type, outStream)
+          }
+
+          const onData = chunk => {
+            if (!data) {
+              // 2. Try reading the mimetype from the first chunk.
+              const type = getFileTypeFromBuffer(chunk)
+              if (type) {
+                done(type)
+              } else {
+                // 3. If that fails, keep collecting all chunks and determine
+                //    the mimetype using the full data.
+                stream.once('end', () =>
+                  done(
+                    getFileTypeFromBuffer(data) || 'application/octet-stream'
+                  )
+                )
+              }
+            }
+            data = data ? Buffer.concat([data, chunk]) : chunk
+          }
+
+          stream.on('data', onData)
+        }
+      },
+
+      metadata: (req, file, cb) => {
+        // Store the determined width and height as meta-data on the s3 object
+        // as well. You never know, it may become useful :)
+        const { width, height } = file
+        if (width != null || height != null) {
+          cb(null, {
+            width: `${width}`,
+            height: `${height}`
+          })
+        } else {
+          cb(null, {})
+        }
+      }
+    })
+  }
+
+  // @override
+  _getFilePath(_file) {
+    // There is no "local" file-path to files on S3.
+    return undefined
+  }
+
+  // @override
+  _getFileUrl(file) {
+    return this._getUrl(file.key) ?? file.url
+  }
+
+  // @override
+  async _addFile(file, data) {
+    const result = await this.s3.putObject({
+      Bucket: this.bucket,
+      ACL: this.acl,
+      Key: file.key,
+      ContentType: file.type,
+      Body: data
+    })
+    // In `Storage.addFile()` this will get overridden with the result of
+    // `_getUrl()` if it exists, but is used as a fallback otherwise,
+    // see `_getFileUrl()`.
+    file.url = result.Location
+  }
+
+  // @override
+  async _removeFile(file) {
+    await this.s3.deleteObject({
+      Bucket: this.bucket,
+      Key: file.key
+    })
+    // TODO: Check for errors and throw?
+  }
+
+  // @override
+  async _readFile(file) {
+    const {
+      ContentType: type,
+      Body: stream
+    } = await this.s3.getObject({
+      Bucket: this.bucket,
+      Key: file.key
+    })
+    const buffer = await consumers.buffer(stream)
+    // See `AssetFile.data` setter:
+    buffer.type = type
+    return buffer
+  }
+
+  // @override
+  async _listKeys() {
+    const files = []
+    const params = { Bucket: this.bucket }
+    let result
+    do {
+      result = await this.s3.listObjectsV2(params)
+      for (const { Key: key } of result.Contents ?? []) {
+        files.push(key)
+      }
+      // Continue it if results are truncated.
+      params.ContinuationToken = result.NextContinuationToken
+    } while (result.IsTruncated)
+    return files
+  }
+}
+
+function getFileTypeFromBuffer(buffer) {
+  try {
+    // Use leather as fall-back for better media file mime type detection.
+    return fileTypeFromBuffer(buffer)?.mime || readMediaAttributes(buffer)?.mime
+  } catch {}
+  return null
+}

package/src/storage/Storage.js

@@ -0,0 +1,231 @@
+import path from 'path'
+import { URL } from 'url'
+import multer from '@koa/multer'
+import picomatch from 'picomatch'
+import { PassThrough } from 'stream'
+import { readMediaAttributes } from 'leather'
+import { hyphenate, toPromiseCallback } from '@ditojs/utils'
+import { AssetFile } from './AssetFile.js'
+
+const storageClasses = {}
+
+export class Storage {
+  initialized = false
+
+  constructor(app, config) {
+    this.app = app
+    this.config = config
+    this.name = config.name
+    this.url = config.url
+    this.path = config.path
+    // Use a default concurrency of 8 for storage IO, e.g. the importing of
+    // foreign assets.
+    this.concurrency = config.concurrency ?? 8
+    // The actual multer storage object.
+    this.storage = null
+  }
+
+  // @overridable
+  async setup() {}
+
+  // @overridable
+  async initialize() {}
+
+  static register(storageClass) {
+    const type = (
+      storageClass.type ||
+      hyphenate(storageClass.name.match(/^(.*?)(?:Storage|)$/)[1])
+    )
+    storageClass.type = type
+    storageClasses[type] = storageClass
+  }
+
+  static get(type) {
+    return storageClasses[type] || null
+  }
+
+  getUploadStorage(config) {
+    // Returns a storage that inherits from this.storage but overrides
+    // _handleFile to pass on `config` to the call of `handleUpload()`
+    return this.storage
+      ? Object.setPrototypeOf(
+          {
+            _handleFile: async (req, file, callback) => {
+              try {
+                callback(null, await this._handleUpload(req, file, config))
+              } catch (err) {
+                callback(err)
+              }
+            }
+          },
+          this.storage
+        )
+      : null
+  }
+
+  getUploadHandler(config) {
+    const storage = this.getUploadStorage(config)
+    return storage ? multer({ ...config, storage }).any() : null
+  }
+
+  getUniqueKey(name) {
+    return AssetFile.getUniqueKey(name)
+  }
+
+  isImportSourceAllowed(url) {
+    return picomatch.isMatch(url, this.config.allowedImports || [])
+  }
+
+  convertAssetFile(file) {
+    AssetFile.convert(file, this)
+  }
+
+  convertStorageFile(storageFile) {
+    // Convert multer file object to our own file object format:
+    return {
+      key: storageFile.key,
+      name: storageFile.originalname,
+      type: storageFile.mimetype,
+      size: storageFile.size,
+      url: this._getFileUrl(storageFile),
+      // In case `config.readDimensions` is set:
+      width: storageFile.width,
+      height: storageFile.height
+    }
+  }
+
+  convertStorageFiles(storageFiles) {
+    return storageFiles.map(storageFile => this.convertStorageFile(storageFile))
+  }
+
+  async addFile(file, data) {
+    await this._addFile(file, data)
+    file.size = Buffer.byteLength(data)
+    file.url = this._getFileUrl(file)
+    // TODO: Support `config.readDimensions`, but this can only be done once
+    // there are separate storage instances per model assets config!
+    this.convertAssetFile(file)
+    return file
+  }
+
+  async removeFile(file) {
+    await this._removeFile(file)
+  }
+
+  async readFile(file) {
+    return this._readFile(file)
+  }
+
+  async listKeys() {
+    return this._listKeys()
+  }
+
+  getFilePath(file) {
+    return this._getFilePath(file)
+  }
+
+  getFileUrl(file) {
+    return this._getFileUrl(file)
+  }
+
+  _getUrl(...parts) {
+    return this.url
+      ? new URL(path.posix.join(...parts), this.url).toString()
+      : undefined // So that it doesn't show up in JSON data.
+  }
+
+  _getPath(...parts) {
+    return this.path
+      ? path.resolve(this.app.basePath, this.path, ...parts)
+      : undefined // So that it doesn't show up in JSON data.
+  }
+
+  // @overridable
+  _getFilePath(_file) {}
+
+  // @overridable
+  _getFileUrl(_file) {}
+
+  // @overridable
+  async _addFile(_file, _data) {}
+
+  // @overridable
+  async _removeFile(_file) {}
+
+  // @overridable
+  async _readFile(_file) {}
+
+  // @overridable
+  async _listKeys() {}
+
+  async _handleUpload(req, file, config) {
+    if (
+      config.readDimensions &&
+      /^(image|video)\//.test(file.mimetype)
+    ) {
+      return this._handleMediaFile(req, file)
+    } else {
+      return this._handleFile(req, file)
+    }
+  }
+
+  _handleFile(req, file, stream = null) {
+    // Calls the original `storage._handleFile()`, wrapped in a promise:
+    return new Promise((resolve, reject) => {
+      if (stream) {
+        // Replace the original `file.stream` with the pass-through stream:
+        Object.defineProperty(file, 'stream', {
+          configurable: true,
+          enumerable: false,
+          value: stream
+        })
+      }
+      this.storage._handleFile(req, file, toPromiseCallback(resolve, reject))
+    })
+  }
+
+  async _handleMediaFile(req, file) {
+    const { size, stream } = await new Promise(resolve => {
+      let data = null
+
+      const done = size => {
+        const stream = new PassThrough()
+        stream.write(data)
+        file.stream
+          .off('data', onData)
+          .off('end', onEnd)
+          .pipe(stream)
+        resolve({ size, stream })
+      }
+
+      const onEnd = () => {
+        this.app.emit('error', 'Unable to determine image size')
+        done(null)
+      }
+
+      const onData = chunk => {
+        data = data ? Buffer.concat([data, chunk]) : chunk
+        try {
+          const size = readMediaAttributes(data)
+          // On partial data, sometimes we get results back from leather without
+          // actual dimensions, so check for that.
+          if (size.mime && (size.width > 0 || size.height > 0)) {
+            done(size)
+          }
+        } catch {
+          // Ignore errors in `readMediaAttributes()` on partial data.
+        }
+      }
+
+      file.stream
+        .on('data', onData)
+        .on('end', onEnd)
+    })
+
+    if (size) {
+      file.width = size.width
+      file.height = size.height
+    }
+    return this._handleFile(req, file, stream)
+  }
+}

package/src/storage/index.js

@@ -0,0 +1,9 @@
+import { Storage } from './Storage.js'
+import { DiskStorage } from './DiskStorage.js'
+import { S3Storage } from './S3Storage.js'
+
+Storage.register(DiskStorage)
+Storage.register(S3Storage)
+
+export * from './AssetFile.js'
+export { Storage }

package/src/utils/duration.js

@@ -0,0 +1,15 @@
+import { isNumber } from '@ditojs/utils'
+import parseDuration from 'parse-duration'
+
+export function getDuration(duration) {
+  return isNumber(duration) ? duration : parseDuration(duration)
+}
+
+export function addDuration(date, duration) {
+  date.setMilliseconds(date.getMilliseconds() + getDuration(duration))
+  return date
+}
+
+export function subtractDuration(date, duration) {
+  return addDuration(date, -getDuration(duration))
+}

package/src/utils/emitter.js

@@ -0,0 +1,8 @@
+import { mapConcurrently } from '@ditojs/utils'
+
+export function emitAsync(emitter, event, ...args) {
+  return mapConcurrently(
+    emitter.listeners(event),
+    listener => listener.call(emitter, ...args)
+  )
+}