@ditojs/server 1.25.0 → 1.26.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ditojs/server",
-  "version": "1.25.0",
+  "version": "1.26.0",
   "type": "module",
   "description": "Dito.js Server – Dito.js is a declarative and modern web framework, based on Objection.js, Koa.js and Vue.js",
   "repository": "https://github.com/ditojs/dito/tree/master/packages/server",
@@ -22,10 +22,10 @@
     "node >= 18"
   ],
   "dependencies": {
-    "@ditojs/admin": "^1.25.0",
+    "@ditojs/admin": "^1.26.0",
     "@ditojs/build": "^1.25.0",
-    "@ditojs/router": "^1.25.0",
-    "@ditojs/utils": "^1.25.0",
+    "@ditojs/router": "^1.26.0",
+    "@ditojs/utils": "^1.26.0",
     "@koa/cors": "^4.0.0",
     "@koa/multer": "^3.0.2",
     "@originjs/vite-plugin-commonjs": "^1.0.3",
@@ -90,7 +90,7 @@
     "typescript": "^4.9.5"
   },
   "types": "types",
-  "gitHead": "29e7f8d8c613c48a856d90d1ee876e85c8d84c08",
+  "gitHead": "198c9832c863003c8c8fcd147bf61a5f99a39c91",
   "scripts": {
     "types": "tsc --noEmit ./src/index.d.ts"
   },

package/src/app/Application.js

@@ -6,7 +6,6 @@ import Koa from 'koa'
 import Knex from 'knex'
 import pico from 'picocolors'
 import pino from 'pino'
-import parseDuration from 'parse-duration'
 import bodyParser from 'koa-bodyparser'
 import cors from '@koa/cors'
 import compose from 'koa-compose'
@@ -21,9 +20,9 @@ import responseTime from 'koa-response-time'
 import { Model, knexSnakeCaseMappers, ref } from 'objection'
 import Router from '@ditojs/router'
 import {
-  isArray, isObject, isString, asArray, isPlainObject, isModule,
-  hyphenate, clone, merge, parseDataPath, normalizeDataPath, toPromiseCallback,
-  mapConcurrently
+  isArray, isObject, asArray, isPlainObject, isModule,
+  hyphenate, clone, merge, parseDataPath, normalizeDataPath,
+  toPromiseCallback, mapConcurrently
 } from '@ditojs/utils'
 import SessionStore from './SessionStore.js'
 import { Validator } from './Validator.js'
@@ -32,7 +31,7 @@ import { Controller, AdminController } from '../controllers/index.js'
 import { Service } from '../services/index.js'
 import { Storage } from '../storage/index.js'
 import { convertSchema } from '../schema/index.js'
-import { deprecate } from '../utils/index.js'
+import { getDuration, subtractDuration, deprecate } from '../utils/index.js'
 import {
   ResponseError,
   ValidationError,
@@ -66,6 +65,7 @@ export class Application extends Koa {
       // Pluck keys out of `config.app` to keep them secret
       app: { keys, ...app } = {},
       log,
+      assets,
       logger,
       ...rest
     } = config
@@ -74,6 +74,7 @@ export class Application extends Koa {
       log: log === false || log?.silent || process.env.DITO_SILENT
         ? {}
         : getOptions(log),
+      assets: merge(defaultAssetOptions, getOptions(assets)),
       logger: merge(defaultLoggerOptions, getOptions(logger)),
       ...rest
     }
@@ -805,16 +806,6 @@ export class Application extends Koa {
     removedFiles,
     trx = null
   ) {
-    const {
-      assets: {
-        cleanupTimeThreshold = 0
-      } = {}
-    } = this.config
-    // Only remove unused assets that haven't seen changes for given time frame.
-    const timeThreshold = isString(cleanupTimeThreshold)
-      ? parseDuration(cleanupTimeThreshold)
-      : cleanupTimeThreshold
-
     let importedFiles = []
     const AssetModel = this.getModel('Asset')
     if (AssetModel) {
@@ -837,18 +828,20 @@ export class Application extends Koa {
           changeCount(addedFiles, 1),
           changeCount(removedFiles, -1)
         ])
-        if (timeThreshold > 0) {
+        const cleanupTimeThreshold =
+          getDuration(this.config.assets.cleanupTimeThreshold)
+        if (cleanupTimeThreshold > 0) {
           setTimeout(
             // Don't pass `trx` here, as we want this delayed execution to
             // create its own transaction.
-            () => this.releaseUnusedAssets(timeThreshold),
-            timeThreshold
+            () => this.releaseUnusedAssets(),
+            cleanupTimeThreshold
           )
         }
       }
       // Also execute releaseUnusedAssets() immediately in the same
       // transaction, to potentially clean up other pending assets.
-      await this.releaseUnusedAssets(timeThreshold, trx)
+      await this.releaseUnusedAssets(null, trx)
       return importedFiles
     }
   }
@@ -867,31 +860,28 @@ export class Application extends Koa {
               const { url } = file
               if (!storage.isImportSourceAllowed(url)) {
                 throw new AssetError(
-                    `Unable to import asset from foreign source: '${
-                      file.name
-                    }' ('${
-                      url
-                    }'): The source needs to be explicitly allowed.`
+                  `Unable to import asset from foreign source: '${
+                    file.name
+                  }' ('${
+                    url
+                  }'): The source needs to be explicitly allowed.`
                 )
               }
-              console.info(
-                  `${
-                    pico.red('INFO:')
-                  } Asset ${
-                    pico.green(`'${file.name}'`)
-                  } is from a foreign source, fetching from ${
-                    pico.green(`'${url}'`)
-                  } and adding to storage ${
-                    pico.green(`'${storage.name}'`)
-                  }...`
+              this.logger.info(
+                `Asset ${
+                  pico.green(`'${file.name}'`)
+                } is from a foreign source, fetching from ${
+                  pico.green(`'${url}'`)
+                } and adding to storage ${
+                  pico.green(`'${storage.name}'`)
+                }...`
               )
               if (url.startsWith('file://')) {
                 const filepath = path.resolve(url.substring(7))
                 data = await fs.readFile(filepath)
               } else {
                 const response = await fetch(url)
-                const buffer = await response.arrayBuffer()
-                data = new DataView(buffer)
+                data = await response.arrayBuffer()
               }
             }
             const importedFile = await storage.addFile(file, data)
@@ -903,11 +893,11 @@ export class Application extends Koa {
             importedFiles.push(importedFile)
           } else {
             throw new AssetError(
-                `Unable to import asset from foreign source: '${
-                  file.name
-                }' ('${
-                  file.key
-                }')`
+              `Unable to import asset from foreign source: '${
+                file.name
+              }' ('${
+                file.key
+              }')`
             )
           }
         } else {
@@ -951,21 +941,34 @@ export class Application extends Koa {
     return modifiedFiles
   }
 
-  async releaseUnusedAssets(timeThreshold = 0, trx = null) {
+  async releaseUnusedAssets(timeThreshold = null, trx = null) {
     const AssetModel = this.getModel('Asset')
     if (AssetModel) {
+      const { assets } = this.config
+      const cleanupTimeThreshold =
+        getDuration(timeThreshold ?? assets.cleanupTimeThreshold)
+      const danglingTimeThreshold =
+        getDuration(timeThreshold ?? assets.danglingTimeThreshold)
       return AssetModel.transaction(trx, async trx => {
-        // Determine the time threshold in JS instead of SQL, as there is no
-        // easy cross-SQL way to do `now() - interval X hours`:
-        const date = new Date()
-        date.setMilliseconds(date.getMilliseconds() - timeThreshold)
+        // Calculate the date math in JS instead of SQL, as there is no easy
+        // cross-SQL way to do `now() - interval X hours`:
+        const now = new Date()
+        const cleanupDate = subtractDuration(now, cleanupTimeThreshold)
+        const danglingDate = subtractDuration(now, danglingTimeThreshold)
         const orphanedAssets = await AssetModel
           .query(trx)
           .where('count', 0)
-          .andWhere('updatedAt', '<=', date)
-          // Protect freshly created assets from being deleted again right away,
-          // .e.g. when `config.assets.cleanupTimeThreshold = 0`
-          .andWhere('updatedAt', '>', ref('createdAt'))
+          .andWhere(
+            query => query
+              .where('updatedAt', '<=', cleanupDate)
+              .orWhere(
+                // Protect freshly created assets from being deleted again right
+                // away, .e.g. when `config.assets.cleanupTimeThreshold = 0`
+                query => query
+                  .where('updatedAt', '=', ref('createdAt'))
+                  .andWhere('updatedAt', '<=', danglingDate)
+              )
+          )
         if (orphanedAssets.length > 0) {
           const orphanedKeys = await mapConcurrently(
             orphanedAssets,
@@ -998,6 +1001,17 @@ function getOptions(options) {
   return isObject(options) ? options : {}
 }
 
+const defaultAssetOptions = {
+  // Only remove unused or dangling assets that haven't seen changes for
+  // these given time frames. Set to `0` to clean up instantly.
+  cleanupTimeThreshold: '24h',
+  // Dangling assets are those that got uploaded but never actually persisted in
+  // the model. This can happen when the admin uploads a file but doesn't store
+  // the associated form. This cannot be set to 0 or else the the file would be
+  // deleted immediately after upload.
+  danglingTimeThreshold: '24h'
+}
+
 const { err, req, res } = pino.stdSerializers
 const defaultLoggerOptions = {
   level: 'info',

package/src/controllers/CollectionController.js

@@ -24,6 +24,7 @@ export class CollectionController extends Controller {
 
   // @override
   setup() {
+    this.logController()
     this.collection = this.setupActions('collection')
     this.member = this.isOneToOne ? {} : this.setupActions('member')
     this.assets = this.setupAssets()

package/src/controllers/Controller.js

@@ -26,7 +26,7 @@ export class Controller {
   constructor(app, namespace) {
     this.app = app
     this.namespace = namespace
-    this.logging = this.app.config.log.routes
+    this.logRoutes = this.app.config.log.routes
     this.level = 0
   }
 
@@ -60,21 +60,12 @@ export class Controller {
       // mapped parameters or wildcards. Consider `path` / `route` instead?
       const url = path ? `/${path}` : ''
       this.url = namespace ? `/${namespace}${url}` : url
-      this.log(
-        `${
-          namespace ? pico.green(`/${namespace}/`) : ''
-        }${
-          pico.cyan(path)
-        }${
-          pico.white(':')
-        }`,
-        this.level
-      )
     }
   }
 
   // @overridable
   setup() {
+    this.logController()
     this.actions ||= this.reflectActionsObject()
     // Now that the instance fields are reflected in the `controller` object
     // we can use the normal inheritance mechanism through `setupActions()`:
@@ -94,6 +85,40 @@ export class Controller {
     // middleware. For normal routes, use `this.app.addRoute()` instead.
   }
 
+  // @overridable
+  logController() {
+    const { path, namespace } = this
+    this.logRoute(
+      `${
+        namespace ? pico.green(`/${namespace}/`) : ''
+      }${
+        pico.cyan(path)
+      }${
+        pico.white(':')
+      }`,
+      this.level
+    )
+  }
+
+  logRoute(str, indent = 0) {
+    if (this.logRoutes) {
+      console.info(`${'  '.repeat(indent)}${str}`)
+    }
+  }
+
+  // Only use this method to get a logger instance that is bound to the context,
+  // otherwise use the cached getter.
+  getLogger(ctx) {
+    const logger = ctx?.logger ?? this.app.logger
+    return logger.child({ name: this.name })
+  }
+
+  get logger() {
+    const value = this.getLogger()
+    Object.defineProperty(this, 'logger', { value })
+    return value
+  }
+
   reflectActionsObject() {
     // On base controllers, the actions can be defined directly in the class
     // instead of inside an actions object, as is done with model and relation
@@ -122,7 +147,7 @@ export class Controller {
   }
 
   setupRoute(method, url, transacted, authorize, action, middlewares) {
-    this.log(
+    this.logRoute(
       `${
         pico.magenta(method.toUpperCase())
       } ${
@@ -382,7 +407,6 @@ export class Controller {
         // so add its own keys to the already allowed inherited keys so far.
         Object.assign(allowMap, getFilteredMap(getOwnKeys(current)))
       }
-      // console.log('allow', Object.keys(allowMap))
     }
 
     const handleAuthorize = authorize => {
@@ -538,12 +562,6 @@ export class Controller {
       throw new AuthorizationError()
     }
   }
-
-  log(str, indent = 0) {
-    if (this.logging) {
-      console.info(`${'  '.repeat(indent)}${str}`)
-    }
-  }
 }
 
 EventEmitter.mixin(Controller.prototype)

package/src/controllers/RelationController.js

@@ -48,6 +48,11 @@ export class RelationController extends CollectionController {
     super.configure()
   }
 
+  // @override
+  logController() {
+    // The parent controller logs itself already, that's enough.
+  }
+
   // @override
   inheritValues(type) {
     // Since RelationController are mapped to nested `relations` objects in

package/src/services/Service.js

@@ -32,7 +32,7 @@ export class Service {
   }
 
   // Only use this method to get a logger instance that is bound to the context,
-  // otherwise use the getter.
+  // otherwise use the cached getter.
   getLogger(ctx) {
     const logger = ctx?.logger ?? this.app.logger
     return logger.child({ name: this.#loggerName })

package/src/storage/DiskStorage.js

@@ -44,7 +44,6 @@ export class DiskStorage extends Storage {
     const dir = path.dirname(filePath)
     await fs.mkdir(dir, { recursive: true })
     await fs.writeFile(filePath, data)
-    return file
   }
 
   // @override

package/src/storage/S3Storage.js

@@ -111,12 +111,10 @@ export class S3Storage extends Storage {
       ContentType: file.type,
       Body: data
     })
-    // "Convert" `file` to something looking more like a S3 `storageFile`.
-    // For now, only the `location` property is of interest:
-    return {
-      ...file,
-      location: result.Location
-    }
+    // In `Storage.addFile()` this will get overridden with the result of
+    // `_getUrl()` if it exists, but is used as a fallback otherwise,
+    // see `_getFileUrl()`.
+    file.url = result.Location
   }
 
   // @override

package/src/storage/Storage.js

@@ -18,7 +18,9 @@ export class Storage {
     this.name = config.name
     this.url = config.url
     this.path = config.path
-    this.concurrency = config.concurrency ?? null
+    // Use a default concurrency of 8 for storage IO, e.g. the importing of
+    // foreign assets.
+    this.concurrency = config.concurrency ?? 8
     // The actual multer storage object.
     this.storage = null
   }
@@ -96,10 +98,10 @@ export class Storage {
   }
 
   async addFile(file, data) {
-    const storageFile = await this._addFile(file, data)
+    await this._addFile(file, data)
     file.size = Buffer.byteLength(data)
-    file.url = this._getFileUrl(storageFile)
-    // TODO: Support `config.readImageSize`, but this can only be done onces
+    file.url = this._getFileUrl(file)
+    // TODO: Support `config.readImageSize`, but this can only be done once
     // there are separate storage instances per model assets config!
     return this.convertAssetFile(file)
   }

package/src/utils/date.js

@@ -0,0 +1,15 @@
+import { isNumber } from '@ditojs/utils'
+import parseDuration from 'parse-duration'
+
+export function getDuration(duration) {
+  return isNumber(duration) ? duration : parseDuration(duration)
+}
+
+export function addDuration(date, duration) {
+  date.setMilliseconds(date.getMilliseconds() + getDuration(duration))
+  return date
+}
+
+export function subtractDuration(date, duration) {
+  return addDuration(date, -getDuration(duration))
+}

package/src/utils/index.js

@@ -1,3 +1,4 @@
+export * from './date.js'
 export * from './decorator.js'
 export * from './deprecate.js'
 export * from './emitter.js'