dobo 2.20.1 → 2.21.0

package/extend/bajo/intl/en-US.json

@@ -150,6 +150,7 @@
   "maxPageError%s%s": "Page number (%s) above the allowed threshold (%s)",
   "duplicateRefKeys%s%s": "Duplicate reference keys found in '%s' (%s)",
   "sanitizeBodyError": "Error sanitizing body",
+  "virtualFieldIn%s%s": "Virtual field can't be used in '%s' on %s",
   "field": {
     "id": "ID",
     "code": "Kode",

package/extend/bajo/intl/id.json

@@ -148,6 +148,7 @@
   "maxPageError%s%s": "Nomor halaman (%s) melampaui batas yang diijinkan (%s)",
   "duplicateRefKeys%s%s": "Ditemukan kunci referensi duplikat di '%s' (%s)",
   "sanitizeBodyError": "Kesalahan saat sanitasi body",
+  "virtualFieldIn%s%s": "Kolom virtual tidak bisa digunakan di '%s' pada %s",
   "field": {
     "id": "ID",
     "code": "Kode",

package/extend/dobo/feature/dt.js

@@ -1,6 +1,6 @@
 async function dt (opts = {}) {
   opts.field = opts.field ?? 'dt'
-  opts.type = opts.type ??'datetime'
+  opts.type = opts.type ?? 'datetime'
   opts.formatInt = opts.formatInt ?? false
   opts.formatValueInt = opts.formatValueInt ?? false
   const prop = {
@@ -12,13 +12,13 @@ async function dt (opts = {}) {
   if (opts.type === 'integer') {
     if (opts.formatInt) {
       prop.format = async function (val, data, { req } = {}) {
-        const dt = new Date(data._orig[opts.field])
+        const dt = new Date(data[opts.field])
         return req ? req.format(dt, 'datetime') : this.app.bajo.format(dt, 'datetime', { lang: req.lang })
       }
     }
     if (opts.formatValueInt) {
       prop.format = async function (val, data, { req } = {}) {
-        return new Date(data._orig[opts.field])
+        return new Date(data[opts.field])
       }
     }
   }

package/index.js

@@ -82,7 +82,7 @@ const propertyType = {
   }
 }
 
-const commonPropertyTypes = ['name', 'type', 'required', 'rules', 'validator', 'ref', 'default', 'values', 'rulesMsg', 'immutable', 'feature', 'format']
+const commonPropertyTypes = ['name', 'type', 'required', 'rules', 'validator', 'ref', 'default', 'values', 'rulesMsg', 'immutable', 'feature', 'format', 'getValue', 'virtual']
 
 /**
  * Plugin factory

package/lib/collect-models.js

@@ -28,20 +28,29 @@ async function sanitizeProp (model, prop, indexes) {
       return { value: item, text: item }
     })
   } else if (!isString(prop.values)) delete prop.values
-  if (prop.index) {
-    if (prop.index === true || prop.index === 'true') prop.index = 'index'
-    const [idx, idxName] = prop.index.split(':')
-    const index = { name: idxName ?? `${model.collName}_${prop.name}_${idx}`, fields: [prop.name], type: idx }
-    indexes.push(index)
-  }
   if (prop.hidden) model.hidden.push(prop.name)
-  if (keys(propType).includes(prop.type)) model.properties.push(pick(prop, allPropKeys))
-  else {
-    const feature = this.getFeature(prop.type)
-    if (!feature) this.fatal('unknownPropType%s%s', prop.type, model.name)
-    const opts = omit(prop, ['name', 'type'])
-    opts.field = prop.name
-    await applyFeature.call(this, model, feature, opts, indexes)
+  if (prop.virtual) {
+    const keys = Object.keys(propType)
+    if (!keys.includes(prop.type)) this.fatal('unknownPropType%s%s', `${prop.name}:${prop.type}`, model.name)
+    for (const key of ['required', 'rules', 'index', 'validator', 'ref', 'rulesMsg', 'immutable', 'feature']) {
+      delete prop[key]
+    }
+    model.properties.push(prop)
+  } else {
+    if (prop.index) {
+      if (prop.index === true || prop.index === 'true') prop.index = 'index'
+      const [idx, idxName] = prop.index.split(':')
+      const index = { name: idxName ?? `${model.collName}_${prop.name}_${idx}`, fields: [prop.name], type: idx }
+      indexes.push(index)
+    }
+    if (keys(propType).includes(prop.type)) model.properties.push(pick(prop, allPropKeys))
+    else {
+      const feature = this.getFeature(prop.type)
+      if (!feature) this.fatal('unknownPropType%s%s', prop.type, model.name)
+      const opts = omit(prop, ['name', 'type'])
+      opts.field = prop.name
+      await applyFeature.call(this, model, feature, opts, indexes)
+    }
   }
 }
 
@@ -330,8 +339,19 @@ async function collectModels () {
     const model = new DoboModel(plugin, schema)
     me.models.push(model)
   }
+  // last sanitizing & checking
   for (const model of me.models) {
     await sanitizeRef.call(this, model, me.models)
+    for (const item of model.indexes) {
+      for (const field of item.fields) {
+        const prop = model.properties.find(p => p.name === field)
+        if (!prop || (prop && prop.virtual)) throw this.error('virtualFieldIn%s%s', 'index', model.name)
+      }
+    }
+    for (const field of model.scanable) {
+      const prop = model.properties.find(p => p.name === field)
+      if (!prop || (prop && prop.virtual)) throw this.error('virtualFieldIn%s%s', 'scanable', model.name)
+    }
   }
   this.log.debug('collected%s%d', this.t('model'), this.models.length)
 }

package/lib/factory/driver.js

@@ -12,7 +12,7 @@ const defIdField = {
 
 async function driverFactory () {
   const { Tools } = this.app.baseClass
-  const { cloneDeep, has, uniq, without, isEmpty } = this.app.lib._
+  const { pick, cloneDeep, has, uniq, without, isEmpty, omit, isFunction } = this.app.lib._
   const { isSet } = this.app.lib.aneka
 
   /**
@@ -167,10 +167,17 @@ async function driverFactory () {
       return await this.dropModel(model, options)
     }
 
+    getRealFields (model) {
+      return model.getProperties({ noVirtual: true, namesOnly: true })
+    }
+
+    getVirtualFields (model) {
+      return model.getVirtualProperties({ namesOnly: true })
+    }
+
     async _prepBodyForCreate (model, body = {}, options = {}) {
       const { isSet, generateId } = this.app.lib.aneka
-      const { isFunction } = this.app.lib._
-      for (const prop of model.properties) {
+      for (const prop of model.getProperties({ noVirtual: true })) {
         if (!isSet(body[prop.name]) && isSet(prop.default)) {
           if (isFunction(prop.default)) body[prop.name] = await prop.default.call(model)
           else if (typeof prop.default !== 'string') body[prop.name] = prop.default
@@ -198,6 +205,7 @@ async function driverFactory () {
           }
         }
       }
+      return pick(body, this.getRealFields(model))
     }
 
     async _prepIdForCreate (model, body = {}, options = {}) {
@@ -221,19 +229,19 @@ async function driverFactory () {
       result.warnings.push(...(options.warnings ?? []))
     }
 
-    async _createRecord (model, body = {}, options = {}) {
+    async _createRecord (model, input = {}, options = {}) {
       const { isSet } = this.app.lib.aneka
-      await this._prepBodyForCreate(model, body, options)
+      let body = await this._prepBodyForCreate(model, input, options)
       await this._prepIdForCreate(model, body, options)
       if (!options.noUniqueCheck) {
         if (!this.support.uniqueIndex) await this._checkUnique(model, body, options)
       }
       if (!options.noIdCheck && isSet(body.id)) {
-        const resp = await this.getRecord(model, body.id, { noHook: true })
+        const resp = await this.getRecord(model, body.id, { noMagic: true })
         if (!isEmpty(resp.data)) throw this.plugin.error('recordExists%s%s', body.id, model.name)
       }
-      const input = this.sanitizeBody(model, body)
-      const result = await this.createRecord(model, input, options)
+      body = this.sanitizeBody(model, body)
+      const result = await this.createRecord(model, body, options)
       if (options.noResult) return
       result.data = this.sanitizeRecord(model, result.data)
       this._injectMeta(result, options)
@@ -245,8 +253,7 @@ async function driverFactory () {
       let { chunkSize = this.maxChunkSize } = options
       if (chunkSize > this.maxChunkSize) chunkSize = this.maxChunkSize
       for (const idx in bodies) {
-        const body = bodies[idx]
-        await this._prepBodyForCreate(model, body, options)
+        const body = await this._prepBodyForCreate(model, bodies[idx], options)
         await this._prepIdForCreate(model, body, options)
         bodies[idx] = this.sanitizeBody(model, body)
       }
@@ -264,18 +271,19 @@ async function driverFactory () {
       return result
     }
 
-    async _updateRecord (model, id, body = {}, options = {}) {
+    async _updateRecord (model, id, input = {}, options = {}) {
+      let body = omit(input, this.getVirtualFields())
       if (!options.noUniqueCheck) {
         if (!this.support.uniqueIndex) await this._checkUnique(model, body, options)
       }
       if (!options._data) {
-        const resp = await this.getRecord(model, id, { noHook: true })
+        const resp = await this.getRecord(model, id, { noMagic: true })
         if (!resp.data) throw this.plugin.error('recordNotFound%s%s', id, model.name)
         options._data = resp.data
       }
-      const input = this.sanitizeBody(model, body, true)
-      delete input.id
-      const result = await this.updateRecord(model, id, input, options)
+      body = this.sanitizeBody(model, body, true)
+      delete body.id
+      const result = await this.updateRecord(model, id, body, options)
       if (options.noResult) return
       result.oldData = this.sanitizeRecord(model, result.oldData)
       result.data = this.sanitizeRecord(model, result.data)
@@ -283,19 +291,20 @@ async function driverFactory () {
       return result
     }
 
-    async _upsertRecord (model, body = {}, options = {}) {
+    async _upsertRecord (model, input = {}, options = {}) {
+      let body = omit(input, this.getVirtualFields())
       if (!options.noUniqueCheck) {
         if (!this.uniqueIndexSupport) await this._checkUnique(model, body, options)
       }
       if (isSet(body.id)) {
         if (!options._data) {
-          const resp = await this.getRecord(model, body.id, { noHook: true })
+          const resp = await this.getRecord(model, body.id, { noMagic: true })
           if (!resp.data) throw this.plugin.error('recordNotFound%s%s', body.id, model.name)
           options._data = resp.data
         }
       }
-      const input = this.sanitizeBody(model, body)
-      const result = await this.upsertRecord(model, input, options)
+      body = this.sanitizeBody(model, body)
+      const result = await this.upsertRecord(model, body, options)
       if (options.noResult) return
       if (result.oldData) result.oldData = this.sanitizeRecord(model, result.oldData)
       result.data = this.sanitizeRecord(model, result.data)
@@ -305,7 +314,7 @@ async function driverFactory () {
 
     async _removeRecord (model, id, options = {}) {
       if (!options._data) {
-        const resp = await this.getRecord(model, id, { noHook: true })
+        const resp = await this.getRecord(model, id, { noMagic: true })
         if (!resp.data) throw this.plugin.error('recordNotFound%s%s', id, model.name)
         options._data = resp.data
       }

package/lib/factory/model/_util.js

@@ -173,10 +173,10 @@ export async function handleAttachmentUpload (id, trigger, options = {}) {
 async function _getRef ({ ref, rModel, prop, key, options, filter } = {}) {
   if (!((typeof options.refs === 'string' && ['*', 'all'].includes(options.refs)) || options.refs.includes(key))) return
   if (ref.fields.length === 0) return
-  const { formatValue, retainOriginalValue } = options
+  const { fmt } = options
   const fields = [...ref.fields]
   if (!fields.includes(prop.name)) fields.push(prop.name)
-  const rOptions = { dataOnly: true, refs: [], formatValue, retainOriginalValue, fields }
+  const rOptions = { dataOnly: true, refs: [], fmt, fields }
   const results = await rModel.findRecord(filter, rOptions)
   return { rOptions, results }
 }
@@ -234,7 +234,7 @@ export async function getMultiRefs (records = [], options = {}) {
           for (const r of records) {
             matches.push(rModel.sanitizeId(r[prop.name]))
           }
-          matches = uniq(without(matches, undefined, null, NaN))
+          matches = uniq(without(matches, undefined, null, NaN)).map(i => i + '')
           let query = {}
           query[ref.field] = { $in: matches }
           if (ref.query) query = { $and: [query, parseQuery(ref.query, rModel)] }
@@ -290,14 +290,20 @@ function sanitizeQuery (query = {}, parent) {
 
   keys.forEach(k => {
     const v = obj[k]
-    const prop = find(this.properties, { name: k })
-    if (isPlainObject(v)) obj[k] = sanitizeQuery.call(this, v, k)
-    else if (isArray(v)) {
-      v.forEach((i, idx) => {
-        if (isPlainObject(i)) obj[k][idx] = sanitizeQuery.call(this, i, k)
-        else obj[k][idx] = sanitizeField(prop, i)
-      })
-    } else obj[k] = sanitizeChild(k, v, parent)
+    const props = this.getProperties({ noVirtual: true })
+    const fields = props.map(p => p.name)
+    const prop = find(props, { name: k })
+    if (k[0] !== '$' && !fields.includes(k)) {
+      delete keys[k]
+    } else {
+      if (isPlainObject(v)) obj[k] = sanitizeQuery.call(this, v, k)
+      else if (isArray(v)) {
+        v.forEach((i, idx) => {
+          if (isPlainObject(i)) obj[k][idx] = sanitizeQuery.call(this, i, k)
+          else obj[k][idx] = sanitizeField(prop, i)
+        })
+      } else obj[k] = sanitizeChild(k, v, parent)
+    }
   })
   return obj
 }
@@ -439,3 +445,17 @@ export async function clearCache (id) {
   await clear({ key: `dobo|${this.name}|findAllRecord` })
   await clear({ key: `dobo|${this.name}|findOneRecord` })
 }
+
+export async function buildPropValues (prop, opts) {
+  const { isString, camelCase } = this.app.lib._
+  const { callHandler } = this.app.bajo
+  const values = (isString(prop.values) ? await callHandler(prop.values) : [...prop.values]).map(v => {
+    if (isString(v)) v = { value: v, text: v }
+    if (opts.req) {
+      const key = camelCase(`${prop.name} ${v.text}`)
+      if (opts.req.te(key)) v.text = opts.req.t(key)
+    }
+    return v
+  })
+  return values
+}

package/lib/factory/model/load-fixtures.js

@@ -4,49 +4,39 @@ async function exec ({ item, spinner, options, result, items } = {}) {
   const { isArray, isString } = this.app.lib._
   const { getPluginDataDir } = this.app.bajo
   const { fs } = this.app.lib
-  const resp = await this.createRecord(item, options)
-  if (isArray(item._attachments) && item._attachments.length > 0) {
-    for (let att of item._attachments) {
-      if (isString(att)) att = { field: 'file', file: att }
-      const fname = path.basename(att.file)
-      if (fs.existsSync(att.file)) {
-        const dest = `${getPluginDataDir(this.plugin.ns)}/${resp.id}/${att.field}/${fname}`
-        try {
-          fs.copySync(att.file, dest)
-        } catch (err) {}
+
+  await this.transaction(async (trx) => {
+    const resp = await this.createRecord(item, { ...options, trx })
+    if (isArray(item._attachments) && item._attachments.length > 0) {
+      for (let att of item._attachments) {
+        if (isString(att)) att = { field: 'file', file: att }
+        const fname = path.basename(att.file)
+        if (fs.existsSync(att.file)) {
+          const dest = `${getPluginDataDir(this.plugin.ns)}/${resp.id}/${att.field}/${fname}`
+          try {
+            fs.copySync(att.file, dest)
+          } catch (err) {}
+        }
       }
     }
-  }
+  })
   result.success++
   if (spinner) spinner.setText('recordsAdded%s%d%d', this.name, result.success, items.length)
 }
 
 async function loadFixtures ({ spinner, ignoreError = true, collectItems = false, noLookup = false } = {}, options = {}) {
-  const { readConfig, eachPlugins } = this.app.bajo
+  const { readConfig } = this.app.bajo
   const { resolvePath } = this.app.lib.aneka
-  const { isEmpty, isArray, omit } = this.app.lib._
+  const { isEmpty } = this.app.lib._
   if (this.connection.proxy) {
     this.log.warn('proxiedConnBound%s', this.name)
     return
   }
   const result = { success: 0, failed: 0 }
   const base = path.basename(this.file, path.extname(this.file))
-  // original
   const pattern = resolvePath(`${path.dirname(this.file)}/../fixture/${base}.*`)
-  let items = await readConfig(pattern, { ns: this.plugin.ns, ignoreError: true })
-  if (isEmpty(items)) items = []
-  // override
-  const overrides = await readConfig(`${this.app.main.dir.pkg}/extend/dobo/override/${this.plugin.ns}/fixture/${base}.*`, { ns: this.plugin.ns, ignoreError: true })
-
-  if (isArray(overrides) && !isEmpty(overrides)) items = overrides
-  // extend
-  const me = this
-  await eachPlugins(async function ({ dir }) {
-    const { ns } = this
-    const extend = await readConfig(`${dir}/extend/dobo/extend/${me.plugin.ns}/fixture/${base}.*`, { ns, ignoreError: true })
-    if (isArray(extend) && !isEmpty(extend)) items.push(...extend)
-  })
-  const opts = { noHook: true, noCache: true, ...(omit(options, ['noHook', 'noCache'])) }
+  const items = await readConfig(pattern, { ns: this.plugin.ns, baseNs: 'dobo', checkOverride: true, defValue: [] })
+  const opts = { ...options, noMagic: true }
   for (const item of items) {
     for (const k in item) {
       const v = item[k]
@@ -63,14 +53,14 @@ async function loadFixtures ({ spinner, ignoreError = true, collectItems = false
   for (const item of items) {
     if (ignoreError) {
       try {
-        await exec.call(this, { item, spinner, opts, options, result, items })
+        await exec.call(this, { item, spinner, options, result, items })
       } catch (err) {
-        // console.log(err)
+        if (this.app.bajo.config.log.applet) console.error(err)
         err.model = this.name
         if (this.app.applet) this.plugin.print.fail(this.app.dobo.validationErrorMessage(err))
         result.failed++
       }
-    } else await exec.call(this, { item, spinner, opts, options, result, items })
+    } else await exec.call(this, { item, spinner, options, result, items })
   }
   return result
 }

package/lib/factory/model/sanitize-record.js

@@ -1,3 +1,5 @@
+import { buildPropValues } from './_util.js'
+
 /**
  * Sanitize record to conform with the model's definition
  *
@@ -23,39 +25,33 @@ async function sanitizeRecord (record = {}, opts = {}) {
   newFields = without(newFields, ...allHidden)
   const body = fillObject(record, newFields, null)
   const newRecord = await this.sanitizeBody({ body, noDefault: true })
-  if (opts.formatValue) {
-    if (opts.retainOriginalValue) newRecord._orig = cloneDeep(newRecord)
+  for (const key in newRecord) {
+    const prop = this.getProperty(key)
+    const val = ['object', 'array'].includes(prop.type) ? cloneDeep(newRecord[key]) : newRecord[key]
+    if (isFunction(prop.getValue)) newRecord[key] = await prop.getValue.call(this, val, newRecord, opts)
+    else if (isString(prop.getValue)) newRecord[key] = await callHandler(this.plugin, this, val, newRecord, opts)
+  }
+
+  if (opts.fmt) {
+    newRecord._fmt = cloneDeep(newRecord)
     for (const key in newRecord) {
       const prop = this.getProperty(key)
       if (!prop) continue
-      if (prop.formatValue && opts.retainOriginalValue) {
-        const val = ['object', 'array'].includes(prop.type) ? cloneDeep(newRecord._orig[key]) : newRecord._orig[key]
-        if (isFunction(prop.formatValue)) newRecord._orig[key] = await prop.formatValue.call(this, val, newRecord._orig, opts)
-        else if (isString(prop.formatValue)) newRecord._orig[key] = await callHandler(this.plugin, this, val, newRecord._orig, opts)
-      }
       let value = ['object', 'array'].includes(prop.type) ? cloneDeep(newRecord[key]) : newRecord[key]
       if (prop.values) {
-        const values = (isString(prop.values) ? await callHandler(prop.values) : [...prop.values]).map(v => {
-          if (isString(v)) v = { value: v, text: v }
-          if (opts.req) {
-            const { camelCase } = this.app.lib._
-            const key = camelCase(`${prop.name} ${v.text}`)
-            if (opts.req.te(key)) v.text = opts.req.t(key)
-          }
-          return v
-        })
+        const values = await buildPropValues.call(this, prop, opts)
         value = (values.find(v => v.value === value) ?? {}).text ?? value
       }
-      if (prop.format === false) newRecord[key] = value + ''
-      else if (isFunction(prop.format)) newRecord[key] = await prop.format.call(this, value, newRecord, opts)
-      else if (isString(prop.format)) newRecord[key] = await callHandler(this.plugin, this, value, newRecord, opts)
+      if (prop.format === false) newRecord._fmt[key] = value + ''
+      else if (isFunction(prop.format)) newRecord._fmt[key] = await prop.format.call(this, value, newRecord, opts)
+      else if (isString(prop.format)) newRecord._fmt[key] = await callHandler(this.plugin, this, value, newRecord, opts)
       else {
         const options = {
           lang: get(opts, 'req.lang'),
           latitude: ['lat', 'latitude'].includes(key),
           longitude: ['lon', 'lng', 'longitude'].includes(key)
         }
-        newRecord[key] = format(value, prop.type, options)
+        newRecord._fmt[key] = format(value, prop.type, options)
       }
     }
   }

package/lib/factory/model.js

@@ -91,6 +91,20 @@ async function modelFactory () {
       return this.properties.find(prop => prop.name === name)
     }
 
+    getProperties = ({ noVirtual, namesOnly } = {}) => {
+      const items = noVirtual ? this.properties.filter(prop => !prop.virtual) : this.properties
+      return namesOnly ? items.map(item => item.name) : items
+    }
+
+    getVirtualProperties = ({ namesOnly }) => {
+      const items = this.properties.filter(prop => prop.virtual)
+      return namesOnly ? items.map(item => item.name) : items
+    }
+
+    getIndexes = () => {
+      return this.indexes
+    }
+
     hasProperty = (name) => {
       return !!this.getProperty(name)
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dobo",
-  "version": "2.20.1",
+  "version": "2.21.0",
   "description": "DBMS for Bajo Framework",
   "main": "index.js",
   "scripts": {

package/wiki/CHANGES.md

@@ -1,5 +1,18 @@
 # Changes
 
+## 2026-04-25
+
+- [2.21.0] Change ```options.formatValue``` to ```options.fmt```
+- [2.21.0] Remove ```options.retainOriginalValue``` since it is not needed anymore
+- [2.21.0] Remove ```formatValue``` altogether
+- [2.21.0] Remove ```record._orig```, in exchange switch the formatted value to ```record._fmt```
+- [2.21.0] Add ```property.virtual``` to set property is a virtual/calculated property
+- [2.21.0] Add necessary safe guards for virtual property
+- [2.21.0] Add ```model.getProperties()```
+- [2.21.0] Add ```model.getIndexes()```
+- [2.21.0] Add ```model.getVirtualPropertties()```
+- [2.21.0] Activate transaction on ```loadFixtures()```
+
 ## 2026-04-23
 
 - [2.20.1] Bug fix in ```collect-models.js```, now with deep merge in advance