@tetherto/mdk-core 0.0.0 → 0.12.1

package/lib/services/stats.service.js

@@ -0,0 +1,205 @@
+'use strict'
+
+const debug = require('debug')('mdk:stats')
+const async = require('async')
+const utilsStore = require('@tetherto/hp-svc-facs-store/utils')
+const gLibStats = require('../stats')
+
+const STAT_RTD = 'stat-rtd'
+
+/**
+ * Aggregation over per-device `last` snapshots. `lib` is the stats spec
+ * module ({ specs, conf }) passed in directly. Device state comes from
+ * injected accessors:
+ *   listDeviceIds()        → [deviceId]
+ *   getDeviceMeta(id)      → { id, tags, info, opts, type }
+ *   getLast(id)            → { snap, alerts, err, ts } | undefined
+ *   getRealtimeData(id)    → latest snap without a device round-trip
+ */
+class StatsService {
+  constructor ({ lib, specTags, baseType, listDeviceIds, getDeviceMeta, getLast, getRealtimeData, processAlerts, logs, conf }) {
+    if (!logs) throw new Error('ERR_STATS_LOGS_REQUIRED')
+    this.lib = lib || null
+    this.specTags = specTags || []
+    this.baseType = baseType || 'thing'
+    this.listDeviceIds = listDeviceIds || (() => [])
+    this.getDeviceMeta = getDeviceMeta || (() => null)
+    this.getLast = getLast || (() => undefined)
+    this.getRealtimeData = getRealtimeData || null
+    this.processAlerts = processAlerts || (() => null)
+    this.logs = logs
+    this.conf = conf || {}
+    this._buildingStats = {}
+    this._collectingRtd = false
+  }
+
+  async buildStats (sk, fireTime) {
+    if (sk === STAT_RTD) {
+      return await this.saveRealTimeData()
+    }
+
+    if (!this.lib) {
+      return
+    }
+
+    const lkr = `_buildingStats_${sk}`
+
+    if (this._buildingStats[lkr]) {
+      return
+    }
+
+    this._buildingStats[lkr] = true
+
+    try {
+      await this.buildStatsForKey(sk, fireTime)
+    } finally {
+      this._buildingStats[lkr] = false
+    }
+  }
+
+  async saveRealTimeData () {
+    if (this._collectingRtd) return
+    if (!this.getRealtimeData) return
+    this._collectingRtd = true
+
+    const thgsRtd = {}
+
+    try {
+      await async.eachLimit(
+        this.listDeviceIds(),
+        this.conf.thingRtdConcurrency || 500,
+        async (deviceId) => {
+          const meta = this.getDeviceMeta(deviceId)
+          if (!meta) return
+          thgsRtd[deviceId] = { ...meta, last: this.getLast(deviceId) || {} }
+          try {
+            const rtd = await this.getRealtimeData(deviceId)
+            if (!rtd) return
+
+            const dev = { ...meta, last: this.getLast(deviceId) || {} }
+            thgsRtd[deviceId] = {
+              ...meta,
+              last: { snap: rtd, alerts: this.processAlerts(dev) }
+            }
+          } catch (e) {
+            debug('ERR_RTD %s %s', deviceId, e.message)
+          }
+        }
+      )
+    } finally {
+      this._collectingRtd = false
+    }
+
+    if (!Object.keys(thgsRtd).length) return
+
+    try {
+      const aggrData = this.aggrStats(Object.keys(thgsRtd), {}, thgsRtd)
+      const log = await this.logs.getBeeTimeLog(
+        `${STAT_RTD}-t-${this.baseType}`,
+        0,
+        true
+      )
+      await log.put(STAT_RTD, Buffer.from(JSON.stringify(aggrData)))
+      await this.logs.releaseBeeTimeLog(log)
+    } catch (e) {
+      debug('ERR_SAVING_RTD %s', e.message)
+    }
+  }
+
+  aggrStats (thgIds, opts = {}, thgs = null) {
+    const acc = {}
+    if (!this.lib) return acc
+
+    const specs = this.lib.specs
+    const ids = thgIds || this.listDeviceIds()
+
+    this.specTags.forEach(stype => {
+      if (!specs[stype]) {
+        return
+      }
+
+      const state = {}
+      state.ops = specs[stype].ops
+
+      for (const thgId of ids) {
+        const thg = thgs ? thgs[thgId] : this._deviceEntry(thgId)
+        if (!thg) continue
+        const last = thg.last || {}
+        const info = thg.info || {}
+        gLibStats.applyStats(
+          state,
+          acc,
+          { last, info },
+          { id: thg.id, tags: thg.tags, info, opts: thg.opts, type: thg.type }
+        )
+      }
+
+      gLibStats.tallyStats(state, acc)
+    })
+
+    return acc
+  }
+
+  _deviceEntry (deviceId) {
+    const meta = this.getDeviceMeta(deviceId)
+    if (!meta) return null
+    return { ...meta, last: this.getLast(deviceId) || {} }
+  }
+
+  async buildStatsForKey (logKey, fireTime) {
+    const now = Math.floor(fireTime.getTime() / 1000) * 1000
+
+    // Null prototype: a stored tag named `__proto__` or `constructor` must not
+    // resolve to a built-in and break the build for every device.
+    const tagSpecs = Object.create(null)
+    const thgIdsAll = this.listDeviceIds()
+    if (!this.lib) return
+
+    const skipTagPfxs = this.lib.conf?.skipTagPrefixes || []
+
+    thgIdsAll.forEach(thgId => {
+      const meta = this.getDeviceMeta(thgId)
+      if (!meta) return
+
+      const tags = Array.isArray(meta.tags) ? meta.tags : []
+      tags.forEach(tag => {
+        if (typeof tag !== 'string' || !tag) return
+        if (skipTagPfxs.find(p => tag.startsWith(p))) {
+          return
+        }
+
+        if (!tagSpecs[tag]) {
+          tagSpecs[tag] = { thgIds: [] }
+        }
+
+        tagSpecs[tag].thgIds.push(thgId)
+      })
+    })
+
+    const tags = Object.keys(tagSpecs)
+
+    for (const tag of tags) {
+      const tagSpec = tagSpecs[tag]
+
+      const acc = this.aggrStats(tagSpec.thgIds)
+      const log = await this.logs.getBeeTimeLog(`${logKey}-${tag}`, 0, true)
+
+      try {
+        await log.put(
+          utilsStore.convIntToBin(now),
+          Buffer.from(JSON.stringify({
+            ts: now,
+            ...acc
+          }))
+        )
+      } catch (e) {
+        debug('ERR_BUILD_STATS %s %s', tag, e.message)
+      }
+
+      await this.logs.releaseBeeTimeLog(log)
+    }
+  }
+}
+
+module.exports = StatsService
+module.exports.STAT_RTD = STAT_RTD

package/lib/stats/README.md

@@ -0,0 +1,80 @@
+# Lib-stats
+
+Telemetry aggregation library. Provides a set of composable statistical operations that device Workers use to compute time-bucketed metrics (hashrate averages, power totals, group counts, etc.).
+
+Part of `@tetherto/mdk-core` and exported as `libStats`. It is used by `StatsService`
+([`lib/services/stats.service.js`](../services/stats.service.js)) and by each Worker plugin's `lib/templates/stats.js`, for example
+[the Avalon plugin's](../../../../workers/miners/avalon/lib/templates/stats.js).
+
+## Usage
+
+```js
+const { libStats } = require('@tetherto/mdk-core')
+const { applyStats, tallyStats, defaults } = libStats
+
+// Define what to compute
+const state = {
+  ops: {
+    total_power: { op: 'sum', field: 'power_w' },
+    avg_hashrate: { op: 'avg', field: 'hashrate_mhs' },
+    by_container: { op: 'group_by', field: 'container' }
+  }
+}
+
+const pack = {}
+
+// Feed entries one at a time
+for (const entry of deviceSnapshots) {
+  applyStats(state, pack, entry, extData)
+}
+
+// Finalize (resolve running accumulators to result values)
+tallyStats(state, pack)
+
+// pack.total_power, pack.avg_hashrate, pack.by_container now contain results
+```
+
+## API
+
+### `applyStats(state, pack, entry, ext)`
+
+Apply all configured operations to a single `entry`. Accumulates intermediate state in `pack`. Call once per device snapshot per time bucket.
+
+### `tallyStats(state, pack)`
+
+Finalize the accumulation. Resolves running counters/sums into their final values and replaces each key in `pack` with its `.res` field.
+
+### `defaults.timeframes`
+
+Array of `[label, cronExpression]` pairs for the standard time buckets:
+
+```js
+[
+  ['5m',  '0 */5 * * * *'],
+  ['30m', '0 */30 * * * *'],
+  ['3h',  '0 0 */3 * * *'],
+  ['1D',  '0 0 0 * * *']
+]
+```
+
+## Operations
+
+| Op name | File | Description |
+|---------|------|-------------|
+| `cnt` | [`ops/cnt.js`](./ops/cnt.js) | Count entries |
+| `sum` | [`ops/sum.js`](./ops/sum.js) | Numeric sum |
+| `avg` | [`ops/avg.js`](./ops/avg.js) | Running average |
+| `group` | [`ops/group.js`](./ops/group.js) | Group entries by a field |
+| `group_sum` | [`ops/group_sum.js`](./ops/group_sum.js) | Sum within groups |
+| `group_avg` | [`ops/group_avg.js`](./ops/group_avg.js) | Average within groups |
+| `group_cnt` | [`ops/group_cnt.js`](./ops/group_cnt.js) | Count within groups |
+| `group_max` | [`ops/group_max.js`](./ops/group_max.js) | Max within groups |
+| `group_multiple_stats` | [`ops/group_multiple_stats.js`](./ops/group_multiple_stats.js) | Multiple aggregations per group |
+| `arr_concat` | [`ops/arr_concat.js`](./ops/arr_concat.js) | Concatenate arrays across entries |
+| `obj_concat` | [`ops/obj_concat.js`](./ops/obj_concat.js) | Merge objects across entries |
+| `nested_obj_concat` | [`ops/nested_obj_concat.js`](./ops/nested_obj_concat.js) | Deep-merge nested objects |
+| `array_obj_calc` | [`ops/array_obj_calc.js`](./ops/array_obj_calc.js) | Compute over arrays of objects |
+| `alerts_aggr` | [`ops/alerts_aggr.js`](./ops/alerts_aggr.js) | Aggregate alert states |
+| `alerts_group_cnt` | [`ops/alerts_group_cnt.js`](./ops/alerts_group_cnt.js) | Count alerts by group |
+
+Each operation module exports `{ calc, tally }`. `calc` is called per entry; `tally` is called once to finalize.

package/lib/stats/index.js

@@ -0,0 +1,57 @@
+'use strict'
+
+const { getVal, groupBy } = require('./utils')
+const { getOp } = require('./ops')
+
+function applyStats (state, pack, entry, ext) {
+  const opKeys = Object.keys(state.ops)
+
+  opKeys.forEach(ok => {
+    const op = state.ops[ok]
+
+    const met = getOp(op.op)
+
+    if (!met.calc) {
+      return
+    }
+
+    pack[ok] = met.calc(op, pack[ok], entry, ext)
+  })
+}
+
+function tallyStats (state, pack) {
+  const opKeys = Object.keys(state.ops)
+
+  opKeys.forEach(ok => {
+    const op = state.ops[ok]
+
+    const met = getOp(op.op)
+
+    if (!pack[ok]) {
+      return
+    }
+
+    if (met.tally) {
+      pack[ok] = met.tally(op, pack[ok])
+    }
+
+    pack[ok] = pack[ok].res
+  })
+}
+
+const DEFAULT_TIMEFRAMES = [
+  ['5m', '0 */5 * * * *'],
+  ['30m', '0 */30 * * * *'],
+  ['3h', '0 0 */3 * * *'],
+  ['1D', '0 0 0 * * *']
+]
+
+module.exports = {
+  defaults: {
+    timeframes: DEFAULT_TIMEFRAMES
+  },
+  applyStats,
+  tallyStats,
+  getVal,
+  groupBy
+}

package/lib/stats/ops/alerts_aggr.js

@@ -0,0 +1,40 @@
+'use strict'
+
+const lUtils = require('./../utils')
+
+function alertsAggr (op, cur, entry, ext) {
+  if (!cur) {
+    cur = { res: lUtils.createResMap() }
+  }
+
+  if (!entry) {
+    return cur
+  }
+
+  if (op.filter && !op.filter(entry)) {
+    return cur
+  }
+
+  const groups = lUtils.getVal(entry, op.src)
+
+  if (!groups) {
+    return cur
+  }
+
+  Object.keys(groups).forEach(gk => {
+    if (!lUtils.isSafeKey(gk)) return
+    const gv = groups[gk]
+
+    if (!cur.res[gk]) {
+      cur.res[gk] = 0
+    }
+
+    cur.res[gk] += gv
+  })
+
+  return cur
+}
+
+module.exports = {
+  calc: alertsAggr
+}

package/lib/stats/ops/alerts_group_cnt.js

@@ -0,0 +1,43 @@
+'use strict'
+
+const lUtils = require('./../utils')
+
+function alertsGroupCnt (op, cur, entry, ext) {
+  if (!cur) {
+    cur = { res: lUtils.createResMap() }
+  }
+
+  if (!entry) {
+    return cur
+  }
+
+  if (op.filter && !op.filter(entry)) {
+    return cur
+  }
+
+  const alerts = lUtils.getVal(entry, op.src)
+
+  if (!alerts) {
+    return cur
+  }
+
+  alerts.forEach(al => {
+    if (!al.severity) {
+      al.severity = 'medium'
+    }
+
+    if (!lUtils.isSafeKey(al.severity)) return
+
+    if (!cur.res[al.severity]) {
+      cur.res[al.severity] = 0
+    }
+
+    cur.res[al.severity]++
+  })
+
+  return cur
+}
+
+module.exports = {
+  calc: alertsGroupCnt
+}

package/lib/stats/ops/arr_concat.js

@@ -0,0 +1,27 @@
+'use strict'
+
+const lUtils = require('./../utils')
+
+function arrConcat (op, cur, entry, ext) {
+  if (!cur) {
+    cur = { res: [] }
+  }
+
+  if (!entry) {
+    return cur
+  }
+
+  if (op.filter && !op.filter(entry)) {
+    return cur
+  }
+
+  const val = lUtils.getVal(entry, op.src)
+
+  if (val !== null && val !== undefined) cur.res = cur.res.concat(val)
+
+  return cur
+}
+
+module.exports = {
+  calc: arrConcat
+}

package/lib/stats/ops/array_obj_calc.js

@@ -0,0 +1,41 @@
+'use strict'
+
+const lUtils = require('../utils')
+const { getOp } = require('./index')
+
+function arrayObjCalc (op, cur, entry, ext) {
+  if (!cur) {
+    cur = { res: lUtils.createResMap() }
+  }
+
+  if (!entry) {
+    return cur
+  }
+
+  if (op.filter && !op.filter(entry)) {
+    return cur
+  }
+
+  const arr = lUtils.getVal(entry, op.src)
+  arr?.forEach(obj => {
+    op.objOps.forEach(keyOp => {
+      if (!lUtils.isSafeKey(keyOp.src)) return
+      const val = lUtils.getVal(obj, keyOp.src)
+      if (!val) return
+      if (!cur.res[keyOp.src]) {
+        cur.res[keyOp.src] = val
+        return
+      }
+
+      const calcFn = getOp(keyOp.op)
+      const calcRes = calcFn.calc(keyOp, { res: cur.res[keyOp.src] }, obj)
+      cur.res[keyOp.src] = calcRes.res
+    })
+  })
+
+  return cur
+}
+
+module.exports = {
+  calc: arrayObjCalc
+}

package/lib/stats/ops/avg.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const lUtils = require('./../utils')
+
+function avg (op, cur, entry, ext) {
+  if (!cur) {
+    cur = { acc: 0, cnt: 0, res: 0 }
+  }
+
+  if (!entry) {
+    return cur
+  }
+
+  if (op.filter && !op.filter(entry)) {
+    return cur
+  }
+
+  const val = lUtils.getVal(entry, op.src)
+
+  cur.acc += val ? +val : 0
+
+  cur.cnt++
+
+  return cur
+}
+
+function tally (op, cur) {
+  if (cur.cnt) {
+    cur = {
+      res: cur.acc / cur.cnt
+    }
+  } else {
+    cur = { res: 0 }
+  }
+
+  return cur
+}
+
+module.exports = {
+  calc: avg,
+  tally
+}

package/lib/stats/ops/cnt.js

@@ -0,0 +1,19 @@
+'use strict'
+
+function cnt (op, cur, entry, ext) {
+  if (!cur) {
+    cur = { res: 0 }
+  }
+
+  if (op.filter && !op.filter(entry)) {
+    return cur
+  }
+
+  cur.res++
+
+  return cur
+}
+
+module.exports = {
+  calc: cnt
+}

package/lib/stats/ops/group.js

@@ -0,0 +1,38 @@
+'use strict'
+
+const lUtils = require('./../utils')
+
+function group (op, cur, entry, ext) {
+  if (!cur) {
+    cur = { res: lUtils.createResMap() }
+  }
+
+  if (!entry) {
+    return cur
+  }
+
+  if (!op.src) {
+    return cur
+  }
+
+  if (op.filter && !op.filter(entry)) {
+    return cur
+  }
+
+  if (!op.group) {
+    return cur
+  }
+  const groupName = op.group(entry, ext)
+
+  if (!groupName || !lUtils.isSafeKey(groupName)) {
+    return cur
+  }
+
+  cur.res[groupName] = lUtils.getVal(entry, op.src)
+
+  return cur
+}
+
+module.exports = {
+  calc: group
+}

package/lib/stats/ops/group_avg.js

@@ -0,0 +1,53 @@
+'use strict'
+
+const lUtils = require('../utils')
+
+function groupAvg (op, cur, entry, ext) {
+  if (!cur) {
+    cur = { res: lUtils.createResMap() }
+  }
+
+  if (!entry) {
+    return cur
+  }
+
+  if (op.filter && !op.filter(entry)) {
+    return cur
+  }
+
+  if (!op.group) {
+    return cur
+  }
+  const groupName = op.group(entry, ext)
+
+  if (!groupName || !lUtils.isSafeKey(groupName)) {
+    return cur
+  }
+
+  if (!cur.res[groupName]) {
+    cur.res[groupName] = { acc: 0, cnt: 0, res: 0 }
+  }
+
+  const val = lUtils.getVal(entry, op.src)
+
+  cur.res[groupName].acc += val ? +val : 0
+  cur.res[groupName].cnt++
+
+  return cur
+}
+
+function tally (op, cur) {
+  for (const key in cur.res) {
+    if (cur.res[key].cnt) {
+      cur.res[key] = cur.res[key].acc / cur.res[key].cnt
+    } else {
+      cur.res[key] = null
+    }
+  }
+  return cur
+}
+
+module.exports = {
+  calc: groupAvg,
+  tally
+}

package/lib/stats/ops/group_cnt.js

@@ -0,0 +1,38 @@
+'use strict'
+
+const lUtils = require('../utils')
+
+function groupCnt (op, cur, entry, ext) {
+  if (!cur) {
+    cur = { res: lUtils.createResMap() }
+  }
+
+  if (!entry) {
+    return cur
+  }
+
+  if (op.filter && !op.filter(entry)) {
+    return cur
+  }
+
+  if (!op.group) {
+    return cur
+  }
+  const groupName = op.group(entry, ext)
+
+  if (!groupName || !lUtils.isSafeKey(groupName)) {
+    return cur
+  }
+
+  if (!cur.res[groupName]) {
+    cur.res[groupName] = 0
+  }
+
+  cur.res[groupName]++
+
+  return cur
+}
+
+module.exports = {
+  calc: groupCnt
+}