@tetherto/mdk-core 0.0.0 → 0.12.1

package/lib/services/comments.service.js

@@ -0,0 +1,118 @@
+'use strict'
+
+/**
+ * Comments live on the persisted device record (`comments` array). The
+ * storage collaborators are injected — wire them to a
+ * DeviceProvisioningService (loadThing/saveThing/checkThingExists/generateId).
+ *
+ * Author identity: each method takes `(req, ctx)`. `ctx.user` is the
+ * authenticated identity supplied by the transport (gateway session /
+ * envelope requester) and always wins; a `req.user` that disagrees with it is
+ * rejected. Without `ctx.user` the service falls back to `req.user` — unless
+ * `requireAuthenticatedUser: true`, in which case the call is refused. Either
+ * way a user is mandatory, and only the comment's author may edit or delete
+ * it (comments stored without an author are not editable by anyone).
+ */
+class CommentsService {
+  constructor ({ loadThing, saveThing, checkThingExists, generateId, requireAuthenticatedUser = false }) {
+    this.loadThing = loadThing
+    this.saveThing = saveThing
+    this.checkThingExists = checkThingExists
+    this.generateId = generateId
+    this.requireAuthenticatedUser = requireAuthenticatedUser
+  }
+
+  _resolveUser (req, ctx = {}) {
+    const authUser = ctx && ctx.user
+    if (authUser !== undefined && authUser !== null) {
+      if (typeof authUser !== 'string' || !authUser) throw new Error('ERR_COMMENT_USER_INVALID')
+      if (req.user !== undefined && req.user !== null && req.user !== authUser) {
+        throw new Error('ERR_COMMENT_USER_MISMATCH')
+      }
+      return authUser
+    }
+    if (this.requireAuthenticatedUser) throw new Error('ERR_COMMENT_USER_UNAUTHENTICATED')
+    if (typeof req.user !== 'string' || !req.user) throw new Error('ERR_COMMENT_USER_REQUIRED')
+    return req.user
+  }
+
+  _findCommentIndex (thg, req) {
+    if (req.id) {
+      return thg.comments.findIndex(c => c.id === req.id)
+    }
+    if (req.ts) {
+      return thg.comments.findIndex(c => c.ts === req.ts)
+    }
+    return -1
+  }
+
+  _checkCommentPermission (thg, commentIndex, user) {
+    const owner = thg.comments[commentIndex].user
+    if (typeof owner !== 'string' || !owner || owner !== user) {
+      throw new Error('ERR_COMMENT_ACCESS_DENIED')
+    }
+  }
+
+  async saveThingComment (req, ctx) {
+    const user = this._resolveUser(req, ctx)
+    this.checkThingExists(req)
+
+    const thg = await this.loadThing(req)
+
+    if (!Array.isArray(thg.comments)) thg.comments = []
+
+    thg.comments.push({
+      id: this.generateId(),
+      ts: Date.now(),
+      comment: req.comment,
+      user,
+      ...(req.pos ? { pos: req.pos } : {})
+    })
+
+    await this.saveThing(thg)
+
+    return 1
+  }
+
+  async editThingComment (req, ctx) {
+    const user = this._resolveUser(req, ctx)
+    this.checkThingExists(req)
+
+    const thg = await this.loadThing(req)
+
+    if (!Array.isArray(thg.comments)) throw new Error('ERR_THING_COMMENTS_NOTFOUND')
+
+    const commentIndex = this._findCommentIndex(thg, req)
+    if (commentIndex === -1) throw new Error('ERR_THING_COMMENT_NOTFOUND')
+
+    this._checkCommentPermission(thg, commentIndex, user)
+
+    thg.comments[commentIndex].comment = req.comment
+
+    await this.saveThing(thg)
+
+    return 1
+  }
+
+  async deleteThingComment (req, ctx) {
+    const user = this._resolveUser(req, ctx)
+    this.checkThingExists(req)
+
+    const thg = await this.loadThing(req)
+
+    if (!Array.isArray(thg.comments)) throw new Error('ERR_THING_COMMENTS_NOTFOUND')
+
+    const commentIndex = this._findCommentIndex(thg, req)
+    if (commentIndex === -1) throw new Error('ERR_THING_COMMENT_NOTFOUND')
+
+    this._checkCommentPermission(thg, commentIndex, user)
+
+    thg.comments.splice(commentIndex, 1)
+
+    await this.saveThing(thg)
+
+    return 1
+  }
+}
+
+module.exports = CommentsService

package/lib/services/log-history.service.js

@@ -0,0 +1,307 @@
+'use strict'
+
+const utilsStore = require('@tetherto/hp-svc-facs-store/utils')
+const gLibUtilBase = require('@bitfinex/lib-js-util-base')
+const { getLogsCountForTimeRange, getLogMaxHeight, aggregateLogs, applyFilters, projection } = require('../utils')
+
+// Hard ceiling on rows a single read may return; conf.logMaxLimit lowers or
+// raises it per worker.
+const DEFAULT_MAX_LOG_LIMIT = 10000
+const LOG_KEY_RE = /^[A-Za-z0-9_-]{1,64}$/
+const LOG_TAG_RE = /^[A-Za-z0-9_.:-]{1,160}$/
+// Per-device snapshot log written by SnapsService (`thing-5m-<deviceId>`).
+const DEVICE_LOG_KEY = 'thing-5m'
+
+/**
+ * Reads time logs written by LogsService: tailLog(key, tag) walks rotated
+ * bees until the requested range/limit is filled; getHistoricalLogs serves
+ * the `alerts` / `info` history logs, resolving device metadata through the
+ * injected getDeviceInfo(deviceId).
+ *
+ * Every caller-supplied paging value is validated: `offset` is an integer in
+ * 0..logMaxHeight, `limit` an integer capped at conf.logMaxLimit (default
+ * 10000), `start`/`end` finite timestamps. tailLog only opens the per-device
+ * snapshot log (`thing-5m`, tag = the device id) and `stat-*` aggregate
+ * logs, plus any key listed in conf.tailLogKeys; when the request is scoped
+ * to a device (thingId) the tag must belong to that device.
+ */
+class LogHistoryService {
+  constructor ({ logs, getDeviceInfo, statTimeframes, conf, tailLogHook = async () => {} }) {
+    if (!logs) throw new Error('ERR_LOG_HISTORY_LOGS_REQUIRED')
+    this.logs = logs
+    this.getDeviceInfo = getDeviceInfo || (() => undefined)
+    this.statTimeframes = statTimeframes || []
+    this.conf = conf || {}
+    this.tailLogHook = tailLogHook
+  }
+
+  _maxLimit () {
+    const n = this.conf.logMaxLimit
+    return Number.isInteger(n) && n > 0 ? n : DEFAULT_MAX_LOG_LIMIT
+  }
+
+  // Normalises the paging fields of a caller request; throws ERR_LOG_* on
+  // anything outside the accepted shape.
+  _sanitizeRange (req) {
+    const out = { ...req }
+    const maxHeight = getLogMaxHeight(this.conf.logKeepCount)
+    if (req.offset !== undefined && req.offset !== null) {
+      const offset = Number(req.offset)
+      if (!Number.isInteger(offset) || offset < 0 || offset > maxHeight) throw new Error('ERR_LOG_OFFSET_INVALID')
+      out.offset = offset
+    }
+    if (req.limit !== undefined && req.limit !== null) {
+      const limit = Number(req.limit)
+      if (!Number.isInteger(limit) || limit < 0) throw new Error('ERR_LOG_LIMIT_INVALID')
+      out.limit = Math.min(limit, this._maxLimit())
+    }
+    for (const k of ['start', 'end']) {
+      if (req[k] === undefined || req[k] === null) continue
+      const v = Number(req[k])
+      if (!Number.isFinite(v) || v < 0) throw new Error(`ERR_LOG_${k.toUpperCase()}_INVALID`)
+      out[k] = v
+    }
+    return out
+  }
+
+  async _getLogs (req, logKey, errorMsg, transformFn) {
+    const offset = req.offset || 0
+    const log = await this.logs.getBeeTimeLog(logKey, offset)
+
+    if (!log) {
+      throw new Error(errorMsg)
+    }
+
+    const res = await this._parseHistLog(log, {
+      reverse: true,
+      limit: req.limit,
+      start: req.start,
+      end: req.end
+    })
+
+    if (!Array.isArray(res)) {
+      throw new Error('ERR_HIST_LOG_NOTFOUND')
+    }
+
+    await this.logs.releaseBeeTimeLog(log)
+    return transformFn(res, req)
+  }
+
+  _pickDevice (deviceId) {
+    const dev = this.getDeviceInfo(deviceId)
+    return gLibUtilBase.pick(dev, ['id', 'info', 'tags', 'type', 'code'])
+  }
+
+  _transformAlerts (res, req) {
+    const flattenedAlerts = res.flatMap(log => Object.values(log)).slice(0, req.limit)
+    return flattenedAlerts.map(alert => {
+      const { thingId, ...rest } = alert
+      return { ...rest, thing: this._pickDevice(thingId) }
+    })
+  }
+
+  _transformInfoHistory (res, req) {
+    const flattenedHistory = res.flat(1).slice(0, req.limit)
+    const historyWithCurrentInfo = flattenedHistory.map(change => {
+      const { id, ...rest } = change
+      return { ...rest, thing: this._pickDevice(id) }
+    })
+
+    return gLibUtilBase.isEmpty(req.fields) ? historyWithCurrentInfo : projection(historyWithCurrentInfo, req.fields)
+  }
+
+  async getHistoricalLogs (req) {
+    req = this._sanitizeRange(req)
+    const logType = req.logType
+    if (!logType) {
+      throw new Error('ERR_INFO_HISTORY_LOG_TYPE_INVALID')
+    }
+    if (logType === 'alerts') {
+      return this._getLogs(
+        req,
+        'thing-alerts',
+        'ERR_LOG_NOTFOUND',
+        (res, r) => this._transformAlerts(res, r)
+      )
+    }
+    if (logType === 'info') {
+      const logs = await this._getLogs(
+        req,
+        'thing-history-log',
+        'ERR_INFO_HISTORY_LOG_NOTFOUND',
+        (res, r) => this._transformInfoHistory(res, r)
+      )
+      return applyFilters(
+        logs,
+        { ...req, offset: req.offset ?? 0, limit: req.limit ?? 100 },
+        true
+      )
+    }
+  }
+
+  async _getLogResponse (req, offset) {
+    const log = await this.logs.getBeeTimeLog(`${req.key}-${req.tag}`, offset)
+
+    if (!log) {
+      throw new Error('ERR_LOG_NOTFOUND')
+    }
+
+    const res = await this._parseHistLog(log, {
+      reverse: true,
+      limit: req.limit,
+      start: req.start,
+      end: req.end
+    })
+    await this.logs.releaseBeeTimeLog(log)
+    return res
+  }
+
+  _getDefaultTaillogLimit (req) {
+    const { limit, start, end } = req
+    // A pure time-range read still stops at the hard ceiling.
+    return limit || (start || end ? this._maxLimit() : 100)
+  }
+
+  // Which logs a tailLog request may open. Keys and tags are interpolated
+  // into a bee name, so they are charset-restricted first; then the key must
+  // be the per-device snapshot log or a stat log, and a device-scoped request
+  // (thingId) may only read that device's own logs.
+  _checkTailLogScope (req) {
+    const { key, tag, thingId } = req
+    if (typeof key !== 'string' || !LOG_KEY_RE.test(key)) throw new Error('ERR_LOG_KEY_INVALID')
+    if (typeof tag !== 'string' || !LOG_TAG_RE.test(tag)) throw new Error('ERR_LOG_TAG_INVALID')
+
+    const extraKeys = Array.isArray(this.conf.tailLogKeys) ? this.conf.tailLogKeys : []
+    const isDeviceLog = key === DEVICE_LOG_KEY
+    if (!isDeviceLog && !key.startsWith('stat-') && !extraKeys.includes(key)) {
+      throw new Error('ERR_LOG_KEY_INVALID')
+    }
+
+    if (isDeviceLog) {
+      // tag === thingId here when the request is device-scoped (tailLog
+      // derives it); otherwise it must still name a provisioned device.
+      if (!this.getDeviceInfo(tag)) throw new Error('ERR_LOG_TAG_FORBIDDEN')
+      return
+    }
+
+    if (thingId) {
+      const dev = this.getDeviceInfo(thingId)
+      const tags = (dev && Array.isArray(dev.tags)) ? dev.tags : []
+      if (!tags.includes(tag)) throw new Error('ERR_LOG_TAG_FORBIDDEN')
+    }
+  }
+
+  async _getTailLogWithOffset (req, offset) {
+    const { start, end, key } = req
+    const limit = this._getDefaultTaillogLimit(req)
+    const numberOfLogsInRange = getLogsCountForTimeRange(
+      start,
+      end,
+      key,
+      this.statTimeframes
+    )
+    const logMaxHeight = getLogMaxHeight(this.conf.logKeepCount)
+
+    if (!numberOfLogsInRange && !limit) return await this._getLogResponse(req, offset)
+
+    const remaining =
+      numberOfLogsInRange && limit
+        ? Math.min(numberOfLogsInRange, limit)
+        : numberOfLogsInRange || limit
+
+    let allLogs = []
+    if (offset >= logMaxHeight) return await this._getLogResponse(req, offset)
+    while (allLogs.length < remaining && offset <= logMaxHeight) {
+      try {
+        const logs = await this._getLogResponse(req, offset++)
+        if (!Array.isArray(logs)) break
+
+        allLogs = allLogs.concat(logs).slice(0, remaining)
+      } catch (error) {
+        if (allLogs.length === 0) throw error
+        break
+      }
+    }
+
+    return allLogs
+  }
+
+  async tailLog (req) {
+    if (!req.key) {
+      throw new Error('ERR_LOG_KEY_NOTFOUND')
+    }
+
+    // The per-device snapshot log is addressed by the device, never by a
+    // caller-chosen tag.
+    if (req.key === DEVICE_LOG_KEY && req.thingId) {
+      req = { ...req, tag: req.thingId }
+    }
+
+    if (!req.tag) {
+      throw new Error('ERR_LOG_TAG_INVALID')
+    }
+
+    this._checkTailLogScope(req)
+    req = this._sanitizeRange(req)
+    const offset = req.offset || 0
+
+    const res = await this._getTailLogWithOffset(req, offset)
+
+    await this.tailLogHook(res, req)
+
+    if (req.groupRange) {
+      const aggregated = aggregateLogs(res, req.groupRange, req.shouldCalculateAvg)
+
+      if (!gLibUtilBase.isEmpty(req.fields)) {
+        return projection(aggregated, req.fields)
+      }
+      return aggregated
+    }
+
+    if (!gLibUtilBase.isEmpty(req.fields)) {
+      return projection(res, req.fields)
+    }
+
+    return res
+  }
+
+  async _parseHistLog (log, req) {
+    const query = {}
+
+    if (req.start) {
+      query.gte = utilsStore.convIntToBin(req.start)
+    }
+
+    if (req.end) {
+      query.lte = utilsStore.convIntToBin(req.end)
+    }
+
+    if (req.limit) {
+      query.limit = req.limit
+    }
+
+    if (req.reverse) {
+      query.reverse = true
+    }
+
+    if (req.startExcl) {
+      query.gt = utilsStore.convIntToBin(req.startExcl)
+    }
+
+    if (req.endExcl) {
+      query.lt = utilsStore.convIntToBin(req.endExcl)
+    }
+
+    const stream = log.createReadStream(query)
+
+    const res = []
+
+    for await (const chunk of stream) {
+      res.push(JSON.parse(chunk.value.toString()))
+    }
+
+    return res
+  }
+}
+
+module.exports = LogHistoryService

package/lib/services/logs.service.js

@@ -0,0 +1,249 @@
+'use strict'
+
+const debug = require('debug')('mdk:logs')
+const utilsStore = require('@tetherto/hp-svc-facs-store/utils')
+const { getLogMaxHeight } = require('../utils')
+
+function getLogName (n) {
+  return n + '-5'
+}
+
+/**
+ * Bee-backed time logs. One meta entry per log key tracks the current
+ * rotation point (`cur`); the physical bee for (key, point) is
+ * `<key>-5-<point>`. All values are JSON, keys are binary-encoded ints (ts).
+ */
+class LogsService {
+  constructor ({ store, metaLogs, conf }) {
+    if (!store) throw new Error('ERR_LOGS_STORE_REQUIRED')
+    if (!metaLogs) throw new Error('ERR_LOGS_META_LOGS_REQUIRED')
+    this.store = store
+    this.metaLogs = metaLogs
+    this.conf = conf || {}
+    this._logCache = {}
+  }
+
+  async initBeeLogMeta (logKey) {
+    const meta = { cur: 0 }
+
+    await this.metaLogs.put(
+      logKey,
+      Buffer.from(JSON.stringify(meta))
+    )
+
+    return meta
+  }
+
+  async rotateBeeLog (logKey) {
+    const meta = await this.getBeeLogMeta(logKey)
+
+    if (!meta) {
+      throw new Error('ERR_BEE_LOG_META_NOTFOUND')
+    }
+
+    meta.cur++
+
+    await this.metaLogs.put(
+      logKey,
+      Buffer.from(JSON.stringify(meta))
+    )
+
+    return meta
+  }
+
+  async getBeeLogMeta (logKey, init = false) {
+    let meta = await this.metaLogs.get(logKey)
+
+    if (meta) {
+      meta = JSON.parse(meta.value.toString())
+    } else if (init) {
+      meta = await this.initBeeLogMeta(logKey)
+    }
+
+    return meta
+  }
+
+  async releaseBeeTimeLog (log) {
+    try {
+      await log.close()
+    } catch (e) {
+      debug('ERR_RELEASE_BEE %s', e.message)
+    }
+  }
+
+  // `offset` counts rotations back from the current bee. It reaches the
+  // store as part of a bee name, and store.getBee() creates whatever it is
+  // asked for — so a negative, fractional or out-of-window offset would mint
+  // a fresh hypercore per call. Only integers in 0..logMaxHeight are valid.
+  _checkOffset (offset) {
+    if (!Number.isInteger(offset) || offset < 0) throw new Error('ERR_LOG_OFFSET_INVALID')
+    return offset <= getLogMaxHeight(this.conf.logKeepCount)
+  }
+
+  async getBeeTimeLog (logKey, offset = 0, init = false) {
+    if (!this._checkOffset(offset)) return null
+
+    const meta = await this.getBeeLogMeta(logKey, init)
+
+    if (!meta) {
+      return null
+    }
+
+    const point = meta.cur - offset
+
+    if (point < 0) {
+      return null
+    }
+
+    let log = null
+
+    try {
+      log = await this.getBeeForLogKey(logKey, point)
+
+      if (offset === 0 && log && (await this.isLogCorrupted(log))) {
+        const updatedMeta = await this.rotateBeeLog(logKey)
+        if (!updatedMeta) {
+          throw new Error('ERR_BEE_ROTATED_LOG_META_NOTFOUND')
+        }
+        log = await this.getBeeForLogKey(logKey, updatedMeta.cur)
+      }
+
+      if (!log) {
+        return null
+      }
+    } catch (error) {
+      debug('ERR_GET_BEE_TIME_LOG %s', error.message)
+      return null
+    }
+
+    try {
+      await log.ready()
+    } catch (e) {
+      debug('ERR_BEE_READY %s', e.message)
+      log = null
+    }
+
+    return log
+  }
+
+  async getBeeForLogKey (logKey, point) {
+    return await this.store.getBee(
+      {
+        name: `${getLogName(logKey)}-${point}`
+      },
+      { keyEncoding: 'binary' }
+    )
+  }
+
+  async isLogCorrupted (log) {
+    let validated = true
+    try {
+      for await (const entry of log.createReadStream({ limit: 1 })) {
+        validated = !!entry
+      }
+    } catch (e) {
+      validated = false
+      debug('LOG_CORRUPTED %s', e.message)
+    }
+
+    return !validated
+  }
+
+  async rotateLogs () {
+    if (!this.conf.logRotateMaxLength) {
+      return []
+    }
+
+    const stream = this.metaLogs.createReadStream({})
+    const res = []
+
+    for await (const chunk of stream) {
+      const meta = JSON.parse(chunk.value.toString())
+      const log = await this.getBeeTimeLog(chunk.key, 0)
+
+      if (log) {
+        if (log.core.length >= this.conf.logRotateMaxLength) {
+          await this.rotateBeeLog(chunk.key)
+          res.push([chunk.key, meta, log.core.length])
+          debug('ROTATE: log-key=%s,cur=%d,len=%d', chunk.key, meta.cur, log.core.length)
+        }
+
+        await this.releaseBeeTimeLog(log)
+      }
+    }
+
+    return res
+  }
+
+  async refreshLogsCache () {
+    if (!this.conf.logKeepCount) {
+      return []
+    }
+
+    const stream = this.metaLogs.createReadStream({})
+    const mem = this._logCache
+
+    for (const ck of Object.keys(mem)) {
+      mem[ck].offset = -1
+    }
+
+    for await (const chunk of stream) {
+      const meta = JSON.parse(chunk.value.toString())
+      const maxHeight = getLogMaxHeight(this.conf.logKeepCount)
+
+      for (let i = 0; i < maxHeight; i++) {
+        const log = await this.getBeeTimeLog(chunk.key, i)
+
+        if (!log) {
+          continue
+        }
+
+        const dkey = log.core.key.toString('hex')
+
+        if (mem[dkey]) {
+          mem[dkey].offset = i
+          await this.releaseBeeTimeLog(log)
+          continue
+        }
+
+        debug('REFRESH: log-key=%s,offset=%d,cur=%d', chunk.key, i, meta.cur)
+        mem[dkey] = { log, offset: i, desc: chunk.key }
+      }
+    }
+
+    await this._cleanupLogs(mem)
+  }
+
+  async _cleanupLogs (mem) {
+    for (const ck of Object.keys(mem)) {
+      const entry = mem[ck]
+
+      if (entry.offset < 0) {
+        await this.releaseBeeTimeLog(entry.log)
+        debug('REFRESH[RELEASE]: log-key=%s,cache-key=%s', entry.desc, ck)
+        delete mem[ck]
+
+        try {
+          await this.store.unlink(ck)
+          break
+        } catch (e) {
+          debug('LOG_NOT_FOUND=%s %s', ck, e.message)
+        }
+      }
+    }
+  }
+
+  async saveLogData (key, ts, data, offset = 0, init = false) {
+    const log = await this.getBeeTimeLog(key, offset, init)
+    if (!log) return
+
+    await log.put(utilsStore.convIntToBin(ts), Buffer.from(JSON.stringify(data)))
+    await this.releaseBeeTimeLog(log)
+  }
+
+  getLogName (n) {
+    return getLogName(n)
+  }
+}
+
+module.exports = LogsService

package/lib/services/pool-utils/constants.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const SCHEDULER_TIMES = {
+  _1M: { time: '0 */1 * * * *', key: '1m' },
+  _5M: { time: '0 */5 * * * *', key: '5m' },
+  _1D: { time: '0 0 0 * * *', key: '1D' }
+}
+
+const BTC_SATS = 100000000 // 1 BTC = 100,000,000 satoshis
+const MINUTE_MS = 60 * 1000
+const HOUR_MS = 60 * 60 * 1000
+const HOURS_24_MS = 24 * 60 * 60 * 1000
+
+module.exports = {
+  SCHEDULER_TIMES,
+  BTC_SATS,
+  MINUTE_MS,
+  HOUR_MS,
+  HOURS_24_MS
+}