@mojaloop/central-ledger 19.11.4-cache.0 → 19.11.4-cache.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mojaloop/central-ledger",
-  "version": "19.11.4-cache.0",
+  "version": "19.11.4-cache.2",
   "description": "Central ledger hosted by a scheme to record and settle transfers",
   "license": "Apache-2.0",
   "author": "ModusBox",
@@ -44,7 +44,7 @@
     "lint": "npx standard",
     "lint:fix": "npx standard --fix",
     "test": "npm run test:unit:spec",
-    "test:unit": "npx tape 'test/unit/**/*.test.js'",
+    "test:unit": "tape 'test/unit/**/*.test.js'",
     "test:unit:spec": "npm run test:unit | tap-spec",
     "test:unit:distLock": "npx tape 'test/unit/lib/distLock/**/*.test.js'",
     "test:unit:participants-routes": "npx tape test/unit/api/participants/routes.test.js",
@@ -85,6 +85,7 @@
   "dependencies": {
     "@hapi/basic": "7.0.2",
     "@hapi/catbox-memory": "6.0.2",
+    "@hapi/catbox": "12.1.1",
     "@hapi/good": "9.0.1",
     "@hapi/hapi": "21.4.4",
     "@hapi/inert": "7.1.0",
@@ -138,7 +139,7 @@
     "jsonpath": "1.1.1",
     "mock-knex": "0.4.13",
     "nodemon": "3.1.11",
-    "npm-check-updates": "19.1.2",
+    "npm-check-updates": "19.2.0",
     "nyc": "17.1.0",
     "pre-commit": "1.2.2",
     "proxyquire": "2.1.3",

package/src/lib/cache.js

@@ -1,41 +1,30 @@
 'use strict'
 
 const CatboxMemory = require('@hapi/catbox-memory')
+const Catbox = require('@hapi/catbox')
 const Config = require('../lib/config')
 
-let enabled = true
-let ttl
-let catboxMemoryClient = null
+const expiresIn = parseInt(Config.CACHE_CONFIG.EXPIRES_IN_MS)
+// Init memory client
+const catboxMemoryClient = new CatboxMemory.Engine({ maxByteSize: Config.CACHE_CONFIG.MAX_BYTE_SIZE })
+catboxMemoryClient.start()
 
 class CacheClient {
-  constructor (meta) {
-    this.meta = meta
-  }
-
-  getMeta () {
-    return this.meta
-  }
-
-  createKey (id) {
-    return {
-      segment: this.meta.id,
-      id
-    }
+  constructor (segment, generateFunc) {
+    this.generateFunc = generateFunc
+    if (Config.CACHE_CONFIG.CACHE_ENABLED) this.policy = new Catbox.Policy({ generateFunc, expiresIn, generateTimeout: false }, catboxMemoryClient, segment)
   }
 
   get (key) {
-    if (enabled) {
-      return catboxMemoryClient.get(key)
-    }
-    return null
+    return Config.CACHE_CONFIG.CACHE_ENABLED ? this.policy.get(key) : this.generateFunc(key)
   }
 
-  set (key, value) {
-    return catboxMemoryClient.set(key, value, ttl)
+  drop (key) {
+    return this.policy?.drop(key)
   }
 
-  drop (key) {
-    catboxMemoryClient.drop(key)
+  setGenerateFunc (generateFunc) {
+    this.policy?.options({ generateFunc, expiresIn, generateTimeout: false })
   }
 }
 
@@ -50,32 +39,22 @@ class CacheClient {
 */
 let cacheClients = {}
 
-const registerCacheClient = (clientMeta) => {
-  const newClient = new CacheClient(clientMeta)
-  cacheClients[clientMeta.id] = newClient
+const registerCacheClient = (id, generateFunc) => {
+  const newClient = new CacheClient(id, generateFunc)
+  cacheClients[id] = newClient
   return newClient
 }
 
 const initCache = async function () {
   // Read config
-  ttl = parseInt(Config.CACHE_CONFIG.EXPIRES_IN_MS)
-  enabled = Config.CACHE_CONFIG.CACHE_ENABLED
-
-  // Init catbox.
-  catboxMemoryClient = new CatboxMemory.Engine({
-    maxByteSize: Config.CACHE_CONFIG.MAX_BYTE_SIZE
-  })
-  catboxMemoryClient.start()
-
-  for (const clientId in cacheClients) {
-    const clientMeta = cacheClients[clientId].getMeta()
-    await clientMeta.preloadCache()
+  for (const client of Object.values(cacheClients)) {
+    await client.get('all') // preload cache
   }
 }
 
 const destroyCache = async function () {
-  catboxMemoryClient?.stop()
-  catboxMemoryClient = null
+  catboxMemoryClient.stop()
+  catboxMemoryClient.start()
 }
 
 const dropClients = function () {
@@ -83,7 +62,7 @@ const dropClients = function () {
 }
 
 const isCacheEnabled = function () {
-  return enabled
+  return Config.CACHE_CONFIG.CACHE_ENABLED
 }
 
 module.exports = {
@@ -96,6 +75,6 @@ module.exports = {
   isCacheEnabled,
 
   // exposed for tests
-  CatboxMemory,
+  Catbox,
   dropClients
 }

package/src/lib/enumCached.js

@@ -31,23 +31,17 @@ const Cache = require('../lib/cache')
 const Enums = require('./enum')
 
 let cacheClient
-let enumAllCacheKey
+const enumAllCacheKey = 'all'
 
 /*
   Private API
 */
-const _getAllEnums = async function () {
-  let allEnums
-  const allEnumsFromCache = await cacheClient.get(enumAllCacheKey)
-  if (allEnumsFromCache === null) {
-    allEnums = {}
-    for (const enumId of Enums.enumsIds) {
-      allEnums[enumId] = await Enums[enumId]()
-    }
-    await cacheClient.set(enumAllCacheKey, allEnums)
-  } else {
-    // unwrap from catbox structure
-    allEnums = allEnumsFromCache.item
+const _getAllEnums = () => cacheClient.get(enumAllCacheKey)
+
+const generateFunc = async function (key) {
+  const allEnums = {}
+  for (const enumId of Enums.enumsIds) {
+    allEnums[enumId] = await Enums[enumId]()
   }
   return allEnums
 }
@@ -65,13 +59,7 @@ exports.getEnums = async (id) => {
 
 exports.initialize = async () => {
   /* Register as cache client */
-  const enumCacheClientMeta = {
-    id: 'enum',
-    preloadCache: _getAllEnums
-  }
-
-  cacheClient = Cache.registerCacheClient(enumCacheClientMeta)
-  enumAllCacheKey = cacheClient.createKey('all')
+  cacheClient = Cache.registerCacheClient('enum', generateFunc)
 }
 
 exports.invalidateEnumCache = async () => {

package/src/models/participant/externalParticipantCached.js

@@ -26,13 +26,12 @@
  --------------
  **********/
 
-const Metrics = require('@mojaloop/central-services-metrics')
 const cache = require('../../lib/cache')
 const externalParticipantModel = require('./externalParticipant')
 const rethrow = require('../../shared/rethrow')
 
 let cacheClient
-let epAllCacheKey
+const epAllCacheKey = 'all'
 
 const buildUnifiedCachedData = (allExternalParticipants) => {
   // build indexes  - optimization for byId and byName access
@@ -52,28 +51,11 @@ const buildUnifiedCachedData = (allExternalParticipants) => {
   }
 }
 
-const getExternalParticipantsCached = async () => {
-  const queryName = 'model_getExternalParticipantsCached'
-  const histTimer = Metrics.getHistogram(
-    'model_externalParticipant',
-    `${queryName} - Metrics for externalParticipant model`,
-    ['success', 'queryName', 'hit']
-  ).startTimer()
-
-  let cachedParticipants = await cacheClient.get(epAllCacheKey)
-  const hit = !!cachedParticipants
-
-  if (!cachedParticipants) {
-    const allParticipants = await externalParticipantModel.getAll()
-    cachedParticipants = buildUnifiedCachedData(allParticipants)
-    await cacheClient.set(epAllCacheKey, cachedParticipants)
-  } else {
-    // unwrap participants list from catbox structure
-    cachedParticipants = cachedParticipants.item
-  }
-  histTimer({ success: true, queryName, hit })
+const getExternalParticipantsCached = () => cacheClient.get(epAllCacheKey)
 
-  return cachedParticipants
+const generateFunc = async function (key) {
+  const allParticipants = await externalParticipantModel.getAll()
+  return buildUnifiedCachedData(allParticipants)
 }
 
 /*
@@ -81,13 +63,7 @@ const getExternalParticipantsCached = async () => {
 */
 const initialize = () => {
   /* Register as cache client */
-  const cacheClientMeta = {
-    id: 'externalParticipants',
-    preloadCache: getExternalParticipantsCached
-  }
-
-  cacheClient = cache.registerCacheClient(cacheClientMeta)
-  epAllCacheKey = cacheClient.createKey('all')
+  cacheClient = cache.registerCacheClient('externalParticipants', generateFunc)
 }
 
 const invalidateCache = async () => {

package/src/models/participant/participantCached.js

@@ -29,11 +29,10 @@
 
 const Cache = require('../../lib/cache')
 const ParticipantModel = require('../../models/participant/participant')
-const Metrics = require('@mojaloop/central-services-metrics')
 const rethrow = require('../../shared/rethrow')
 
 let cacheClient
-let participantsAllCacheKey
+const participantsAllCacheKey = 'all'
 
 /*
   Private API
@@ -65,42 +64,18 @@ const buildUnifiedParticipantsData = (allParticipants) => {
   return unifiedParticipants
 }
 
-const getParticipantsCached = async () => {
-  const histTimer = Metrics.getHistogram(
-    'model_participant',
-    'model_getParticipantsCached - Metrics for participant model',
-    ['success', 'queryName', 'hit']
-  ).startTimer()
-  // Do we have valid participants list in the cache ?
-  let cachedParticipants = await cacheClient.get(participantsAllCacheKey)
-  if (!cachedParticipants) {
-    // No participants in the cache, so fetch from participant API
-    const allParticipants = await ParticipantModel.getAll()
-    cachedParticipants = buildUnifiedParticipantsData(allParticipants)
-
-    // store in cache
-    await cacheClient.set(participantsAllCacheKey, cachedParticipants)
-    histTimer({ success: true, queryName: 'model_getParticipantsCached', hit: false })
-  } else {
-    // unwrap participants list from catbox structure
-    cachedParticipants = cachedParticipants.item
-    histTimer({ success: true, queryName: 'model_getParticipantsCached', hit: true })
-  }
-  return cachedParticipants
+const getParticipantsCached = () => cacheClient.get(participantsAllCacheKey)
+
+const generateFunc = async function (key) {
+  const allParticipants = await ParticipantModel.getAll()
+  return buildUnifiedParticipantsData(allParticipants)
 }
 
 /*
   Public API
 */
 exports.initialize = async () => {
-  /* Register as cache client */
-  const participantCacheClientMeta = {
-    id: 'participants',
-    preloadCache: getParticipantsCached
-  }
-
-  cacheClient = Cache.registerCacheClient(participantCacheClientMeta)
-  participantsAllCacheKey = cacheClient.createKey('all')
+  cacheClient = Cache.registerCacheClient('participants', generateFunc)
 }
 
 exports.invalidateParticipantsCache = async () => {
@@ -150,3 +125,5 @@ exports.create = withInvalidate('create')
 exports.update = withInvalidate('update')
 exports.destroyByName = withInvalidate('destroyByName')
 exports.destroyParticipantEndpointByParticipantId = withInvalidate('destroyParticipantEndpointByParticipantId')
+
+exports.build = buildUnifiedParticipantsData

package/src/models/participant/participantCurrencyCached.js

@@ -30,11 +30,10 @@
 const Cache = require('../../lib/cache')
 const Config = require('../../../src/lib/config')
 const ParticipantCurrencyModel = require('../../models/participant/participantCurrency')
-const Metrics = require('@mojaloop/central-services-metrics')
 const rethrow = require('../../shared/rethrow')
 
 let cacheClient
-let participantCurrencyAllCacheKey
+const participantCurrencyAllCacheKey = 'all'
 
 /*
   Private API
@@ -62,28 +61,11 @@ const buildUnifiedParticipantsCurrencyData = (allCurrencyParticipants) => {
   return unifiedCurrencyParticipants
 }
 
-const getParticipantCurrencyCached = async () => {
-  const histTimer = Metrics.getHistogram(
-    'model_participant',
-    'model_getParticipantCurrencyCached - Metrics for participant model',
-    ['success', 'queryName', 'hit']
-  ).startTimer()
-  // Do we have valid participantsCurrency list in the cache ?
-  let cachedCurrencyParticipants = await cacheClient.get(participantCurrencyAllCacheKey)
-  if (!cachedCurrencyParticipants) {
-    // No participantsCurrency in the cache, so fetch from participantsCurrency API
-    const allCurrencyParticipants = await ParticipantCurrencyModel.getAll()
-    cachedCurrencyParticipants = buildUnifiedParticipantsCurrencyData(allCurrencyParticipants)
-
-    // store in cache
-    await cacheClient.set(participantCurrencyAllCacheKey, cachedCurrencyParticipants)
-    histTimer({ success: true, queryName: 'model_getParticipantCurrencyCached', hit: false })
-  } else {
-    // unwrap participants list from catbox structure
-    cachedCurrencyParticipants = cachedCurrencyParticipants.item
-    histTimer({ success: true, queryName: 'model_getParticipantCurrencyCached', hit: true })
-  }
-  return cachedCurrencyParticipants
+const getParticipantCurrencyCached = () => cacheClient.get(participantCurrencyAllCacheKey)
+
+const generateFunc = async function (key) {
+  const allCurrencyParticipants = await ParticipantCurrencyModel.getAll()
+  return buildUnifiedParticipantsCurrencyData(allCurrencyParticipants)
 }
 
 /*
@@ -91,13 +73,7 @@ const getParticipantCurrencyCached = async () => {
 */
 exports.initialize = async () => {
   /* Register as cache client */
-  const participantCurrencyCacheClientMeta = {
-    id: 'participantCurrency',
-    preloadCache: getParticipantCurrencyCached
-  }
-
-  cacheClient = Cache.registerCacheClient(participantCurrencyCacheClientMeta)
-  participantCurrencyAllCacheKey = cacheClient.createKey('all')
+  cacheClient = Cache.registerCacheClient('participantCurrency', generateFunc)
 }
 
 exports.invalidateParticipantCurrencyCache = async () => {
@@ -170,3 +146,4 @@ const withInvalidate = (theFunctionName) => {
 exports.create = withInvalidate('create')
 exports.update = withInvalidate('update')
 exports.destroyByParticipantId = withInvalidate('destroyByParticipantId')
+exports.build = buildUnifiedParticipantsCurrencyData

package/src/models/participant/participantLimitCached.js

@@ -29,11 +29,10 @@
 
 const Cache = require('../../lib/cache')
 const ParticipantLimitModel = require('../../models/participant/participantLimit')
-const Metrics = require('@mojaloop/central-services-metrics')
 const rethrow = require('../../shared/rethrow')
 
 let cacheClient
-let participantLimitAllCacheKey
+const participantLimitAllCacheKey = 'all'
 
 /*
   Private API
@@ -61,28 +60,11 @@ const buildUnifiedParticipantsLimitData = (allLimitParticipants) => {
   return unifiedLimitParticipants
 }
 
-const getParticipantLimitCached = async () => {
-  const histTimer = Metrics.getHistogram(
-    'model_participant',
-    'model_getParticipantLimitCached - Metrics for participant model',
-    ['success', 'queryName', 'hit']
-  ).startTimer()
-  // Do we have valid participantsLimit list in the cache ?
-  let cachedLimitParticipants = await cacheClient.get(participantLimitAllCacheKey)
-  if (!cachedLimitParticipants) {
-    // No participantsLimit in the cache, so fetch from participantsLimit API
-    const allLimitParticipants = await ParticipantLimitModel.getAll()
-    cachedLimitParticipants = buildUnifiedParticipantsLimitData(allLimitParticipants)
-
-    // store in cache
-    await cacheClient.set(participantLimitAllCacheKey, cachedLimitParticipants)
-    histTimer({ success: true, queryName: 'model_getParticipantLimitCached', hit: false })
-  } else {
-    // unwrap participants list from catbox structure
-    cachedLimitParticipants = cachedLimitParticipants.item
-    histTimer({ success: true, queryName: 'model_getParticipantLimitCached', hit: true })
-  }
-  return cachedLimitParticipants
+const getParticipantLimitCached = () => cacheClient.get(participantLimitAllCacheKey)
+
+const generateFunc = async function (key) {
+  const allLimitParticipants = await ParticipantLimitModel.getAll()
+  return buildUnifiedParticipantsLimitData(allLimitParticipants)
 }
 
 /*
@@ -90,13 +72,7 @@ const getParticipantLimitCached = async () => {
 */
 exports.initialize = async () => {
   /* Register as cache client */
-  const participantLimitCacheClientMeta = {
-    id: 'participantLimit',
-    preloadCache: getParticipantLimitCached
-  }
-
-  cacheClient = Cache.registerCacheClient(participantLimitCacheClientMeta)
-  participantLimitAllCacheKey = cacheClient.createKey('all')
+  cacheClient = Cache.registerCacheClient('participantLimit', generateFunc)
 }
 
 exports.invalidateParticipantLimitCache = async () => {
@@ -125,3 +101,4 @@ exports.insert = withInvalidate('insert')
 exports.update = withInvalidate('update')
 exports.destroyByParticipantCurrencyId = withInvalidate('destroyByParticipantCurrencyId')
 exports.destroyByParticipantId = withInvalidate('destroyByParticipantId')
+exports.build = buildUnifiedParticipantsLimitData

package/src/models/participant/participantPositionChange.js

@@ -51,7 +51,7 @@ const getByParticipantPositionId = async (participantPositionId) => {
   const histTimer = Metrics.getHistogram(
     'model_participant',
     'model_getByParticipantPositionId - Metrics for participant model',
-    ['success', 'queryName', 'hit']
+    ['success', 'queryName']
   ).startTimer()
   try {
     return await Db.from('participantPositionChange').query(async (builder) => {
@@ -60,11 +60,11 @@ const getByParticipantPositionId = async (participantPositionId) => {
         .select('participantPositionChange.*')
         .orderBy('participantPositionChangeId', 'desc')
         .first()
-      histTimer({ success: true, queryName: 'model_getByParticipantPositionId', hit: false })
+      histTimer({ success: true, queryName: 'model_getByParticipantPositionId' })
       return result
     })
   } catch (err) {
-    histTimer({ success: false, queryName: 'model_getByParticipantPositionId', hit: false })
+    histTimer({ success: false, queryName: 'model_getByParticipantPositionId' })
     rethrow.rethrowDatabaseError(err)
   }
 }

package/src/models/position/batchCached.js

@@ -31,7 +31,6 @@
 'use strict'
 
 const Cache = require('../../lib/cache')
-const Metrics = require('@mojaloop/central-services-metrics')
 const BatchPositionModel = require('./batch')
 const rethrow = require('../../shared/rethrow')
 
@@ -70,27 +69,11 @@ const buildUnifiedParticipantCurrencyData = (allParticipantCurrency) => {
   return unifiedParticipantsCurrency
 }
 
-const getParticipantCurrencyCached = async (trx) => {
-  const histTimer = Metrics.getHistogram(
-    'model_participant_batch',
-    'model_getParticipantsCached - Metrics for participant model',
-    ['success', 'queryName', 'hit']
-  ).startTimer()
-  // Do we have valid participants list in the cache ?
-  let cachedParticipantCurrency = await cacheClient.get(participantCurrencyAllCacheKey)
-  if (!cachedParticipantCurrency) {
-    const allParticipantCurrency = await BatchPositionModel.getAllParticipantCurrency(trx)
-    cachedParticipantCurrency = buildUnifiedParticipantCurrencyData(allParticipantCurrency)
-
-    // store in cache
-    await cacheClient.set(participantCurrencyAllCacheKey, cachedParticipantCurrency)
-    histTimer({ success: true, queryName: 'model_getParticipantCurrencyBatchCached', hit: false })
-  } else {
-    // unwrap participants list from catbox structure
-    cachedParticipantCurrency = cachedParticipantCurrency.item
-    histTimer({ success: true, queryName: 'model_getParticipantCurrencyBatchCached', hit: true })
-  }
-  return cachedParticipantCurrency
+const getParticipantCurrencyCached = (trx) => cacheClient.get({ id: participantCurrencyAllCacheKey, trx })
+
+const generateFunc = async function (key) {
+  const allParticipantCurrency = await BatchPositionModel.getAllParticipantCurrency(key.trx)
+  return buildUnifiedParticipantCurrencyData(allParticipantCurrency)
 }
 
 /*
@@ -98,13 +81,7 @@ const getParticipantCurrencyCached = async (trx) => {
 */
 exports.initialize = async () => {
   /* Register as cache client */
-  const participantCurrencyCacheClientMeta = {
-    id: 'participantCurrency',
-    preloadCache: getParticipantCurrencyCached
-  }
-
-  cacheClient = Cache.registerCacheClient(participantCurrencyCacheClientMeta)
-  participantCurrencyAllCacheKey = cacheClient.createKey('participantCurrency')
+  cacheClient = Cache.registerCacheClient('positionBatch', generateFunc)
 }
 
 exports.getParticipantCurrencyByIds = async (trx, participantCurrencyIds) => {
@@ -132,3 +109,5 @@ exports.getParticipantCurrencyByParticipantIds = async (trx, participantIds) =>
     rethrow.rethrowCachedDatabaseError(err)
   }
 }
+
+exports.build = buildUnifiedParticipantCurrencyData

package/src/models/settlement/settlementModelCached.js

@@ -29,11 +29,10 @@
 
 const Cache = require('../../lib/cache')
 const SettlementModel = require('../../models/settlement/settlementModel')
-const Metrics = require('@mojaloop/central-services-metrics')
 const rethrow = require('../../shared/rethrow')
 
 let cacheClient
-let settlementModelsAllCacheKey
+const settlementModelsAllCacheKey = 'all'
 
 /*
   Private API
@@ -63,28 +62,11 @@ const buildUnifiedSettlementModelsData = (allSettlementModels) => {
   return unifiedSettlementModels
 }
 
-const getSettlementModelsCached = async () => {
-  const histTimer = Metrics.getHistogram(
-    'model_settlementModel',
-    'model_getSettlementModelsCached - Metrics for settlementModel model',
-    ['success', 'queryName', 'hit']
-  ).startTimer()
-  // Do we have valid settlement models list in the cache ?
-  let cachedSettlementModels = await cacheClient.get(settlementModelsAllCacheKey)
-  if (!cachedSettlementModels) {
-    // No settlement models in the cache, so fetch from participant API
-    const allSettlementModels = await SettlementModel.getAll()
-    cachedSettlementModels = buildUnifiedSettlementModelsData(allSettlementModels)
-
-    // store in cache
-    await cacheClient.set(settlementModelsAllCacheKey, cachedSettlementModels)
-    histTimer({ success: true, queryName: 'model_getSettlementModelsCached', hit: false })
-  } else {
-    // unwrap settlement models list from catbox structure
-    cachedSettlementModels = cachedSettlementModels.item
-    histTimer({ success: true, queryName: 'model_getSettlementModelsCached', hit: true })
-  }
-  return cachedSettlementModels
+const getSettlementModelsCached = () => cacheClient.get(settlementModelsAllCacheKey)
+
+const generateFunc = async function (key) {
+  const allSettlementModels = await SettlementModel.getAll()
+  return buildUnifiedSettlementModelsData(allSettlementModels)
 }
 
 /*
@@ -92,13 +74,7 @@ const getSettlementModelsCached = async () => {
 */
 exports.initialize = async () => {
   /* Register as cache client */
-  const settlementModelCacheClientMeta = {
-    id: 'settlementModels',
-    preloadCache: getSettlementModelsCached
-  }
-
-  cacheClient = Cache.registerCacheClient(settlementModelCacheClientMeta)
-  settlementModelsAllCacheKey = cacheClient.createKey('all')
+  cacheClient = Cache.registerCacheClient('settlementModels', generateFunc)
 }
 
 exports.invalidateSettlementModelsCache = async () => {
@@ -140,3 +116,5 @@ exports.getAll = async () => {
     rethrow.rethrowCachedDatabaseError(err)
   }
 }
+
+exports.build = buildUnifiedSettlementModelsData