@chainlink/external-adapter-framework 0.0.10 → 0.0.12

package/src/cache/metrics.ts

@@ -0,0 +1,112 @@
+import * as client from 'prom-client'
+
+interface CacheMetricsLabels {
+  participant_id: string
+  feed_id: string
+  cache_type: string
+  // Is_from_ws?: string
+}
+
+export const cacheGet = (label: CacheMetricsLabels, value: unknown, staleness: number) => {
+  if (typeof value === 'number' || typeof value === 'string') {
+    const parsedValue = Number(value)
+    if (!Number.isNaN(parsedValue) && Number.isFinite(parsedValue)) {
+      cacheDataGetValues.labels(label).set(parsedValue)
+    }
+  }
+  cacheDataGetCount.labels(label).inc()
+  cacheDataStalenessSeconds.labels(label).set(staleness)
+}
+
+export const cacheSet = (label: CacheMetricsLabels, maxAge: number) => {
+  cacheDataSetCount.labels(label).inc()
+  cacheDataMaxAge.labels(label).set(maxAge)
+  cacheDataStalenessSeconds.labels(label).set(0)
+}
+
+export const cacheMetricsLabel = (cacheKey: string, feedId: string, cacheType: string) => ({
+  participant_id: cacheKey,
+  feed_id: feedId,
+  cache_type: cacheType,
+})
+
+export enum CacheTypes {
+  Redis = 'redis',
+  Local = 'local',
+}
+
+export enum CMD_SENT_STATUS {
+  TIMEOUT = 'TIMEOUT',
+  FAIL = 'FAIL',
+  SUCCESS = 'SUCCESS',
+}
+
+const baseLabels = [
+  'feed_id',
+  'participant_id',
+  'cache_type',
+  'is_from_ws',
+  'experimental',
+] as const
+
+// Skipping this metrics for v3
+// const cache_execution_duration_seconds = new client.Histogram({
+//   name: 'cache_execution_duration_seconds',
+//   help: 'A histogram bucket of the distribution of cache execution durations',
+//   labelNames: [...baseLabels, 'cache_hit'] as const,
+//   buckets: [0.01, 0.1, 1, 10],
+// })
+
+const cacheDataGetCount = new client.Counter({
+  name: 'cache_data_get_count',
+  help: 'A counter that increments every time a value is fetched from the cache',
+  labelNames: baseLabels,
+})
+
+const cacheDataGetValues = new client.Gauge({
+  name: 'cache_data_get_values',
+  help: 'A gauge keeping track of values being fetched from cache',
+  labelNames: baseLabels,
+})
+
+const cacheDataMaxAge = new client.Gauge({
+  name: 'cache_data_max_age',
+  help: 'A gauge tracking the max age of stored values in the cache',
+  labelNames: baseLabels,
+})
+
+const cacheDataSetCount = new client.Counter({
+  name: 'cache_data_set_count',
+  help: 'A counter that increments every time a value is set to the cache',
+  labelNames: [...baseLabels, 'status_code'],
+})
+
+const cacheDataStalenessSeconds = new client.Gauge({
+  name: 'cache_data_staleness_seconds',
+  help: 'Observes the staleness of the data returned',
+  labelNames: baseLabels,
+})
+
+// Redis Metrics
+export const redisConnectionsOpen = new client.Counter({
+  name: 'redis_connections_open',
+  help: 'The number of redis connections that are open',
+})
+
+export const redisRetriesCount = new client.Counter({
+  name: 'redis_retries_count',
+  help: 'The number of retries that have been made to establish a redis connection',
+})
+
+export const redisCommandsSentCount = new client.Counter({
+  name: 'redis_commands_sent_count',
+  help: 'The number of redis commands sent',
+  labelNames: ['status', 'function_name'],
+})
+
+// Cache Warmer Metrics
+export const cacheWarmerCount = new client.Gauge({
+  name: 'cache_warmer_get_count',
+  help: 'The number of cache warmers running',
+  labelNames: ['isBatched'] as const,
+})

package/src/cache/redis.ts

@@ -0,0 +1,93 @@
+import Redis from 'ioredis'
+import { AdapterResponse, makeLogger } from '../util'
+import { Cache, CacheEntry } from './index'
+import * as cacheMetrics from './metrics'
+
+const logger = makeLogger('RedisCache')
+
+/**
+ * Redis implementation of a Cache. It uses a simple js Object, storing entries with both
+ * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
+ *
+ * @typeParam T - the type for the entries' values
+ */
+export class RedisCache<T = unknown> implements Cache<T> {
+  constructor(private client: Redis) {}
+
+  async get(key: string): Promise<T | undefined> {
+    logger.trace(`Getting key ${key}`)
+    const value = await this.client.get(key)
+
+    // Record get command sent to Redis
+    cacheMetrics.redisCommandsSentCount
+      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'get' })
+      .inc()
+
+    if (!value) {
+      logger.debug(`No entry in redis cache for key "${key}", returning undefined`)
+      return undefined
+    }
+
+    return JSON.parse(value) as T
+  }
+
+  async delete(key: string): Promise<void> {
+    logger.trace(`Deleting key ${key}`)
+    await this.client.del(key)
+
+    // Record delete command sent to Redis
+    cacheMetrics.redisCommandsSentCount
+      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'delete' })
+      .inc()
+  }
+
+  async set(key: string, value: T, ttl: number): Promise<void> {
+    logger.trace(`Setting key ${key}`)
+    await this.client.set(key, JSON.stringify(value), 'PX', ttl)
+
+    // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
+    const feedId = (value as unknown as AdapterResponse).meta?.metrics?.feedId
+    if (feedId) {
+      // Record cache set count, max age, and staleness (set to 0 for cache set)
+      const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Redis)
+      cacheMetrics.cacheSet(label, ttl)
+    }
+
+    // Record set command sent to Redis
+    cacheMetrics.redisCommandsSentCount
+      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'set' })
+      .inc()
+  }
+
+  async setMany(entries: CacheEntry<T>[], ttl: number): Promise<void> {
+    logger.trace(`Setting a bunch of keys`)
+    // Unfortunately, there's no ttl for mset
+    let chain = this.client.multi()
+
+    for (const entry of entries) {
+      chain = chain.set(entry.key, JSON.stringify(entry.value), 'PX', ttl)
+    }
+
+    await chain.exec()
+
+    // Loop again, but this time to record these in metrics
+    for (const entry of entries) {
+      // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
+      const feedId = (entry.value as unknown as AdapterResponse).meta?.metrics?.feedId
+      if (feedId) {
+        // Record cache set count, max age, and staleness (set to 0 for cache set)
+        const label = cacheMetrics.cacheMetricsLabel(
+          entry.key,
+          feedId,
+          cacheMetrics.CacheTypes.Redis,
+        )
+        cacheMetrics.cacheSet(label, ttl)
+      }
+    }
+
+    // Record setMany command sent to Redis
+    cacheMetrics.redisCommandsSentCount
+      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'exec' })
+      .inc()
+  }
+}