@tryghost/metrics 3.4.2 → 3.5.0

package/README.md

@@ -29,6 +29,42 @@ const custom = new metrics.GhostMetrics({
 });
 ```
 
+### Sampling
+
+High-volume metrics can be sampled so only a proportion of them are shipped.
+`sampleRate` is a number between 0 and 1, and defaults to `1` (ship
+everything). Configure a default rate, per-metric rates, or both:
+
+```js
+const metrics = require('@tryghost/metrics');
+
+const sampled = new metrics.GhostMetrics({
+    metrics: {
+        transports: ['elasticsearch'],
+        sampleRate: 0.5,
+        sampleRates: {
+            'request-duration': 0.01,
+        },
+    },
+});
+```
+
+A rate can also be overridden per call, which takes precedence over both:
+
+```js
+await sampled.metric('request-duration', duration, { sampleRate: 0.001 });
+```
+
+Sampled metrics carry the rate they survived, so consumers can scale counts
+back up by dividing by it. The Elasticsearch shipper writes it as a
+`sampleRate` field, and the stdout shipper appends it to the log line. Metrics
+shipped at a rate of `1` are not tagged, so an absent `sampleRate` means the
+metric was not sampled.
+
+An invalid configured rate throws when the instance is constructed. An invalid
+per-call rate is ignored in favour of the configured rate, so a bad call site
+cannot break the code path it is measuring.
+
 ### Types
 
 Types ship with the package. `GhostMetrics` is exported as both a value (the
@@ -40,6 +76,7 @@ import type {
     GhostMetrics as GhostMetricsInstance,
     GhostMetricsOptions,
     MetricsOptions,
+    MetricOptions,
     ElasticsearchOptions,
     MetricShipper,
 } from '@tryghost/metrics';

package/lib/GhostMetrics.d.ts

@@ -20,6 +20,24 @@ interface MetricsOptions {
     transports?: string[];
     /** Property bag of metadata values shipped alongside each metric value. */
     metadata?: Record<string, unknown>;
+    /**
+     * Default proportion of metrics to ship, between 0 and 1. Throws at construction
+     * time if it is not a number in that range. @default 1
+     */
+    sampleRate?: number;
+    /** Per-metric sample rate overrides, keyed by metric name. Same validation as `sampleRate`. */
+    sampleRates?: Record<string, number>;
+}
+
+/**
+ * Per-call options accepted by {@link GhostMetrics.metric}.
+ */
+interface MetricOptions {
+    /**
+     * Proportion of calls to ship, between 0 and 1, taking precedence over any configured
+     * rate. Values outside that range are ignored rather than throwing.
+     */
+    sampleRate?: number;
 }
 
 /**
@@ -37,9 +55,10 @@ interface GhostMetricsOptions {
 }
 
 /**
- * A single metric shipper function keyed by transport name.
+ * A single metric shipper function keyed by transport name. Receives the sample rate the
+ * metric survived, so transports can record it alongside the value.
  */
-type MetricShipper = (name: string, value: unknown) => Promise<unknown>;
+type MetricShipper = (name: string, value: unknown, sampleRate?: number) => Promise<unknown>;
 
 /**
  * Metric shipper class built on the loggingrc config used in Ghost projects.
@@ -50,6 +69,8 @@ declare class GhostMetrics {
     mode: string;
     transports: string[];
     metadata: Record<string, unknown>;
+    sampleRate: number;
+    sampleRates: Record<string, number>;
     shippers: Record<string, MetricShipper>;
 
     constructor(options?: GhostMetricsOptions);
@@ -66,15 +87,28 @@ declare class GhostMetrics {
     setupElasticsearchShipper(): void;
 
     /**
-     * Ship a metric through every configured transport.
+     * Resolve the sample rate for a metric: per-call override, then per-metric config,
+     * then the instance default.
+     */
+    getSampleRate(name: string, options?: MetricOptions): number;
+
+    /**
+     * Ship a metric through every configured transport, unless it is dropped by sampling.
      * @param name Metric name, should be slugified for back-end compatibility (e.g. `"memory-usage"`).
      * @param value Metric value; coerced to an object before being shipped.
+     * @param options Per-call options, e.g. a `sampleRate` override.
      */
-    metric(name: string, value: unknown): Promise<null>;
+    metric(name: string, value: unknown, options?: MetricOptions): Promise<null>;
 }
 
 declare namespace GhostMetrics {
-    export { ElasticsearchOptions, MetricsOptions, GhostMetricsOptions, MetricShipper };
+    export {
+        ElasticsearchOptions,
+        MetricsOptions,
+        MetricOptions,
+        GhostMetricsOptions,
+        MetricShipper,
+    };
 }
 
 export = GhostMetrics;

package/lib/GhostMetrics.js

@@ -1,5 +1,32 @@
 const jsonStringifySafe = require('json-stringify-safe');
 
+/**
+ * @description Check a value is usable as a sample rate: a number between 0 and 1 inclusive
+ * @param {any} rate Candidate sample rate
+ * @returns {boolean}
+ */
+function isValidSampleRate(rate) {
+    return typeof rate === 'number' && Number.isFinite(rate) && rate >= 0 && rate <= 1;
+}
+
+/**
+ * @description Validate a configured sample rate, defaulting to 1 (ship everything) when unset
+ * @param {any} rate Candidate sample rate
+ * @param {string} key Config key used in the error message
+ * @returns {number}
+ */
+function normalizeSampleRate(rate, key) {
+    if (rate === undefined || rate === null) {
+        return 1;
+    }
+
+    if (!isValidSampleRate(rate)) {
+        throw new Error(`${key} must be a number between 0 and 1, got ${jsonStringifySafe(rate)}`);
+    }
+
+    return rate;
+}
+
 /**
  * @description Metric shipper class built on the loggingrc config used in Ghost projects
  */
@@ -9,9 +36,11 @@ class GhostMetrics {
      *
      * domain:             Metadata for metrics in shared databases.
      * mode:               Is used to print short or long log - used for stdout shipper.
-     * metrics.transports: An array of transports for metric shipping (e.g. ['stdout', 'elasticsearch'])
-     * metrics.metadata:   A property bag of metadata values to be shipped alongside the metric value
-     * elasticsearch:      Elasticsearch transport configuration
+     * metrics.transports:  An array of transports for metric shipping (e.g. ['stdout', 'elasticsearch'])
+     * metrics.metadata:    A property bag of metadata values to be shipped alongside the metric value
+     * metrics.sampleRate:  Default proportion of metrics to ship, between 0 and 1 (defaults to 1, ship everything)
+     * metrics.sampleRates: Per-metric sample rate overrides, keyed by metric name
+     * elasticsearch:       Elasticsearch transport configuration
      * @param {object} options Bag of options
      */
     constructor(options) {
@@ -20,12 +49,21 @@ class GhostMetrics {
         this.domain = options.domain || 'localhost';
         this.elasticsearch = options.elasticsearch || {};
         this.mode = process.env.MODE || options.mode || 'short';
-        if ('metrics' in options && typeof options.metrics === 'object') {
+        // Prototype-free so that metric names like "__proto__" are stored and read back
+        // as ordinary keys, and inherited properties are never mistaken for configured rates
+        this.sampleRates = Object.create(null);
+        if (options.metrics !== null && typeof options.metrics === 'object') {
             this.transports = options.metrics.transports || [];
             this.metadata = options.metrics.metadata || {};
+            this.sampleRate = normalizeSampleRate(options.metrics.sampleRate, 'metrics.sampleRate');
+
+            for (const [name, rate] of Object.entries(options.metrics.sampleRates || {})) {
+                this.sampleRates[name] = normalizeSampleRate(rate, `metrics.sampleRates.${name}`);
+            }
         } else {
             this.transports = [];
             this.metadata = {};
+            this.sampleRate = 1;
         }
 
         // CASE: special env variable to enable long mode and level info
@@ -57,9 +95,12 @@ class GhostMetrics {
 
         prettyStdOut.pipe(process.stdout);
 
-        this.shippers.stdout = (name, value) => {
+        this.shippers.stdout = (name, value, sampleRate) => {
+            // Only mention the sample rate when the metric is actually sampled, so unsampled output is unchanged
+            const suffix = sampleRate < 1 ? ` (sample rate: ${sampleRate})` : '';
+
             prettyStdOut.write({
-                msg: `Metric ${name}: ${jsonStringifySafe(value)}`,
+                msg: `Metric ${name}: ${jsonStringifySafe(value)}${suffix}`,
                 level: 30, // Magic number, log level for info
             });
 
@@ -85,7 +126,7 @@ class GhostMetrics {
             proxy: 'proxy' in this.elasticsearch ? this.elasticsearch.proxy : null,
         });
 
-        this.shippers.elasticsearch = (name, value) => {
+        this.shippers.elasticsearch = (name, value, sampleRate) => {
             if (typeof value !== 'object') {
                 value = { value };
             }
@@ -98,19 +139,54 @@ class GhostMetrics {
                 value.metadata = this.metadata;
             }
 
+            // Sampled documents carry their rate so consumers can scale counts back up.
+            // An absent sampleRate means the metric was not sampled (i.e. a rate of 1).
+            if (sampleRate < 1) {
+                value.sampleRate = sampleRate;
+            }
+
             return elasticSearch.index(value, `metrics-${name}`);
         };
     }
 
+    /**
+     * @description Resolve the sample rate for a metric, most specific config first
+     * Invalid per-call overrides are ignored rather than thrown, so a bad call site cannot
+     * take down the code path it is measuring. Invalid config throws at construction time.
+     * @param {string} name Name of the metric
+     * @param {object} [options] Per-call options bag
+     * @returns {number}
+     */
+    getSampleRate(name, options) {
+        if (options && isValidSampleRate(options.sampleRate)) {
+            return options.sampleRate;
+        }
+
+        if (isValidSampleRate(this.sampleRates[name])) {
+            return this.sampleRates[name];
+        }
+
+        return this.sampleRate;
+    }
+
     /**
      * @description Metric shipper function
      * @param {string} name Name of the metric, should be slugified for increased back-end compatibility (e.g. "memory-usage")
      * @param {any} value Value of metric, will be co-erced to an object before being shipped
+     * @param {object} [options] Per-call options bag
+     * @param {number} [options.sampleRate] Proportion of calls to ship, between 0 and 1, overriding any configured rate
      */
-    metric(name, value) {
+    metric(name, value, options) {
+        const sampleRate = this.getSampleRate(name, options);
+
+        // A rate of 1 short-circuits, so unsampled metrics do no extra work
+        if (sampleRate < 1 && Math.random() >= sampleRate) {
+            return Promise.resolve(null);
+        }
+
         const promises = [];
         for (const metricShipper of Object.values(this.shippers)) {
-            promises.push(metricShipper(name, value));
+            promises.push(metricShipper(name, value, sampleRate));
         }
 
         return Promise.allSettled(promises).then(() => null);

package/lib/metrics.d.ts

@@ -16,6 +16,7 @@ declare namespace metrics {
     export type GhostMetrics = GhostMetricsClass;
     export type ElasticsearchOptions = GhostMetricsClass.ElasticsearchOptions;
     export type MetricsOptions = GhostMetricsClass.MetricsOptions;
+    export type MetricOptions = GhostMetricsClass.MetricOptions;
     export type GhostMetricsOptions = GhostMetricsClass.GhostMetricsOptions;
     export type MetricShipper = GhostMetricsClass.MetricShipper;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tryghost/metrics",
-  "version": "3.4.2",
+  "version": "3.5.0",
   "license": "MIT",
   "author": "Ghost Foundation",
   "repository": {
@@ -20,9 +20,9 @@
   },
   "dependencies": {
     "json-stringify-safe": "5.0.1",
-    "@tryghost/elasticsearch": "^5.4.5",
-    "@tryghost/pretty-stream": "^2.3.8",
-    "@tryghost/root-utils": "^2.3.8"
+    "@tryghost/pretty-stream": "^2.3.9",
+    "@tryghost/elasticsearch": "^5.4.6",
+    "@tryghost/root-utils": "^2.3.9"
   },
   "devDependencies": {
     "rewire": "9.0.1",