@wardx/core 0.2.2 → 0.4.0

package/README.md

@@ -61,7 +61,7 @@ The defaults live in `defaults.json`. Do not omit a required key. The loader doe
 
 **When:** You write a runtime that is not Node.js, or you test the engine without HTTP.
 
-**Objective:** Record counters, gauges, histograms, and timers. Then make a frame.
+**Objective:** Record counters, gauges, histograms, distinct estimates, and timers. Then make a frame.
 
 ```js
 import { WardxCore, loadSdkDefaults } from '@wardx/core';
@@ -83,6 +83,7 @@ core.counter('match.completed', { mode: 'ranked' }).inc();
 core.counter('coins.awarded').add(25);
 core.gauge('players.online').set(12921);
 core.histogram('request.duration').observe(42);
+core.distinct('shot.traffic.hids', { result: 'violating' }).add(hid);
 
 const endTimer = core.timer('matchmaking.duration');
 endTimer({ result: 'success' });
@@ -96,7 +97,7 @@ const frames = core.takePendingFrames();
 1. Load the SDK defaults.
 2. Add the identity fields.
 3. Construct `WardxCore`.
-4. Call `counter`, `gauge`, `histogram`, or `timer`.
+4. Call `counter`, `gauge`, `histogram`, `distinct`, or `timer`.
 5. Call `snapshotFrame` when you need a frame.
 6. Call `takePendingFrames` to get the pending frames.
 
@@ -115,6 +116,12 @@ core.histogram('coins.award_size').observe(80, { grantId: 'g-80' });
 
 `timer(name, dims)` starts a timer. The returned function records the duration in milliseconds into a histogram. You can add dimensions when you stop the timer.
 
+`distinct(name, dims).add(identifier)` updates a fixed HyperLogLog sketch. The
+engine hashes the identifier as `SHA-256(privacySalt || 0x00 || identifier)` and
+discards it immediately; the frame contains only 512 HLL registers (`p=9`,
+about 4.6% standard error). Keep `privacySalt` stable across workers and time
+windows so equal identifiers map to equal registers.
+
 If a series is above `maxSeriesPerMetric`, or a dimension is not valid, the engine returns a no-op object. The engine increments `wardx.internal.cardinality_dropped`.
 
 A dimension value must be a string, a number, or a boolean.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wardx/core",
-  "version": "0.2.2",
+  "version": "0.4.0",
   "description": "Runtime-agnostic Wardx engine for metrics, events, logs, Remote Config, and experiments.",
   "keywords": [
     "wardx",

package/src/WardxCore.js

@@ -8,9 +8,10 @@ import { InternalMetrics } from './internal/InternalMetrics.js';
 import { NOOP_COUNTER } from './metrics/Counter.js';
 import { NOOP_GAUGE } from './metrics/Gauge.js';
 import { NOOP_HISTOGRAM } from './metrics/Histogram.js';
+import { NOOP_DISTINCT } from './metrics/HyperLogLog.js';
 import { startTimer } from './metrics/Timer.js';
 import { emit } from './trace/emit.js';
-import { wrapCounter, wrapGauge, wrapHistogram } from './trace/wrap.js';
+import { wrapCounter, wrapDistinct, wrapGauge, wrapHistogram } from './trace/wrap.js';
 
 export class WardxCore {
   constructor(settings) {
@@ -24,6 +25,7 @@ export class WardxCore {
       maxDimensionKeys: settings.maxDimensionKeys,
       maxDimensionValueLength: settings.maxDimensionValueLength,
       defaultHistogramBuckets: settings.histogramBuckets,
+      privacySalt: settings.privacySalt,
       onCardinalityDropped: () => {
         this.internal.cardinalityDropped += 1;
       }
@@ -68,6 +70,12 @@ export class WardxCore {
     );
   }
 
+  distinct(name, dims) {
+    return this._wrap(this.metrics.distinct(name, dims), NOOP_DISTINCT, (series, noop) =>
+      wrapDistinct(this._tracer, series, noop, name, dims)
+    );
+  }
+
   timer(name, dims) {
     if (this._tracer === null) return this.metrics.timer(name, dims);
     return startTimer((duration, endDims) => {
@@ -202,6 +210,7 @@ export class WardxCore {
         counters: physical.metrics.counters.length,
         gauges: physical.metrics.gauges.length,
         histograms: physical.metrics.histograms.length,
+        distincts: physical.metrics.distincts?.length || 0,
         events: physical.events.length,
         logs: physical.logs.length,
         droppedLogs: i === 0 ? batch.droppedLogs : 0,

package/src/frame/FrameBuilder.js

@@ -21,6 +21,7 @@ function rowCount(frame) {
     frame.metrics.counters.length +
     frame.metrics.gauges.length +
     frame.metrics.histograms.length +
+    (frame.metrics.distincts?.length || 0) +
     frame.events.length +
     frame.logs.length
   );
@@ -39,7 +40,8 @@ export class FrameBuilder {
       metrics: {
         counters,
         gauges,
-        histograms
+        histograms,
+        ...(metrics.distincts?.length > 0 ? { distincts: metrics.distincts.slice() } : {})
       },
       events,
       logs
@@ -81,6 +83,7 @@ export class FrameBuilder {
       counters: 0,
       gauges: 0,
       histograms: 0,
+      distincts: 0,
       events: 0,
       logs: 0
     };
@@ -100,11 +103,17 @@ export class FrameBuilder {
       collection(current).push(row);
       if (measure(current).bytes <= maxFrameBytes) return true;
       collection(current).pop();
+      if (collection.kind === 'distincts' && current.metrics.distincts.length === 0) {
+        delete current.metrics.distincts;
+      }
       if (rowCount(current) > 0) {
         finishCurrent();
         collection(current).push(row);
         if (measure(current).bytes <= maxFrameBytes) return true;
         collection(current).pop();
+        if (collection.kind === 'distincts' && current.metrics.distincts.length === 0) {
+          delete current.metrics.distincts;
+        }
       }
       if (countDrop) dropped[collection.kind] += 1;
       return false;
@@ -116,6 +125,11 @@ export class FrameBuilder {
     gauges.kind = 'gauges';
     const histograms = (candidate) => candidate.metrics.histograms;
     histograms.kind = 'histograms';
+    const distincts = (candidate) => {
+      if (!candidate.metrics.distincts) candidate.metrics.distincts = [];
+      return candidate.metrics.distincts;
+    };
+    distincts.kind = 'distincts';
     const events = (candidate) => candidate.events;
     events.kind = 'events';
     const logs = (candidate) => candidate.logs;
@@ -124,6 +138,7 @@ export class FrameBuilder {
     for (const row of frame.metrics.counters) addRow(counters, row);
     for (const row of frame.metrics.gauges) addRow(gauges, row);
     for (const row of frame.metrics.histograms) addRow(histograms, row);
+    for (const row of frame.metrics.distincts || []) addRow(distincts, row);
     for (const row of frame.events) addRow(events, row);
     for (const row of frame.logs) addRow(logs, row);
 
@@ -143,6 +158,7 @@ export class FrameBuilder {
       droppedCounters: dropped.counters,
       droppedGauges: dropped.gauges,
       droppedHistograms: dropped.histograms,
+      droppedDistincts: dropped.distincts,
       droppedEvents: dropped.events,
       droppedLogs: dropped.logs
     };

package/src/index.d.ts

@@ -44,6 +44,10 @@ export interface HistogramHandle {
   observe(value: number, attrs?: Dimensions | null): void;
 }
 
+export interface DistinctHandle {
+  add(identifier: string): void;
+}
+
 export type StopTimer = (dims?: Dimensions | null) => void;
 
 export interface SdkDefaults {
@@ -140,6 +144,12 @@ export type LogRow = [timestamp: number, level: LogLevel, message: string, attrs
 export type CounterRow = [name: string, dims: Dimensions | null, value: number];
 export type GaugeRow = [name: string, dims: Dimensions | null, value: number, timestamp: number];
 export type HistogramRow = [name: string, dims: Dimensions | null, snapshot: HistogramSnapshot];
+export type DistinctRow = [name: string, dims: Dimensions | null, sketch: HllSketch];
+
+export interface HllSketch {
+  precision: 9;
+  registers: string;
+}
 
 export interface HistogramExemplar {
   value: number;
@@ -159,6 +169,7 @@ export interface MetricsSnapshot {
   counters: CounterRow[];
   gauges: GaugeRow[];
   histograms: HistogramRow[];
+  distincts?: DistinctRow[];
 }
 
 export interface Frame {
@@ -177,6 +188,7 @@ export interface FrameBatch {
   droppedCounters: number;
   droppedGauges: number;
   droppedHistograms: number;
+  droppedDistincts: number;
   droppedLogs: number;
   droppedEvents: number;
 }
@@ -197,7 +209,7 @@ export interface InternalSnapshot {
 }
 
 export interface MeasureTraceRecord {
-  type: 'counter' | 'gauge' | 'histogram';
+  type: 'counter' | 'gauge' | 'histogram' | 'distinct';
   name: string;
   dims: Dimensions | null;
   op: 'inc' | 'add' | 'set' | 'observe';
@@ -226,6 +238,7 @@ export interface FrameTraceRecord {
   counters: number;
   gauges: number;
   histograms: number;
+  distincts: number;
   events: number;
   logs: number;
   droppedLogs: number;
@@ -307,11 +320,31 @@ export class Histogram implements HistogramHandle {
   reset(): void;
 }
 
+export class HyperLogLog implements DistinctHandle {
+  name: string;
+  dims: Dimensions | null;
+  privacySalt: string;
+  registers: Uint8Array;
+  dirty: boolean;
+  constructor(name: string, dims: Dimensions | null, privacySalt: string);
+  add(identifier: string): void;
+  snapshot(): HllSketch;
+  reset(): void;
+}
+
+export const HLL_PRECISION: 9;
+export function encodeHllRegisters(registers: Uint8Array): string;
+export function decodeHllRegisters(sketch: HllSketch): Uint8Array;
+export function estimateHllRegisters(registers: Uint8Array): number;
+export function estimateHyperLogLog(sketch: HllSketch): number;
+export function mergeHyperLogLog(left: HllSketch, right: HllSketch): HllSketch;
+
 export interface MetricsRegistryOptions {
   maxSeriesPerMetric: number;
   maxDimensionKeys: number;
   maxDimensionValueLength: number;
   defaultHistogramBuckets: number[];
+  privacySalt?: string;
   onCardinalityDropped: () => void;
 }
 
@@ -320,6 +353,7 @@ export class MetricsRegistry {
   counter(name: string, dims?: Dimensions | null): CounterHandle;
   gauge(name: string, dims?: Dimensions | null): GaugeHandle;
   histogram(name: string, a?: HistogramOptions | null, b?: HistogramOptions | null): HistogramHandle;
+  distinct(name: string, dims?: Dimensions | null): DistinctHandle;
   timer(name: string, dims?: Dimensions | null): StopTimer;
   snapshotAndReset(): MetricsSnapshot;
   isDirty(): boolean;
@@ -403,6 +437,7 @@ export class WardxCore {
   counter(name: string, dims?: Dimensions | null): CounterHandle;
   gauge(name: string, dims?: Dimensions | null): GaugeHandle;
   histogram(name: string, a?: HistogramOptions | null, b?: HistogramOptions | null): HistogramHandle;
+  distinct(name: string, dims?: Dimensions | null): DistinctHandle;
   timer(name: string, dims?: Dimensions | null): StopTimer;
   event(name: string, attrs?: Attrs | null): void;
   identify(subjectId: string | null | undefined): void;

package/src/index.js

@@ -5,6 +5,15 @@ export { ConfigStore } from './config/ConfigStore.js';
 export { Counter } from './metrics/Counter.js';
 export { Gauge } from './metrics/Gauge.js';
 export { Histogram } from './metrics/Histogram.js';
+export {
+  HLL_PRECISION,
+  HyperLogLog,
+  decodeHllRegisters,
+  encodeHllRegisters,
+  estimateHllRegisters,
+  estimateHyperLogLog,
+  mergeHyperLogLog
+} from './metrics/HyperLogLog.js';
 export { MetricsRegistry } from './metrics/MetricsRegistry.js';
 export { EventBuffer } from './buffers/EventBuffer.js';
 export { LogBuffer } from './buffers/LogBuffer.js';

package/src/metrics/HyperLogLog.js

@@ -0,0 +1,106 @@
+import { createHash } from 'node:crypto';
+
+export const HLL_PRECISION = 9;
+export const HLL_REGISTER_COUNT = 1 << HLL_PRECISION;
+export const HLL_MAX_RANK = 64 - HLL_PRECISION + 1;
+
+function assertRegisters(registers) {
+  if (!(registers instanceof Uint8Array) || registers.length !== HLL_REGISTER_COUNT) {
+    throw new Error(`HLL registers must contain exactly ${HLL_REGISTER_COUNT} bytes`);
+  }
+  for (const rank of registers) {
+    if (rank > HLL_MAX_RANK) throw new Error(`HLL register rank must be <= ${HLL_MAX_RANK}`);
+  }
+}
+
+function rankAfterIndex(digest) {
+  let rank = 1;
+  for (let bit = HLL_PRECISION; bit < 64; bit++) {
+    if ((digest[bit >> 3] & (1 << (7 - (bit & 7)))) !== 0) return rank;
+    rank += 1;
+  }
+  return rank;
+}
+
+export function encodeHllRegisters(registers) {
+  assertRegisters(registers);
+  return Buffer.from(registers).toString('base64');
+}
+
+export function decodeHllRegisters(body) {
+  if (!body || body.precision !== HLL_PRECISION || typeof body.registers !== 'string') {
+    throw new Error(`HLL sketch must use precision ${HLL_PRECISION}`);
+  }
+  const decoded = Buffer.from(body.registers, 'base64');
+  if (decoded.toString('base64') !== body.registers) throw new Error('HLL registers must be canonical base64');
+  const registers = Uint8Array.from(decoded);
+  assertRegisters(registers);
+  return registers;
+}
+
+export function estimateHllRegisters(registers) {
+  assertRegisters(registers);
+  let harmonic = 0;
+  let zeros = 0;
+  for (const rank of registers) {
+    harmonic += 2 ** -rank;
+    if (rank === 0) zeros += 1;
+  }
+  const m = HLL_REGISTER_COUNT;
+  const alpha = 0.7213 / (1 + 1.079 / m);
+  const raw = alpha * m * m / harmonic;
+  const corrected = raw <= 2.5 * m && zeros > 0 ? m * Math.log(m / zeros) : raw;
+  return Math.round(corrected);
+}
+
+export function estimateHyperLogLog(body) {
+  return estimateHllRegisters(decodeHllRegisters(body));
+}
+
+export function mergeHyperLogLog(left, right) {
+  const leftRegisters = decodeHllRegisters(left);
+  const rightRegisters = decodeHllRegisters(right);
+  for (let index = 0; index < leftRegisters.length; index++) {
+    if (rightRegisters[index] > leftRegisters[index]) leftRegisters[index] = rightRegisters[index];
+  }
+  return { precision: HLL_PRECISION, registers: encodeHllRegisters(leftRegisters) };
+}
+
+export class HyperLogLog {
+  constructor(name, dims, privacySalt) {
+    if (typeof privacySalt !== 'string' || privacySalt.length === 0) {
+      throw new Error('distinct requires a non-empty privacySalt');
+    }
+    this.name = name;
+    this.dims = dims;
+    this.privacySalt = privacySalt;
+    this.registers = new Uint8Array(HLL_REGISTER_COUNT);
+    this.dirty = false;
+  }
+
+  add(identifier) {
+    if (typeof identifier !== 'string' || identifier.length === 0) {
+      throw new Error('distinct.add requires a non-empty string');
+    }
+    const digest = createHash('sha256')
+      .update(this.privacySalt, 'utf8')
+      .update('\0', 'utf8')
+      .update(identifier, 'utf8')
+      .digest();
+    const index = (digest[0] << 1) | (digest[1] >> 7);
+    const rank = rankAfterIndex(digest);
+    if (rank > this.registers[index]) this.registers[index] = rank;
+    this.dirty = true;
+  }
+
+  snapshot() {
+    return { precision: HLL_PRECISION, registers: encodeHllRegisters(this.registers) };
+  }
+
+  reset() {
+    this.registers.fill(0);
+    this.dirty = false;
+  }
+}
+
+export const NOOP_DISTINCT = Object.freeze({ add() {} });

package/src/metrics/MetricsRegistry.js

@@ -1,6 +1,7 @@
 import { Counter, NOOP_COUNTER } from './Counter.js';
 import { Gauge, NOOP_GAUGE } from './Gauge.js';
 import { Histogram, NOOP_HISTOGRAM } from './Histogram.js';
+import { HyperLogLog, NOOP_DISTINCT } from './HyperLogLog.js';
 import { startTimer } from './Timer.js';
 import { assertMetricName, dimKey, validateDimensions } from './dimensions.js';
 
@@ -48,16 +49,24 @@ export class MetricsRegistry {
     this.maxDimensionKeys = options.maxDimensionKeys;
     this.maxDimensionValueLength = options.maxDimensionValueLength;
     this.defaultHistogramBuckets = options.defaultHistogramBuckets;
+    this.privacySalt = options.privacySalt;
     this.onCardinalityDropped = options.onCardinalityDropped;
     this.countersByName = new Map();
     this.gaugesByName = new Map();
     this.histogramsByName = new Map();
+    this.distinctsByName = new Map();
     this.rejected = new Set();
   }
 
   _series(kindMap, Ctor, name, dims, extra) {
     assertMetricName(name);
-    const kindPrefix = kindMap === this.histogramsByName ? 'h' : kindMap === this.gaugesByName ? 'g' : 'c';
+    const kindPrefix = kindMap === this.histogramsByName
+      ? 'h'
+      : kindMap === this.gaugesByName
+        ? 'g'
+        : kindMap === this.distinctsByName
+          ? 'd'
+          : 'c';
     const checked = validateDimensions(dims, this.maxDimensionKeys, this.maxDimensionValueLength);
     const key = dimKey(checked.ok ? checked.dims : dims);
     const rejectKey = kindPrefix + '\0' + name + '\0' + key;
@@ -109,6 +118,13 @@ export class MetricsRegistry {
     return series;
   }
 
+  distinct(name, dims) {
+    const series = this._series(this.distinctsByName, HyperLogLog, name, dims, (resolvedDims) => {
+      return new HyperLogLog(name, resolvedDims, this.privacySalt);
+    });
+    return series || NOOP_DISTINCT;
+  }
+
   timer(name, dims) {
     const histogram = this.histogram(name, dims);
     return startTimer((duration, endDims) => {
@@ -148,7 +164,16 @@ export class MetricsRegistry {
         }
       }
     }
-    return { counters, gauges, histograms };
+    const distincts = [];
+    for (const byKey of this.distinctsByName.values()) {
+      for (const series of byKey.values()) {
+        if (series.dirty) {
+          distincts.push([series.name, series.dims, series.snapshot()]);
+          series.reset();
+        }
+      }
+    }
+    return { counters, gauges, histograms, distincts };
   }
 
   isDirty() {
@@ -167,6 +192,11 @@ export class MetricsRegistry {
         if (series.count > 0) return true;
       }
     }
+    for (const byKey of this.distinctsByName.values()) {
+      for (const series of byKey.values()) {
+        if (series.dirty) return true;
+      }
+    }
     return false;
   }
 }

package/src/trace/wrap.js

@@ -44,3 +44,14 @@ export function wrapHistogram(tracer, series, noop, name, dims) {
     }
   };
 }
+
+export function wrapDistinct(tracer, series, noop, name, dims) {
+  const n = noop ? name : series.name;
+  const d = noop ? (dims ?? null) : series.dims;
+  return {
+    add(identifier) {
+      series.add(identifier);
+      emit(tracer, 'measure', { type: 'distinct', name: n, dims: d, op: 'add', value: 1, noop });
+    }
+  };
+}