@wardx/core 0.1.7 → 0.4.0

package/README.md

@@ -37,30 +37,31 @@ import { WardxCore, assignVariant, loadSdkDefaults } from '@wardx/core';
 | Key | Description |
 | --- | --- |
 | `endpoint` | Sync URL. The core does not use this key. Runtimes use this key. |
-| `projectKey` | Project credential. Runtimes send this key. If `privacySalt` is empty, the core uses this key as the salt. |
+| `projectKey` | Project credential. Runtimes send this key. It is never reused as the subject-hash salt. |
 | `project` | Project name. |
 | `role` | Runtime identity inside the project. Runtimes send this name. The core does not use this key. |
 | `appVersion` | Application version. |
 | `environment` | Environment name, for example `production`. |
-| `privacySalt` | Salt for the hashed subject. If you omit this key, the core uses `projectKey`. |
+| `privacySalt` | Required stable, non-empty, project-specific salt for one-way subject hashes. |
 | `aggregateIntervalMs` | Default `1000`. Interval to snapshot dirty data. |
 | `syncIntervalMs` | Default `15000`. Interval for the runtime sync. |
 | `maxBufferedEvents` | Default `5000`. |
 | `maxBufferedLogs` | Default `2000`. |
-| `maxFrameBytes` | Default `524288`. |
+| `maxFrameBytes` | Default `524288`; must be at least `1024`. |
 | `maxSeriesPerMetric` | Default `1000`. |
 | `maxDimensionKeys` | Default `8`. |
 | `maxDimensionValueLength` | Default `64`. |
+| `experimentStateMaxSubjects` | Default `100000`. Maximum subjects retained for assignment/exposure state in this SDK instance. |
 | `histogramBuckets` | Default `[10, 25, 50, 100, 250, 500, 1000]`. |
 | `tracer` | Optional. Duck-typed local hook with any of `measure`, `event`, `log`, `frame`. The core does not print. Runtimes may also call `sync`. |
 
-The defaults live in `defaults.json`. Do not omit a required key. The loader does not add a fallback for a missing key. `tracer` is not a default key. Omit it to keep the measure path unchanged.
+The defaults live in `defaults.json`. Do not omit a required key. The loader does not add a fallback for a missing key. `tracer` is not a default key. Omit it to keep the measure path unchanged. The project key authenticates only the project; `role` is client-selected routing metadata, not authorization. Never put secrets in Remote Config.
 
 ## Use case 1: Record metrics in a custom runtime
 
 **When:** You write a runtime that is not Node.js, or you test the engine without HTTP.
 
-**Objective:** Record counters, gauges, histograms, and timers. Then make a frame.
+**Objective:** Record counters, gauges, histograms, distinct estimates, and timers. Then make a frame.
 
 ```js
 import { WardxCore, loadSdkDefaults } from '@wardx/core';
@@ -73,7 +74,7 @@ const settings = {
   role: 'client',
   appVersion: '0.1.0',
   environment: 'development',
-  privacySalt: 'dev_project_key'
+  privacySalt: 'demo-subject-hash-v1'
 };
 
 const core = new WardxCore(settings);
@@ -82,11 +83,12 @@ core.counter('match.completed', { mode: 'ranked' }).inc();
 core.counter('coins.awarded').add(25);
 core.gauge('players.online').set(12921);
 core.histogram('request.duration').observe(42);
+core.distinct('shot.traffic.hids', { result: 'violating' }).add(hid);
 
 const endTimer = core.timer('matchmaking.duration');
 endTimer({ result: 'success' });
 
-const fitted = core.snapshotFrame();
+const batch = core.snapshotFrame();
 const frames = core.takePendingFrames();
 ```
 
@@ -95,7 +97,7 @@ const frames = core.takePendingFrames();
 1. Load the SDK defaults.
 2. Add the identity fields.
 3. Construct `WardxCore`.
-4. Call `counter`, `gauge`, `histogram`, or `timer`.
+4. Call `counter`, `gauge`, `histogram`, `distinct`, or `timer`.
 5. Call `snapshotFrame` when you need a frame.
 6. Call `takePendingFrames` to get the pending frames.
 
@@ -114,6 +116,12 @@ core.histogram('coins.award_size').observe(80, { grantId: 'g-80' });
 
 `timer(name, dims)` starts a timer. The returned function records the duration in milliseconds into a histogram. You can add dimensions when you stop the timer.
 
+`distinct(name, dims).add(identifier)` updates a fixed HyperLogLog sketch. The
+engine hashes the identifier as `SHA-256(privacySalt || 0x00 || identifier)` and
+discards it immediately; the frame contains only 512 HLL registers (`p=9`,
+about 4.6% standard error). Keep `privacySalt` stable across workers and time
+windows so equal identifiers map to equal registers.
+
 If a series is above `maxSeriesPerMetric`, or a dimension is not valid, the engine returns a no-op object. The engine increments `wardx.internal.cardinality_dropped`.
 
 A dimension value must be a string, a number, or a boolean.
@@ -165,6 +173,7 @@ core.applyConfig(13, {
       allocation: 1,
       salt: '3ad8f9',
       primaryMetric: 'message.sent',
+      goalMetric: 'message.sent',
       variants: [
         { key: 'control', weight: 50, values: { 'message.delayMs': 1000 } },
         { key: 'fast', weight: 50, values: { 'message.delayMs': 400 } }
@@ -218,7 +227,7 @@ const experiment = {
 const variant = assignVariant(experiment, 'user-1');
 const hash = assignmentHash(experiment.id, 'user-1', experiment.salt);
 const bucket = hashToUnitInterval(hash);
-const hashedSubject = subjectHash('dev_project_key', 'user-1');
+const hashedSubject = subjectHash('demo-subject-hash-v1', 'user-1');
 ```
 
 Hash input:
@@ -230,7 +239,7 @@ bucket = hash / 2^32
 
 If `bucket >= allocation`, `assignVariant` returns `null`.
 
-`subjectHash` returns 8 lowercase hex digits of `fnv1a32(privacySalt + ':' + subjectId)`.
+`subjectHash` returns 64 lowercase hex digits of `SHA-256(UTF8(privacySalt) || 0x00 || UTF8(subjectId))`.
 
 ## Use case 5: Build a frame for a custom transport
 
@@ -242,8 +251,8 @@ If `bucket >= allocation`, `assignVariant` returns `null`.
 import { FrameBuilder, PROTOCOL_VERSION, SDK_NAME, PLATFORM } from '@wardx/core';
 
 core.counter('match.completed').inc();
-const fitted = core.snapshotIfDirty();
-if (fitted) {
+const batch = core.snapshotIfDirty();
+if (batch) {
   const frames = core.takePendingFrames();
   const envelope = {
     protocol: PROTOCOL_VERSION,
@@ -265,12 +274,9 @@ if (fitted) {
 
 `snapshotIfDirty` returns `null` when there is no new data.
 
-If a frame is larger than `maxFrameBytes`, `FrameBuilder.fitToMaxBytes` discards data in this order:
+`snapshotFrame` uses `FrameBuilder.splitToMaxBytes`. It measures the serialized UTF-8 JSON, preserves row order within counters, gauges, histograms, events, and logs, and emits as many physical frames as needed with consecutive `seq` values. Every emitted frame is at most `maxFrameBytes`.
 
-1. Logs with the lowest severity.
-2. Events from the end of the buffer.
-3. Application histograms.
-4. Application gauges. Internal gauges stay.
+An individual row that cannot fit in an otherwise empty frame is dropped. The batch reports dropped counts by collection and emits their total as `wardx.internal.frame_rows_dropped`. If even that internal row cannot fit, splitting throws. `maxFrameBytes` must be at least `1024`.
 
 Internal series use the prefix `wardx.internal.`.
 
@@ -285,7 +291,7 @@ Internal series use the prefix `wardx.internal.`.
 | `fnv1a32`, `assignmentHash`, `hashToUnitInterval`, `subjectHash` | Hash helpers. |
 | `Counter`, `Gauge`, `Histogram`, `MetricsRegistry` | Metric types. |
 | `EventBuffer`, `LogBuffer` | In-memory buffers. |
-| `FrameBuilder` | Builds and trims frames. |
+| `FrameBuilder` | Builds and splits frames to the serialized byte limit. |
 | `resolveSettings`, `loadSdkDefaults`, `nextSyncDelayMs` | Settings helpers. |
 | `PROTOCOL_VERSION`, `SDK_NAME`, `PLATFORM`, `INTERNAL` | Protocol constants. |
 | `ulid` | Identifier helper. |

package/defaults.json

@@ -9,6 +9,7 @@
   "maxSeriesPerMetric": 1000,
   "maxDimensionKeys": 8,
   "maxDimensionValueLength": 64,
+  "experimentStateMaxSubjects": 100000,
   "httpTimeoutMs": 10000,
   "histogramBuckets": [10, 25, 50, 100, 250, 500, 1000]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wardx/core",
-  "version": "0.1.7",
+  "version": "0.4.0",
   "description": "Runtime-agnostic Wardx engine for metrics, events, logs, Remote Config, and experiments.",
   "keywords": [
     "wardx",
@@ -41,4 +41,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
+}

package/src/WardxCore.js

@@ -8,9 +8,10 @@ import { InternalMetrics } from './internal/InternalMetrics.js';
 import { NOOP_COUNTER } from './metrics/Counter.js';
 import { NOOP_GAUGE } from './metrics/Gauge.js';
 import { NOOP_HISTOGRAM } from './metrics/Histogram.js';
+import { NOOP_DISTINCT } from './metrics/HyperLogLog.js';
 import { startTimer } from './metrics/Timer.js';
 import { emit } from './trace/emit.js';
-import { wrapCounter, wrapGauge, wrapHistogram } from './trace/wrap.js';
+import { wrapCounter, wrapDistinct, wrapGauge, wrapHistogram } from './trace/wrap.js';
 
 export class WardxCore {
   constructor(settings) {
@@ -24,6 +25,7 @@ export class WardxCore {
       maxDimensionKeys: settings.maxDimensionKeys,
       maxDimensionValueLength: settings.maxDimensionValueLength,
       defaultHistogramBuckets: settings.histogramBuckets,
+      privacySalt: settings.privacySalt,
       onCardinalityDropped: () => {
         this.internal.cardinalityDropped += 1;
       }
@@ -33,6 +35,7 @@ export class WardxCore {
     this.configStore = new ConfigStore();
     this.experiments = new ExperimentResolver({
       privacySalt: settings.privacySalt,
+      stateMaxSubjects: settings.experimentStateMaxSubjects,
       onExposure: (payload) => {
         this.event('experiment.exposure', payload);
       }
@@ -67,6 +70,12 @@ export class WardxCore {
     );
   }
 
+  distinct(name, dims) {
+    return this._wrap(this.metrics.distinct(name, dims), NOOP_DISTINCT, (series, noop) =>
+      wrapDistinct(this._tracer, series, noop, name, dims)
+    );
+  }
+
   timer(name, dims) {
     if (this._tracer === null) return this.metrics.timer(name, dims);
     return startTimer((duration, endDims) => {
@@ -137,14 +146,12 @@ export class WardxCore {
       throw new Error('experiment.goal requires a metric name');
     }
     const subject = this.experiments.hashSubject(subjectId);
-    const experiments = this.experiments.relevantExperiments(
-      subjectId,
-      this.configStore.experiments
-    );
+    const assignment = this.experiments.exposedAssignmentForGoal(subjectId, name);
+    if (assignment === null) return;
     const payload = {
       metric: name,
       subject,
-      experiments
+      experiments: [{ experiment: assignment.experiment, variant: assignment.variant }]
     };
     if (context && context.value !== undefined) payload.value = context.value;
     this.event('experiment.goal', payload);
@@ -156,6 +163,7 @@ export class WardxCore {
       values: config.values,
       experiments: config.experiments
     });
+    this.experiments.applySnapshot(this.configStore.experiments);
     this.internal.configVersion = version;
   }
 
@@ -182,7 +190,7 @@ export class WardxCore {
     const logs = this.logs.swap();
     const internal = this.internal.snapshotAndReset();
     const frame = FrameBuilder.build({
-      seq: ++this.seq,
+      seq: this.seq + 1,
       from,
       to,
       metrics,
@@ -190,23 +198,27 @@ export class WardxCore {
       logs,
       internal
     });
-    const fitted = FrameBuilder.fitToMaxBytes(frame, this.settings.maxFrameBytes);
-    this.internal.logsDropped += fitted.droppedLogs;
-    this.internal.eventsDropped += fitted.droppedEvents;
-    this.pendingFrames.push(fitted.frame);
-    emit(this._tracer, 'frame', {
-      seq: fitted.frame.seq,
-      from: fitted.frame.from,
-      to: fitted.frame.to,
-      counters: fitted.frame.metrics.counters.length,
-      gauges: fitted.frame.metrics.gauges.length,
-      histograms: fitted.frame.metrics.histograms.length,
-      events: fitted.frame.events.length,
-      logs: fitted.frame.logs.length,
-      droppedLogs: fitted.droppedLogs,
-      droppedEvents: fitted.droppedEvents
-    });
-    return fitted;
+    const batch = FrameBuilder.splitToMaxBytes(frame, this.settings.maxFrameBytes);
+    this.seq = batch.frames.at(-1).seq;
+    this.pendingFrames.push(...batch.frames);
+    for (let i = 0; i < batch.frames.length; i++) {
+      const physical = batch.frames[i];
+      emit(this._tracer, 'frame', {
+        seq: physical.seq,
+        from: physical.from,
+        to: physical.to,
+        counters: physical.metrics.counters.length,
+        gauges: physical.metrics.gauges.length,
+        histograms: physical.metrics.histograms.length,
+        distincts: physical.metrics.distincts?.length || 0,
+        events: physical.events.length,
+        logs: physical.logs.length,
+        droppedLogs: i === 0 ? batch.droppedLogs : 0,
+        droppedEvents: i === 0 ? batch.droppedEvents : 0,
+        droppedRows: i === 0 ? batch.droppedRows : 0
+      });
+    }
+    return batch;
   }
 
   takePendingFrames() {

package/src/config/ConfigStore.js

@@ -15,9 +15,15 @@ export class ConfigStore {
     if (typeof snapshot.version !== 'number' || !Number.isFinite(snapshot.version)) {
       throw new Error('config snapshot version must be a finite number');
     }
+    const experiments = Array.isArray(snapshot.experiments) ? snapshot.experiments : [];
+    for (const experiment of experiments) {
+      if (typeof experiment.goalMetric !== 'string' || experiment.goalMetric.length === 0) {
+        throw new Error(`experiment ${experiment.id} requires a non-empty goalMetric`);
+      }
+    }
     this.version = snapshot.version;
     this.values = snapshot.values && typeof snapshot.values === 'object' ? snapshot.values : Object.create(null);
-    this.experiments = Array.isArray(snapshot.experiments) ? snapshot.experiments : [];
+    this.experiments = experiments;
     this.experimentsByKey = indexExperimentsByKey(this.experiments);
   }
 

package/src/config/ExperimentResolver.js

@@ -1,5 +1,19 @@
 import { assignmentHash, hashToUnitInterval, subjectHash } from './hash.js';
 
+function canonicalize(value) {
+  if (Array.isArray(value)) return value.map(canonicalize);
+  if (value && typeof value === 'object') {
+    const canonical = {};
+    for (const key of Object.keys(value).sort()) canonical[key] = canonicalize(value[key]);
+    return canonical;
+  }
+  return value;
+}
+
+export function experimentFingerprint(experiment) {
+  return subjectHash('wardx.experiment.snapshot', JSON.stringify(canonicalize(experiment)));
+}
+
 export function assignVariant(experiment, subjectId) {
   if (!experiment.enabled) return null;
   const hash = assignmentHash(experiment.id, subjectId, experiment.salt);
@@ -51,8 +65,9 @@ export class ExperimentResolver {
   constructor(options) {
     this.privacySalt = options.privacySalt;
     this.onExposure = options.onExposure;
-    this.exposureKeys = new Set();
-    this.assignmentsBySubject = new Map();
+    this.stateMaxSubjects = options.stateMaxSubjects;
+    this.stateBySubject = new Map();
+    this.activeFingerprints = new Map();
   }
 
   hashSubject(subjectId) {
@@ -60,19 +75,31 @@ export class ExperimentResolver {
   }
 
   recordAssignment(subjectId, experiment, variant) {
-    let list = this.assignmentsBySubject.get(subjectId);
-    if (!list) {
-      list = [];
-      this.assignmentsBySubject.set(subjectId, list);
-    }
-    for (let i = 0; i < list.length; i++) {
-      if (list[i].experiment === experiment.id) return;
+    const subject = this.hashSubject(subjectId);
+    let state = this.stateBySubject.get(subject);
+    if (!state) {
+      while (this.stateBySubject.size >= this.stateMaxSubjects) {
+        const oldest = this.stateBySubject.keys().next().value;
+        this.stateBySubject.delete(oldest);
+      }
+      state = { assignments: new Map() };
+      this.stateBySubject.set(subject, state);
     }
-    list.push({ experiment: experiment.id, variant: variant.key });
+    if (state.assignments.has(experiment.id)) return state.assignments.get(experiment.id);
+    const assignment = {
+      experiment: experiment.id,
+      variant: variant.key,
+      goalMetric: experiment.goalMetric,
+      fingerprint: experimentFingerprint(experiment),
+      exposed: false
+    };
+    state.assignments.set(experiment.id, assignment);
+    return assignment;
   }
 
   assignmentsFor(subjectId) {
-    return this.assignmentsBySubject.get(subjectId) || [];
+    const state = this.stateBySubject.get(this.hashSubject(subjectId));
+    return state ? Array.from(state.assignments.values()) : [];
   }
 
   resolve(key, remoteValue, subjectId, experimentsByKey) {
@@ -83,36 +110,47 @@ export class ExperimentResolver {
       const variant = assignVariant(experiment, subjectId);
       if (!variant) continue;
       if (!Object.prototype.hasOwnProperty.call(variant.values, key)) continue;
-      this.recordAssignment(subjectId, experiment, variant);
-      this._expose(experiment, variant, subjectId);
+      const assignment = this.recordAssignment(subjectId, experiment, variant);
+      this._expose(assignment, subjectId);
       return variant.values[key];
     }
     return remoteValue;
   }
 
-  _expose(experiment, variant, subjectId) {
+  _expose(assignment, subjectId) {
+    if (assignment.exposed) return;
+    assignment.exposed = true;
     const hashed = this.hashSubject(subjectId);
-    const exposureKey = experiment.id + '\0' + hashed;
-    if (this.exposureKeys.has(exposureKey)) return;
-    this.exposureKeys.add(exposureKey);
     this.onExposure({
-      experiment: experiment.id,
-      variant: variant.key,
+      experiment: assignment.experiment,
+      variant: assignment.variant,
       subject: hashed
     });
   }
 
-  relevantExperiments(subjectId, experiments) {
-    const known = this.assignmentsFor(subjectId);
-    if (known.length > 0) return known;
-    const attached = [];
-    if (!Array.isArray(experiments)) return attached;
+  exposedAssignmentForGoal(subjectId, goalMetric) {
+    const matches = this.assignmentsFor(subjectId).filter(
+      (assignment) => assignment.exposed && assignment.goalMetric === goalMetric
+    );
+    if (matches.length > 1) {
+      throw new Error(`goal metric ${goalMetric} matches multiple exposed experiments`);
+    }
+    return matches[0] || null;
+  }
+
+  applySnapshot(experiments) {
+    const active = new Map();
     for (const experiment of experiments) {
-      if (!experiment.enabled) continue;
-      const variant = assignVariant(experiment, subjectId);
-      if (!variant) continue;
-      attached.push({ experiment: experiment.id, variant: variant.key });
+      if (experiment.enabled) active.set(experiment.id, experimentFingerprint(experiment));
+    }
+    this.activeFingerprints = active;
+    for (const [subject, state] of this.stateBySubject) {
+      for (const [experimentId, assignment] of state.assignments) {
+        if (active.get(experimentId) !== assignment.fingerprint) {
+          state.assignments.delete(experimentId);
+        }
+      }
+      if (state.assignments.size === 0) this.stateBySubject.delete(subject);
     }
-    return attached;
   }
 }

package/src/config/hash.js

@@ -1,3 +1,5 @@
+import { createHash } from 'node:crypto';
+
 const encoder = new TextEncoder();
 
 export const FNV_OFFSET_32 = 0x811c9dc5;
@@ -23,5 +25,9 @@ export function assignmentHash(experimentId, subjectId, salt) {
 }
 
 export function subjectHash(projectSalt, subjectId) {
-  return (fnv1a32(`${projectSalt}:${subjectId}`) >>> 0).toString(16).padStart(8, '0');
+  return createHash('sha256')
+    .update(projectSalt, 'utf8')
+    .update('\0', 'utf8')
+    .update(subjectId, 'utf8')
+    .digest('hex');
 }

package/src/frame/FrameBuilder.js

@@ -1,46 +1,30 @@
 import { INTERNAL } from '../protocol.js';
-import { LOG_RANK } from '../protocol.js';
 
 function measure(frame) {
   const json = JSON.stringify(frame);
   return { json, bytes: Buffer.byteLength(json, 'utf8') };
 }
 
-function logDropOrder(logs) {
-  return logs
-    .map((_, index) => index)
-    .sort((a, b) => {
-      const rankA = LOG_RANK[logs[a][1]];
-      const rankB = LOG_RANK[logs[b][1]];
-      const aKey = rankA === undefined ? Infinity : rankA;
-      const bKey = rankB === undefined ? Infinity : rankB;
-      if (aKey !== bKey) return aKey - bKey;
-      return a - b;
-    });
+function emptyFrame(seq, from, to) {
+  return {
+    seq,
+    from,
+    to,
+    metrics: { counters: [], gauges: [], histograms: [] },
+    events: [],
+    logs: []
+  };
 }
 
-function logsWithoutFirstK(logs, dropOrder, k) {
-  if (k <= 0) return logs;
-  if (k >= logs.length) return [];
-  const drop = new Set(dropOrder.slice(0, k));
-  return logs.filter((_, index) => !drop.has(index));
-}
-
-function leastDrops(maxDrop, fits) {
-  if (maxDrop === 0) return 0;
-  if (!fits(maxDrop)) return maxDrop;
-  let lo = 0;
-  let hi = maxDrop;
-  while (lo < hi) {
-    const mid = (lo + hi) >> 1;
-    if (fits(mid)) hi = mid;
-    else lo = mid + 1;
-  }
-  return lo;
-}
-
-function isInternalGauge(row) {
-  return typeof row[0] === 'string' && row[0].startsWith('wardx.internal.');
+function rowCount(frame) {
+  return (
+    frame.metrics.counters.length +
+    frame.metrics.gauges.length +
+    frame.metrics.histograms.length +
+    (frame.metrics.distincts?.length || 0) +
+    frame.events.length +
+    frame.logs.length
+  );
 }
 
 export class FrameBuilder {
@@ -56,7 +40,8 @@ export class FrameBuilder {
       metrics: {
         counters,
         gauges,
-        histograms
+        histograms,
+        ...(metrics.distincts?.length > 0 ? { distincts: metrics.distincts.slice() } : {})
       },
       events,
       logs
@@ -88,45 +73,94 @@ export class FrameBuilder {
     }
   }
 
-  static fitToMaxBytes(frame, maxFrameBytes) {
-    let { json, bytes } = measure(frame);
-    if (bytes <= maxFrameBytes) return { frame, json, droppedLogs: 0, droppedEvents: 0 };
+  static splitToMaxBytes(frame, maxFrameBytes) {
+    if (!Number.isInteger(maxFrameBytes) || maxFrameBytes < 1024) {
+      throw new Error('maxFrameBytes must be an integer at least 1024');
+    }
+    const frames = [];
+    const jsons = [];
+    const dropped = {
+      counters: 0,
+      gauges: 0,
+      histograms: 0,
+      distincts: 0,
+      events: 0,
+      logs: 0
+    };
+    let current = emptyFrame(frame.seq, frame.from, frame.to);
 
-    let droppedLogs = 0;
-    let droppedEvents = 0;
+    const finishCurrent = () => {
+      const measured = measure(current);
+      if (measured.bytes > maxFrameBytes) {
+        throw new Error('frame splitter produced an oversized frame');
+      }
+      frames.push(current);
+      jsons.push(measured.json);
+      current = emptyFrame(frame.seq + frames.length, frame.from, frame.to);
+    };
 
-    if (frame.logs.length > 0) {
-      const dropOrder = logDropOrder(frame.logs);
-      const k = leastDrops(dropOrder.length, (mid) => {
-        const probe = { ...frame, logs: logsWithoutFirstK(frame.logs, dropOrder, mid) };
-        return measure(probe).bytes <= maxFrameBytes;
-      });
-      frame.logs = logsWithoutFirstK(frame.logs, dropOrder, k);
-      droppedLogs = k;
-      ({ json, bytes } = measure(frame));
-      if (bytes <= maxFrameBytes) return { frame, json, droppedLogs, droppedEvents };
-    }
+    const addRow = (collection, row, countDrop = true) => {
+      collection(current).push(row);
+      if (measure(current).bytes <= maxFrameBytes) return true;
+      collection(current).pop();
+      if (collection.kind === 'distincts' && current.metrics.distincts.length === 0) {
+        delete current.metrics.distincts;
+      }
+      if (rowCount(current) > 0) {
+        finishCurrent();
+        collection(current).push(row);
+        if (measure(current).bytes <= maxFrameBytes) return true;
+        collection(current).pop();
+        if (collection.kind === 'distincts' && current.metrics.distincts.length === 0) {
+          delete current.metrics.distincts;
+        }
+      }
+      if (countDrop) dropped[collection.kind] += 1;
+      return false;
+    };
 
-    if (frame.events.length > 0) {
-      const original = frame.events;
-      const k = leastDrops(original.length, (mid) => {
-        const probe = { ...frame, events: original.slice(0, original.length - mid) };
-        return measure(probe).bytes <= maxFrameBytes;
-      });
-      frame.events = original.slice(0, original.length - k);
-      droppedEvents = k;
-      ({ json, bytes } = measure(frame));
-      if (bytes <= maxFrameBytes) return { frame, json, droppedLogs, droppedEvents };
-    }
+    const counters = (candidate) => candidate.metrics.counters;
+    counters.kind = 'counters';
+    const gauges = (candidate) => candidate.metrics.gauges;
+    gauges.kind = 'gauges';
+    const histograms = (candidate) => candidate.metrics.histograms;
+    histograms.kind = 'histograms';
+    const distincts = (candidate) => {
+      if (!candidate.metrics.distincts) candidate.metrics.distincts = [];
+      return candidate.metrics.distincts;
+    };
+    distincts.kind = 'distincts';
+    const events = (candidate) => candidate.events;
+    events.kind = 'events';
+    const logs = (candidate) => candidate.logs;
+    logs.kind = 'logs';
 
-    if (bytes > maxFrameBytes && frame.metrics.histograms.length > 0) {
-      frame.metrics.histograms = [];
-      ({ json, bytes } = measure(frame));
-    }
-    if (bytes > maxFrameBytes) {
-      frame.metrics.gauges = frame.metrics.gauges.filter(isInternalGauge);
-      ({ json } = measure(frame));
+    for (const row of frame.metrics.counters) addRow(counters, row);
+    for (const row of frame.metrics.gauges) addRow(gauges, row);
+    for (const row of frame.metrics.histograms) addRow(histograms, row);
+    for (const row of frame.metrics.distincts || []) addRow(distincts, row);
+    for (const row of frame.events) addRow(events, row);
+    for (const row of frame.logs) addRow(logs, row);
+
+    const droppedRows = Object.values(dropped).reduce((sum, value) => sum + value, 0);
+    if (
+      droppedRows > 0 &&
+      !addRow(counters, [INTERNAL.frameRowsDropped, null, droppedRows], false)
+    ) {
+      throw new Error('maxFrameBytes cannot contain the frame drop metric');
     }
-    return { frame, json, droppedLogs, droppedEvents };
+    if (frames.length === 0 || rowCount(current) > 0) finishCurrent();
+
+    return {
+      frames,
+      jsons,
+      droppedRows,
+      droppedCounters: dropped.counters,
+      droppedGauges: dropped.gauges,
+      droppedHistograms: dropped.histograms,
+      droppedDistincts: dropped.distincts,
+      droppedEvents: dropped.events,
+      droppedLogs: dropped.logs
+    };
   }
 }

package/src/index.d.ts

@@ -15,6 +15,7 @@ export const INTERNAL: {
   readonly lastSyncMs: 'wardx.internal.last_sync_ms';
   readonly configVersion: 'wardx.internal.config_version';
   readonly processRssBytes: 'wardx.internal.process_rss_bytes';
+  readonly frameRowsDropped: 'wardx.internal.frame_rows_dropped';
 };
 
 export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
@@ -43,6 +44,10 @@ export interface HistogramHandle {
   observe(value: number, attrs?: Dimensions | null): void;
 }
 
+export interface DistinctHandle {
+  add(identifier: string): void;
+}
+
 export type StopTimer = (dims?: Dimensions | null) => void;
 
 export interface SdkDefaults {
@@ -56,6 +61,7 @@ export interface SdkDefaults {
   maxSeriesPerMetric: number;
   maxDimensionKeys: number;
   maxDimensionValueLength: number;
+  experimentStateMaxSubjects: number;
   httpTimeoutMs: number;
   histogramBuckets: number[];
 }
@@ -67,7 +73,7 @@ export interface CreateWardxOptions extends Partial<SdkDefaults> {
   role: string;
   appVersion: string;
   environment: string;
-  privacySalt?: string;
+  privacySalt: string;
   tracer?: Tracer | null;
 }
 
@@ -112,6 +118,7 @@ export interface Experiment {
   enabled: boolean;
   allocation: number;
   salt: string;
+  goalMetric: string;
   primaryMetric?: string;
   variants: ExperimentVariant[];
 }
@@ -137,6 +144,12 @@ export type LogRow = [timestamp: number, level: LogLevel, message: string, attrs
 export type CounterRow = [name: string, dims: Dimensions | null, value: number];
 export type GaugeRow = [name: string, dims: Dimensions | null, value: number, timestamp: number];
 export type HistogramRow = [name: string, dims: Dimensions | null, snapshot: HistogramSnapshot];
+export type DistinctRow = [name: string, dims: Dimensions | null, sketch: HllSketch];
+
+export interface HllSketch {
+  precision: 9;
+  registers: string;
+}
 
 export interface HistogramExemplar {
   value: number;
@@ -156,6 +169,7 @@ export interface MetricsSnapshot {
   counters: CounterRow[];
   gauges: GaugeRow[];
   histograms: HistogramRow[];
+  distincts?: DistinctRow[];
 }
 
 export interface Frame {
@@ -167,9 +181,14 @@ export interface Frame {
   logs: LogRow[];
 }
 
-export interface FittedFrame {
-  frame: Frame;
-  json: string;
+export interface FrameBatch {
+  frames: Frame[];
+  jsons: string[];
+  droppedRows: number;
+  droppedCounters: number;
+  droppedGauges: number;
+  droppedHistograms: number;
+  droppedDistincts: number;
   droppedLogs: number;
   droppedEvents: number;
 }
@@ -190,7 +209,7 @@ export interface InternalSnapshot {
 }
 
 export interface MeasureTraceRecord {
-  type: 'counter' | 'gauge' | 'histogram';
+  type: 'counter' | 'gauge' | 'histogram' | 'distinct';
   name: string;
   dims: Dimensions | null;
   op: 'inc' | 'add' | 'set' | 'observe';
@@ -219,10 +238,12 @@ export interface FrameTraceRecord {
   counters: number;
   gauges: number;
   histograms: number;
+  distincts: number;
   events: number;
   logs: number;
   droppedLogs: number;
   droppedEvents: number;
+  droppedRows: number;
 }
 
 export interface SyncTraceRecord {
@@ -299,11 +320,31 @@ export class Histogram implements HistogramHandle {
   reset(): void;
 }
 
+export class HyperLogLog implements DistinctHandle {
+  name: string;
+  dims: Dimensions | null;
+  privacySalt: string;
+  registers: Uint8Array;
+  dirty: boolean;
+  constructor(name: string, dims: Dimensions | null, privacySalt: string);
+  add(identifier: string): void;
+  snapshot(): HllSketch;
+  reset(): void;
+}
+
+export const HLL_PRECISION: 9;
+export function encodeHllRegisters(registers: Uint8Array): string;
+export function decodeHllRegisters(sketch: HllSketch): Uint8Array;
+export function estimateHllRegisters(registers: Uint8Array): number;
+export function estimateHyperLogLog(sketch: HllSketch): number;
+export function mergeHyperLogLog(left: HllSketch, right: HllSketch): HllSketch;
+
 export interface MetricsRegistryOptions {
   maxSeriesPerMetric: number;
   maxDimensionKeys: number;
   maxDimensionValueLength: number;
   defaultHistogramBuckets: number[];
+  privacySalt?: string;
   onCardinalityDropped: () => void;
 }
 
@@ -312,6 +353,7 @@ export class MetricsRegistry {
   counter(name: string, dims?: Dimensions | null): CounterHandle;
   gauge(name: string, dims?: Dimensions | null): GaugeHandle;
   histogram(name: string, a?: HistogramOptions | null, b?: HistogramOptions | null): HistogramHandle;
+  distinct(name: string, dims?: Dimensions | null): DistinctHandle;
   timer(name: string, dims?: Dimensions | null): StopTimer;
   snapshotAndReset(): MetricsSnapshot;
   isDirty(): boolean;
@@ -344,9 +386,15 @@ export class ConfigStore {
 }
 
 export class ExperimentResolver {
-  constructor(options: { privacySalt: string; onExposure: (payload: ExposurePayload) => void });
+  stateMaxSubjects: number;
+  stateBySubject: Map<string, unknown>;
+  constructor(options: {
+    privacySalt: string;
+    stateMaxSubjects: number;
+    onExposure: (payload: ExposurePayload) => void;
+  });
   hashSubject(subjectId: string): string;
-  recordAssignment(subjectId: string, experiment: Experiment, variant: ExperimentVariant): void;
+  recordAssignment(subjectId: string, experiment: Experiment, variant: ExperimentVariant): Assignment;
   assignmentsFor(subjectId: string): Assignment[];
   resolve(
     key: string,
@@ -354,7 +402,8 @@ export class ExperimentResolver {
     subjectId: string | null | undefined,
     experimentsByKey: Map<string, Experiment[]>
   ): ConfigValue;
-  relevantExperiments(subjectId: string, experiments: Experiment[]): Assignment[];
+  exposedAssignmentForGoal(subjectId: string, goalMetric: string): Assignment | null;
+  applySnapshot(experiments: Experiment[]): void;
 }
 
 export class FrameBuilder {
@@ -368,7 +417,7 @@ export class FrameBuilder {
     internal: InternalSnapshot;
   }): Frame;
   static mergeInternal(counters: CounterRow[], gauges: GaugeRow[], internal: InternalSnapshot): void;
-  static fitToMaxBytes(frame: Frame, maxFrameBytes: number): FittedFrame;
+  static splitToMaxBytes(frame: Frame, maxFrameBytes: number): FrameBatch;
 }
 
 export class WardxCore {
@@ -388,14 +437,15 @@ export class WardxCore {
   counter(name: string, dims?: Dimensions | null): CounterHandle;
   gauge(name: string, dims?: Dimensions | null): GaugeHandle;
   histogram(name: string, a?: HistogramOptions | null, b?: HistogramOptions | null): HistogramHandle;
+  distinct(name: string, dims?: Dimensions | null): DistinctHandle;
   timer(name: string, dims?: Dimensions | null): StopTimer;
   event(name: string, attrs?: Attrs | null): void;
   identify(subjectId: string | null | undefined): void;
   configGet<T>(key: string, fallback: T, context?: SubjectContext): T;
   experimentGoal(name: string, context?: ExperimentGoalContext): void;
   applyConfig(version: number, config: ConfigSnapshot): void;
-  snapshotIfDirty(): FittedFrame | null;
-  snapshotFrame(): FittedFrame;
+  snapshotIfDirty(): FrameBatch | null;
+  snapshotFrame(): FrameBatch;
   takePendingFrames(): Frame[];
 }
 

package/src/index.js

@@ -5,6 +5,15 @@ export { ConfigStore } from './config/ConfigStore.js';
 export { Counter } from './metrics/Counter.js';
 export { Gauge } from './metrics/Gauge.js';
 export { Histogram } from './metrics/Histogram.js';
+export {
+  HLL_PRECISION,
+  HyperLogLog,
+  decodeHllRegisters,
+  encodeHllRegisters,
+  estimateHllRegisters,
+  estimateHyperLogLog,
+  mergeHyperLogLog
+} from './metrics/HyperLogLog.js';
 export { MetricsRegistry } from './metrics/MetricsRegistry.js';
 export { EventBuffer } from './buffers/EventBuffer.js';
 export { LogBuffer } from './buffers/LogBuffer.js';

package/src/metrics/HyperLogLog.js

@@ -0,0 +1,106 @@
+import { createHash } from 'node:crypto';
+
+export const HLL_PRECISION = 9;
+export const HLL_REGISTER_COUNT = 1 << HLL_PRECISION;
+export const HLL_MAX_RANK = 64 - HLL_PRECISION + 1;
+
+function assertRegisters(registers) {
+  if (!(registers instanceof Uint8Array) || registers.length !== HLL_REGISTER_COUNT) {
+    throw new Error(`HLL registers must contain exactly ${HLL_REGISTER_COUNT} bytes`);
+  }
+  for (const rank of registers) {
+    if (rank > HLL_MAX_RANK) throw new Error(`HLL register rank must be <= ${HLL_MAX_RANK}`);
+  }
+}
+
+function rankAfterIndex(digest) {
+  let rank = 1;
+  for (let bit = HLL_PRECISION; bit < 64; bit++) {
+    if ((digest[bit >> 3] & (1 << (7 - (bit & 7)))) !== 0) return rank;
+    rank += 1;
+  }
+  return rank;
+}
+
+export function encodeHllRegisters(registers) {
+  assertRegisters(registers);
+  return Buffer.from(registers).toString('base64');
+}
+
+export function decodeHllRegisters(body) {
+  if (!body || body.precision !== HLL_PRECISION || typeof body.registers !== 'string') {
+    throw new Error(`HLL sketch must use precision ${HLL_PRECISION}`);
+  }
+  const decoded = Buffer.from(body.registers, 'base64');
+  if (decoded.toString('base64') !== body.registers) throw new Error('HLL registers must be canonical base64');
+  const registers = Uint8Array.from(decoded);
+  assertRegisters(registers);
+  return registers;
+}
+
+export function estimateHllRegisters(registers) {
+  assertRegisters(registers);
+  let harmonic = 0;
+  let zeros = 0;
+  for (const rank of registers) {
+    harmonic += 2 ** -rank;
+    if (rank === 0) zeros += 1;
+  }
+  const m = HLL_REGISTER_COUNT;
+  const alpha = 0.7213 / (1 + 1.079 / m);
+  const raw = alpha * m * m / harmonic;
+  const corrected = raw <= 2.5 * m && zeros > 0 ? m * Math.log(m / zeros) : raw;
+  return Math.round(corrected);
+}
+
+export function estimateHyperLogLog(body) {
+  return estimateHllRegisters(decodeHllRegisters(body));
+}
+
+export function mergeHyperLogLog(left, right) {
+  const leftRegisters = decodeHllRegisters(left);
+  const rightRegisters = decodeHllRegisters(right);
+  for (let index = 0; index < leftRegisters.length; index++) {
+    if (rightRegisters[index] > leftRegisters[index]) leftRegisters[index] = rightRegisters[index];
+  }
+  return { precision: HLL_PRECISION, registers: encodeHllRegisters(leftRegisters) };
+}
+
+export class HyperLogLog {
+  constructor(name, dims, privacySalt) {
+    if (typeof privacySalt !== 'string' || privacySalt.length === 0) {
+      throw new Error('distinct requires a non-empty privacySalt');
+    }
+    this.name = name;
+    this.dims = dims;
+    this.privacySalt = privacySalt;
+    this.registers = new Uint8Array(HLL_REGISTER_COUNT);
+    this.dirty = false;
+  }
+
+  add(identifier) {
+    if (typeof identifier !== 'string' || identifier.length === 0) {
+      throw new Error('distinct.add requires a non-empty string');
+    }
+    const digest = createHash('sha256')
+      .update(this.privacySalt, 'utf8')
+      .update('\0', 'utf8')
+      .update(identifier, 'utf8')
+      .digest();
+    const index = (digest[0] << 1) | (digest[1] >> 7);
+    const rank = rankAfterIndex(digest);
+    if (rank > this.registers[index]) this.registers[index] = rank;
+    this.dirty = true;
+  }
+
+  snapshot() {
+    return { precision: HLL_PRECISION, registers: encodeHllRegisters(this.registers) };
+  }
+
+  reset() {
+    this.registers.fill(0);
+    this.dirty = false;
+  }
+}
+
+export const NOOP_DISTINCT = Object.freeze({ add() {} });

package/src/metrics/MetricsRegistry.js

@@ -1,6 +1,7 @@
 import { Counter, NOOP_COUNTER } from './Counter.js';
 import { Gauge, NOOP_GAUGE } from './Gauge.js';
 import { Histogram, NOOP_HISTOGRAM } from './Histogram.js';
+import { HyperLogLog, NOOP_DISTINCT } from './HyperLogLog.js';
 import { startTimer } from './Timer.js';
 import { assertMetricName, dimKey, validateDimensions } from './dimensions.js';
 
@@ -48,16 +49,24 @@ export class MetricsRegistry {
     this.maxDimensionKeys = options.maxDimensionKeys;
     this.maxDimensionValueLength = options.maxDimensionValueLength;
     this.defaultHistogramBuckets = options.defaultHistogramBuckets;
+    this.privacySalt = options.privacySalt;
     this.onCardinalityDropped = options.onCardinalityDropped;
     this.countersByName = new Map();
     this.gaugesByName = new Map();
     this.histogramsByName = new Map();
+    this.distinctsByName = new Map();
     this.rejected = new Set();
   }
 
   _series(kindMap, Ctor, name, dims, extra) {
     assertMetricName(name);
-    const kindPrefix = kindMap === this.histogramsByName ? 'h' : kindMap === this.gaugesByName ? 'g' : 'c';
+    const kindPrefix = kindMap === this.histogramsByName
+      ? 'h'
+      : kindMap === this.gaugesByName
+        ? 'g'
+        : kindMap === this.distinctsByName
+          ? 'd'
+          : 'c';
     const checked = validateDimensions(dims, this.maxDimensionKeys, this.maxDimensionValueLength);
     const key = dimKey(checked.ok ? checked.dims : dims);
     const rejectKey = kindPrefix + '\0' + name + '\0' + key;
@@ -109,6 +118,13 @@ export class MetricsRegistry {
     return series;
   }
 
+  distinct(name, dims) {
+    const series = this._series(this.distinctsByName, HyperLogLog, name, dims, (resolvedDims) => {
+      return new HyperLogLog(name, resolvedDims, this.privacySalt);
+    });
+    return series || NOOP_DISTINCT;
+  }
+
   timer(name, dims) {
     const histogram = this.histogram(name, dims);
     return startTimer((duration, endDims) => {
@@ -148,7 +164,16 @@ export class MetricsRegistry {
         }
       }
     }
-    return { counters, gauges, histograms };
+    const distincts = [];
+    for (const byKey of this.distinctsByName.values()) {
+      for (const series of byKey.values()) {
+        if (series.dirty) {
+          distincts.push([series.name, series.dims, series.snapshot()]);
+          series.reset();
+        }
+      }
+    }
+    return { counters, gauges, histograms, distincts };
   }
 
   isDirty() {
@@ -167,6 +192,11 @@ export class MetricsRegistry {
         if (series.count > 0) return true;
       }
     }
+    for (const byKey of this.distinctsByName.values()) {
+      for (const series of byKey.values()) {
+        if (series.dirty) return true;
+      }
+    }
     return false;
   }
 }

package/src/protocol.js

@@ -14,7 +14,8 @@ export const INTERNAL = {
   bytesCompressed: 'wardx.internal.bytes_compressed',
   lastSyncMs: 'wardx.internal.last_sync_ms',
   configVersion: 'wardx.internal.config_version',
-  processRssBytes: 'wardx.internal.process_rss_bytes'
+  processRssBytes: 'wardx.internal.process_rss_bytes',
+  frameRowsDropped: 'wardx.internal.frame_rows_dropped'
 };
 
 export const LOG_LEVELS = ['debug', 'info', 'warn', 'error'];
@@ -34,7 +35,8 @@ export const REQUIRED_CREATE_KEYS = [
   'project',
   'role',
   'appVersion',
-  'environment'
+  'environment',
+  'privacySalt'
 ];
 
 export const REQUIRED_SDK_DEFAULT_KEYS = [
@@ -48,6 +50,7 @@ export const REQUIRED_SDK_DEFAULT_KEYS = [
   'maxSeriesPerMetric',
   'maxDimensionKeys',
   'maxDimensionValueLength',
+  'experimentStateMaxSubjects',
   'httpTimeoutMs',
   'histogramBuckets'
 ];

package/src/settings.js

@@ -57,8 +57,8 @@ export function resolveSettings(options) {
   if (settings.role === '*') {
     throw new Error('role cannot be *');
   }
-  if (settings.privacySalt === undefined || settings.privacySalt === null || settings.privacySalt === '') {
-    settings.privacySalt = settings.projectKey;
+  if (typeof settings.privacySalt !== 'string' || settings.privacySalt.length === 0) {
+    throw new Error('privacySalt must be a non-empty string');
   }
   assertPositiveNumber(settings, 'aggregateIntervalMs');
   assertPositiveNumber(settings, 'syncIntervalMs');
@@ -68,6 +68,13 @@ export function resolveSettings(options) {
   assertPositiveNumber(settings, 'maxSeriesPerMetric');
   assertPositiveNumber(settings, 'maxDimensionKeys');
   assertPositiveNumber(settings, 'maxDimensionValueLength');
+  assertPositiveNumber(settings, 'experimentStateMaxSubjects');
+  if (!Number.isInteger(settings.experimentStateMaxSubjects)) {
+    throw new Error('experimentStateMaxSubjects must be an integer');
+  }
+  if (settings.maxFrameBytes < 1024) {
+    throw new Error('maxFrameBytes must be at least 1024');
+  }
   assertPositiveNumber(settings, 'httpTimeoutMs');
   assertNumberInRange(settings, 'syncJitterMin', 0, 1);
   assertNumberInRange(settings, 'syncJitterMax', 1, 2);

package/src/trace/wrap.js

@@ -44,3 +44,14 @@ export function wrapHistogram(tracer, series, noop, name, dims) {
     }
   };
 }
+
+export function wrapDistinct(tracer, series, noop, name, dims) {
+  const n = noop ? name : series.name;
+  const d = noop ? (dims ?? null) : series.dims;
+  return {
+    add(identifier) {
+      series.add(identifier);
+      emit(tracer, 'measure', { type: 'distinct', name: n, dims: d, op: 'add', value: 1, noop });
+    }
+  };
+}