@tungthedev/streams-server 0.2.0

package/src/profiles/metrics/normalize.ts

@@ -0,0 +1,366 @@
+import { Result } from "better-result";
+import type { AggSummaryState } from "../../search/agg_format";
+import type { NormalizedMetricsRecord } from "../profile";
+import { expectPlainObjectResult, isPlainObject } from "../profile";
+
+type PrimitiveAttribute = string | number | boolean | bigint;
+
+type HistogramMap = Record<string, number>;
+
+function normalizeString(value: unknown): string | null {
+  if (typeof value !== "string") return null;
+  const trimmed = value.trim();
+  return trimmed === "" ? null : trimmed;
+}
+
+function normalizeFiniteNumber(value: unknown): number | null {
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value === "bigint") return Number(value);
+  if (typeof value === "string" && value.trim() !== "") {
+    const parsed = Number(value);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  return null;
+}
+
+function normalizeInteger(value: unknown): number | null {
+  const numeric = normalizeFiniteNumber(value);
+  if (numeric == null) return null;
+  return Math.trunc(numeric);
+}
+
+function normalizeHistogram(value: unknown): HistogramMap | undefined {
+  if (!isPlainObject(value)) return undefined;
+  const out: HistogramMap = {};
+  for (const [bucket, raw] of Object.entries(value)) {
+    const count = normalizeFiniteNumber(raw);
+    if (count == null || count <= 0) continue;
+    out[String(bucket)] = (out[String(bucket)] ?? 0) + count;
+  }
+  return Object.keys(out).length > 0 ? out : undefined;
+}
+
+function histogramPercentile(histogram: HistogramMap | undefined, percentile: number): number | null {
+  if (!histogram) return null;
+  const entries = Object.entries(histogram)
+    .map(([bucket, count]) => ({ bucket: Number(bucket), count }))
+    .filter((entry) => Number.isFinite(entry.bucket) && Number.isFinite(entry.count) && entry.count > 0)
+    .sort((a, b) => a.bucket - b.bucket);
+  if (entries.length === 0) return null;
+  const total = entries.reduce((sum, entry) => sum + entry.count, 0);
+  if (total <= 0) return null;
+  const threshold = total * percentile;
+  let seen = 0;
+  for (const entry of entries) {
+    seen += entry.count;
+    if (seen >= threshold) return entry.bucket;
+  }
+  return entries[entries.length - 1]?.bucket ?? null;
+}
+
+function normalizeAttributes(value: unknown): Record<string, string> {
+  if (!isPlainObject(value)) return {};
+  const out: Record<string, string> = {};
+  for (const [key, raw] of Object.entries(value)) {
+    if (typeof raw === "string") out[key] = raw;
+    else if (typeof raw === "number" && Number.isFinite(raw)) out[key] = String(raw);
+    else if (typeof raw === "boolean") out[key] = raw ? "true" : "false";
+    else if (typeof raw === "bigint") out[key] = raw.toString();
+  }
+  return Object.fromEntries(Object.entries(out).sort((a, b) => a[0].localeCompare(b[0])));
+}
+
+function buildDimensionPairs(attributes: Record<string, string>): string[] {
+  return Object.entries(attributes)
+    .map(([key, value]) => `${key}=${value}`)
+    .sort((a, b) => a.localeCompare(b));
+}
+
+function buildSeriesKey(args: {
+  metricKind: string;
+  temporality: string;
+  metric: string;
+  unit: string;
+  stream: string | null;
+  instance: string | null;
+  dimensionKey: string | null;
+}): string {
+  return [
+    args.metricKind,
+    args.temporality,
+    args.metric,
+    args.unit,
+    args.stream ?? "",
+    args.instance ?? "",
+    args.dimensionKey ?? "",
+  ].join("|");
+}
+
+function normalizeSummaryResult(value: Record<string, unknown>): Result<AggSummaryState, { message: string }> {
+  const summaryObject = isPlainObject(value.summary) ? value.summary : null;
+  const histogram =
+    normalizeHistogram(value.buckets) ??
+    normalizeHistogram(summaryObject?.histogram ?? undefined);
+  const count =
+    normalizeFiniteNumber(value.count) ??
+    normalizeFiniteNumber(summaryObject?.count);
+  const sum =
+    normalizeFiniteNumber(value.sum) ??
+    normalizeFiniteNumber(summaryObject?.sum);
+  const min =
+    normalizeFiniteNumber(value.min) ??
+    normalizeFiniteNumber(summaryObject?.min);
+  const max =
+    normalizeFiniteNumber(value.max) ??
+    normalizeFiniteNumber(summaryObject?.max);
+
+  if (count == null || count < 0) return Result.err({ message: "metrics interval requires count" });
+  if (sum == null) return Result.err({ message: "metrics interval requires sum" });
+
+  return Result.ok({
+    count,
+    sum,
+    min: count === 0 ? null : min ?? 0,
+    max: count === 0 ? null : max ?? 0,
+    histogram,
+  });
+}
+
+export function normalizeMetricsRecordResult(value: unknown): Result<NormalizedMetricsRecord, { message: string }> {
+  const objRes = expectPlainObjectResult(value, "metrics record");
+  if (Result.isError(objRes)) return objRes;
+  const input = objRes.value;
+
+  const kind = normalizeString(input.kind) ?? "interval";
+  if (kind !== "interval") return Result.err({ message: "metrics record.kind must be interval" });
+
+  const metric = normalizeString(input.metric);
+  if (!metric) return Result.err({ message: "metrics record.metric must be a non-empty string" });
+
+  const unit = normalizeString(input.unit);
+  if (!unit) return Result.err({ message: "metrics record.unit must be a non-empty string" });
+
+  const windowStart = normalizeInteger(input.windowStart);
+  const windowEnd = normalizeInteger(input.windowEnd);
+  if (windowStart == null || windowEnd == null) {
+    return Result.err({ message: "metrics record.windowStart and windowEnd must be integers" });
+  }
+  if (windowEnd < windowStart) return Result.err({ message: "metrics record.windowEnd must be >= windowStart" });
+
+  const intervalMs = normalizeInteger(input.intervalMs) ?? (windowEnd - windowStart);
+  if (intervalMs < 0) return Result.err({ message: "metrics record.intervalMs must be >= 0" });
+
+  const metricKind = normalizeString(input.metricKind) ?? "summary";
+  const temporality = normalizeString(input.temporality) ?? "delta";
+  const stream = normalizeString(input.stream);
+  const instance = normalizeString(input.instance);
+  const attributes = normalizeAttributes(input.attributes ?? input.tags);
+  const dimensionPairs = buildDimensionPairs(attributes);
+  const dimensionKey = dimensionPairs.length > 0 ? dimensionPairs.join("\u0000") : null;
+  const seriesKey = buildSeriesKey({
+    metricKind,
+    temporality,
+    metric,
+    unit,
+    stream,
+    instance,
+    dimensionKey,
+  });
+
+  const summaryRes = normalizeSummaryResult(input);
+  if (Result.isError(summaryRes)) return summaryRes;
+  const summary = summaryRes.value;
+  const avg = normalizeFiniteNumber(input.avg) ?? (summary.count > 0 ? summary.sum / summary.count : 0);
+  const p50 = normalizeFiniteNumber(input.p50) ?? histogramPercentile(summary.histogram, 0.5) ?? 0;
+  const p95 = normalizeFiniteNumber(input.p95) ?? histogramPercentile(summary.histogram, 0.95) ?? 0;
+  const p99 = normalizeFiniteNumber(input.p99) ?? histogramPercentile(summary.histogram, 0.99) ?? 0;
+
+  const normalizedValue: Record<string, unknown> = {
+    apiVersion: "durable.streams/metrics/v1",
+    kind: "interval",
+    metric,
+    unit,
+    metricKind,
+    temporality,
+    windowStart,
+    windowEnd,
+    intervalMs,
+    instance,
+    stream,
+    tags: attributes,
+    attributes,
+    dimensionPairs,
+    dimensionKey,
+    seriesKey,
+    count: summary.count,
+    sum: summary.sum,
+    min: summary.min,
+    max: summary.max,
+    avg,
+    p50,
+    p95,
+    p99,
+    buckets: summary.histogram ?? {},
+    summary: {
+      count: summary.count,
+      sum: summary.sum,
+      min: summary.min,
+      max: summary.max,
+      histogram: summary.histogram ?? {},
+    },
+  };
+
+  return Result.ok({
+    value: normalizedValue,
+    routingKey: seriesKey,
+    companion: {
+      metric,
+      unit,
+      metricKind,
+      temporality,
+      windowStartMs: windowStart,
+      windowEndMs: windowEnd,
+      intervalMs,
+      stream,
+      instance,
+      attributes,
+      dimensionPairs,
+      dimensionKey,
+      seriesKey,
+      summary,
+    },
+  });
+}
+
+export type MetricsBlockRecord = {
+  doc_id: number;
+  metric: string;
+  unit: string;
+  metricKind: string;
+  temporality: string;
+  windowStartMs: number;
+  windowEndMs: number;
+  intervalMs: number;
+  stream: string | null;
+  instance: string | null;
+  attributes: Record<string, string>;
+  dimensionPairs: string[];
+  dimensionKey: string | null;
+  seriesKey: string;
+  summary: AggSummaryState;
+};
+
+export function buildMetricsBlockRecord(docId: number, value: unknown): Result<MetricsBlockRecord, { message: string }> {
+  const normalizedRes = normalizeMetricsRecordResult(value);
+  if (Result.isError(normalizedRes)) return normalizedRes;
+  return Result.ok({
+    doc_id: docId,
+    ...normalizedRes.value.companion,
+  });
+}
+
+export function materializeMetricsBlockRecord(record: MetricsBlockRecord): Record<string, unknown> {
+  const histogram = record.summary.histogram ?? {};
+  const avg = record.summary.count > 0 ? record.summary.sum / record.summary.count : 0;
+  return {
+    apiVersion: "durable.streams/metrics/v1",
+    kind: "interval",
+    metric: record.metric,
+    unit: record.unit,
+    metricKind: record.metricKind,
+    temporality: record.temporality,
+    windowStart: record.windowStartMs,
+    windowEnd: record.windowEndMs,
+    intervalMs: record.intervalMs,
+    instance: record.instance,
+    stream: record.stream,
+    tags: { ...record.attributes },
+    attributes: { ...record.attributes },
+    dimensionPairs: [...record.dimensionPairs],
+    dimensionKey: record.dimensionKey,
+    seriesKey: record.seriesKey,
+    count: record.summary.count,
+    sum: record.summary.sum,
+    min: record.summary.min,
+    max: record.summary.max,
+    avg,
+    p50: histogramPercentile(histogram, 0.5) ?? 0,
+    p95: histogramPercentile(histogram, 0.95) ?? 0,
+    p99: histogramPercentile(histogram, 0.99) ?? 0,
+    buckets: { ...histogram },
+    summary: {
+      count: record.summary.count,
+      sum: record.summary.sum,
+      min: record.summary.min,
+      max: record.summary.max,
+      histogram: { ...histogram },
+    },
+  };
+}
+
+export function buildInternalMetricsRecord(args: {
+  metric: string;
+  unit: string;
+  windowStart: number;
+  windowEnd: number;
+  intervalMs: number;
+  instance: string;
+  stream?: string;
+  tags?: Record<string, PrimitiveAttribute>;
+  count: number;
+  sum: number;
+  min: number;
+  max: number;
+  avg: number;
+  p50: number;
+  p95: number;
+  p99: number;
+  buckets: Record<string, number>;
+}): Record<string, unknown> {
+  const attributes = normalizeAttributes(args.tags);
+  const dimensionPairs = buildDimensionPairs(attributes);
+  const dimensionKey = dimensionPairs.length > 0 ? dimensionPairs.join("\u0000") : null;
+  const seriesKey = buildSeriesKey({
+    metricKind: "summary",
+    temporality: "delta",
+    metric: args.metric,
+    unit: args.unit,
+    stream: args.stream ?? null,
+    instance: args.instance,
+    dimensionKey,
+  });
+  return {
+    apiVersion: "durable.streams/metrics/v1",
+    kind: "interval",
+    metric: args.metric,
+    unit: args.unit,
+    metricKind: "summary",
+    temporality: "delta",
+    windowStart: args.windowStart,
+    windowEnd: args.windowEnd,
+    intervalMs: args.intervalMs,
+    instance: args.instance,
+    stream: args.stream ?? null,
+    tags: attributes,
+    attributes,
+    dimensionPairs,
+    dimensionKey,
+    seriesKey,
+    count: args.count,
+    sum: args.sum,
+    min: args.count === 0 ? null : args.min,
+    max: args.count === 0 ? null : args.max,
+    avg: args.avg,
+    p50: args.p50,
+    p95: args.p95,
+    p99: args.p99,
+    buckets: { ...args.buckets },
+    summary: {
+      count: args.count,
+      sum: args.sum,
+      min: args.count === 0 ? null : args.min,
+      max: args.count === 0 ? null : args.max,
+      histogram: { ...args.buckets },
+    },
+  };
+}

package/src/profiles/metrics/schema.ts

@@ -0,0 +1,319 @@
+import {
+  SCHEMA_REGISTRY_API_VERSION,
+  type SchemaRegistry,
+  type SearchConfig,
+} from "../../schema/registry";
+
+export const METRICS_CANONICAL_SCHEMA = {
+  type: "object",
+  additionalProperties: false,
+  properties: {
+    apiVersion: { type: "string", const: "durable.streams/metrics/v1" },
+    kind: { type: "string", const: "interval" },
+    metric: { type: "string" },
+    unit: { type: "string" },
+    metricKind: { type: "string" },
+    temporality: { type: "string" },
+    windowStart: { type: "integer" },
+    windowEnd: { type: "integer" },
+    intervalMs: { type: "integer" },
+    instance: { type: ["string", "null"] },
+    stream: { type: ["string", "null"] },
+    tags: {
+      type: "object",
+      additionalProperties: { type: "string" },
+    },
+    attributes: {
+      type: "object",
+      additionalProperties: { type: "string" },
+    },
+    dimensionPairs: {
+      type: "array",
+      items: { type: "string" },
+    },
+    dimensionKey: { type: ["string", "null"] },
+    seriesKey: { type: "string" },
+    count: { type: "number" },
+    sum: { type: "number" },
+    min: { type: ["number", "null"] },
+    max: { type: ["number", "null"] },
+    avg: { type: "number" },
+    p50: { type: "number" },
+    p95: { type: "number" },
+    p99: { type: "number" },
+    buckets: {
+      type: "object",
+      additionalProperties: { type: "number" },
+    },
+    summary: {
+      type: "object",
+      additionalProperties: false,
+      properties: {
+        count: { type: "number" },
+        sum: { type: "number" },
+        min: { type: ["number", "null"] },
+        max: { type: ["number", "null"] },
+        histogram: {
+          type: "object",
+          additionalProperties: { type: "number" },
+        },
+      },
+      required: ["count", "sum", "min", "max", "histogram"],
+    },
+  },
+  required: [
+    "apiVersion",
+    "kind",
+    "metric",
+    "unit",
+    "metricKind",
+    "temporality",
+    "windowStart",
+    "windowEnd",
+    "intervalMs",
+    "instance",
+    "stream",
+    "tags",
+    "attributes",
+    "dimensionPairs",
+    "dimensionKey",
+    "seriesKey",
+    "count",
+    "sum",
+    "min",
+    "max",
+    "avg",
+    "p50",
+    "p95",
+    "p99",
+    "buckets",
+    "summary",
+  ],
+} as const;
+
+export const METRICS_DEFAULT_SEARCH_CONFIG: SearchConfig = {
+  profile: "metrics",
+  primaryTimestampField: "windowStart",
+  aliases: {
+    ts: "windowStart",
+    time: "windowStart",
+    name: "metric",
+    dims: "dimensionKey",
+    series: "seriesKey",
+  },
+  defaultFields: [
+    { field: "metric", boost: 2 },
+    { field: "stream", boost: 1.25 },
+    { field: "dimensionPairs", boost: 1 },
+  ],
+  fields: {
+    windowStart: {
+      kind: "date",
+      bindings: [{ version: 1, jsonPointer: "/windowStart" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    windowEnd: {
+      kind: "date",
+      bindings: [{ version: 1, jsonPointer: "/windowEnd" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+    },
+    intervalMs: {
+      kind: "integer",
+      bindings: [{ version: 1, jsonPointer: "/intervalMs" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    metric: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/metric" }],
+      normalizer: "lowercase_v1",
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    unit: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/unit" }],
+      normalizer: "lowercase_v1",
+      exact: true,
+      prefix: true,
+      exists: true,
+      aggregatable: true,
+    },
+    metricKind: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/metricKind" }],
+      normalizer: "lowercase_v1",
+      exact: true,
+      prefix: true,
+      exists: true,
+      aggregatable: true,
+    },
+    temporality: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/temporality" }],
+      normalizer: "lowercase_v1",
+      exact: true,
+      prefix: true,
+      exists: true,
+      aggregatable: true,
+    },
+    stream: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/stream" }],
+      normalizer: "lowercase_v1",
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    instance: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/instance" }],
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    seriesKey: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/seriesKey" }],
+      exact: true,
+      exists: true,
+      sortable: true,
+    },
+    dimensionKey: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/dimensionKey" }],
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    dimensionPairs: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/dimensionPairs" }],
+      exact: true,
+      prefix: true,
+      exists: true,
+    },
+    count: {
+      kind: "integer",
+      bindings: [{ version: 1, jsonPointer: "/count" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    sum: {
+      kind: "float",
+      bindings: [{ version: 1, jsonPointer: "/sum" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    min: {
+      kind: "float",
+      bindings: [{ version: 1, jsonPointer: "/min" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+    },
+    max: {
+      kind: "float",
+      bindings: [{ version: 1, jsonPointer: "/max" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+    },
+    avg: {
+      kind: "float",
+      bindings: [{ version: 1, jsonPointer: "/avg" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+    },
+    p95: {
+      kind: "float",
+      bindings: [{ version: 1, jsonPointer: "/p95" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+    },
+    p99: {
+      kind: "float",
+      bindings: [{ version: 1, jsonPointer: "/p99" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+    },
+  },
+  rollups: {
+    metrics: {
+      timestampField: "windowStart",
+      dimensions: ["metric", "unit", "stream", "instance", "dimensionKey", "metricKind", "temporality"],
+      intervals: ["10s", "1m", "5m", "1h"],
+      measures: {
+        value: {
+          kind: "summary_parts",
+          countJsonPointer: "/count",
+          sumJsonPointer: "/sum",
+          minJsonPointer: "/min",
+          maxJsonPointer: "/max",
+          histogramJsonPointer: "/buckets",
+        },
+      },
+    },
+  },
+};
+
+export function buildMetricsDefaultRegistry(stream: string): SchemaRegistry {
+  return {
+    apiVersion: SCHEMA_REGISTRY_API_VERSION,
+    schema: stream,
+    currentVersion: 1,
+    routingKey: { jsonPointer: "/seriesKey", required: true },
+    search: structuredClone(METRICS_DEFAULT_SEARCH_CONFIG),
+    boundaries: [{ offset: 0, version: 1 }],
+    schemas: {
+      "1": structuredClone(METRICS_CANONICAL_SCHEMA),
+    },
+    lenses: {},
+  };
+}
+
+export function buildInternalMetricsRegistry(stream: string): SchemaRegistry {
+  return {
+    apiVersion: SCHEMA_REGISTRY_API_VERSION,
+    schema: stream,
+    currentVersion: 1,
+    boundaries: [{ offset: 0, version: 1 }],
+    schemas: {
+      "1": structuredClone(METRICS_CANONICAL_SCHEMA),
+    },
+    lenses: {},
+  };
+}