@ngockhoale/ukit 3.0.12 → 3.1.1

package/src/core/observability/analytics/anomalies.js

@@ -0,0 +1,415 @@
+/**
+ * anomalies.js (TASK-011, SPEC §5 DF2-FR11, §7) — deterministic anomaly
+ * detection over TraceSummaries.
+ *
+ *   detectAnomalies(summaries, opts?) → AnomalyRecord[]
+ *
+ * Pure function. No IO, no clock, no randomness — the same summaries
+ * always produce the same anomaly records. Output order is deterministic
+ * (trace_id, then kind).
+ *
+ * Detectors:
+ *   duration-outlier              — critical_path_ms > OUTLIER_FACTOR × the
+ *                                   cohort p50 (cohort = structural
+ *                                   fingerprint; requires ≥ MIN_COHORT
+ *                                   measured members — the honest-sample
+ *                                   rule, mirroring summary.js: below the
+ *                                   floor the claim does not exist).
+ *   retry-storm                   — ≥ RETRY_STORM_MIN spans with
+ *                                   attempt_index > 1 against one operation.
+ *   cache-miss-spike              — ≥ CACHE_MIN_LOOKUPS lookups AND
+ *                                   misses > CACHE_MISS_FACTOR × hits.
+ *   drop-rate-spike               — ≥ DROP_SPIKE_MIN telemetry.dropped
+ *                                   events inside one trace.
+ *   verification-failure-cluster  — ≥ FAILURE_CLUSTER_MIN failed spans
+ *                                   inside one trace.
+ *
+ * Each hit emits ONE anomaly.detected derived record per trace per
+ * detector:
+ *   payload: { detector, kind, evidence_refs[],
+ *              confidence: { type: 'STATISTICAL', value },
+ *              cohort_key }
+ *
+ * Contract:
+ *   - evidence_refs contain ONLY real record_ids. With evidence available
+ *     inside the summary that means the pattern spans' start/end record
+ *     ids; event-level detectors (cache/drop) can additionally cite the
+ *     source records via opts.evidenceRecords (Map<trace_id, record[]>) —
+ *     outside sources are resolved through the map and never invented.
+ *     A hit that cannot cite any real record_id is never emitted.
+ *   - cohort_key is the structural fingerprint hash of the summary's
+ *     cohort (computeFingerprint) — the grouping identity, never a
+ *     per-trace identity.
+ *   - confidence.value ∈ (0, 1]: the measured signal ÷ the threshold that
+ *     defines the anomaly, capped at 0.99 — detection is statistical
+ *     evidence, never certainty.
+ *   - The records are semantic-envelope SKELETONS: trace_id, origin.
+ *     derived_from and payload are complete; boot_id/writer_id/sequence/
+ *     wall_time_utc/monotonic_ns are deliberately absent — the emit lane
+ *     stamps them on persistence (fillEnvelope). The records are
+ *     deterministic and IO-free.
+ *   - Malformed input violates the contract: non-array input or a
+ *     summary missing the TraceSummary shape → TypeError. An EMPTY spans
+ *     array is valid — summarizeTrace emits it for event-only traces.
+ */
+
+import { computeFingerprint } from './fingerprints.js';
+
+/** Cohort floor for relative claims — mirrors opportunities.js. */
+export const MIN_COHORT = 3;
+
+/** Duration outlier: critical path above this multiple of cohort p50. */
+const OUTLIER_FACTOR = 2;
+/** Retry storm: retries (attempt_index > 1) of a single operation. */
+const RETRY_STORM_MIN = 3;
+/** Cache miss spike: lookups floor + misses ÷ hits factor. */
+const CACHE_MIN_LOOKUPS = 3;
+const CACHE_MISS_FACTOR = 2;
+/** Drop spike: telemetry.dropped events in one trace. */
+const DROP_SPIKE_MIN = 3;
+/** Verification-failure cluster: traces in one fingerprint cohort whose
+ *  verification gate returned outcome 'failed'. Cohort floor = MIN_COHORT. */
+const VERIFICATION_CLUSTER_MIN = MIN_COHORT;
+
+function isPlainObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function num(value) {
+  return typeof value === 'number' && Number.isFinite(value) ? value : 0;
+}
+
+/** Median of a sorted finite list (nearest rank, never interpolated). */
+function medianOf(sorted) {
+  if (sorted.length === 0) return null;
+  return sorted[Math.floor((sorted.length - 1) / 2)];
+}
+
+/**
+ * The critical path's dominant span chain, root-first: the root span that
+ * contributes the maximum causal path, then the longest measured child at
+ * each level. Used to cite the spans that embody a duration outlier.
+ */
+function criticalPathSpans(summary) {
+  const spans = summary.spans.filter(isPlainObject);
+  const children = new Map();
+  const roots = [];
+  const byId = new Set(spans.map((s) => s.span_id));
+  for (const span of spans) {
+    if (span.parent_span_id && byId.has(span.parent_span_id)) {
+      const list = children.get(span.parent_span_id) || [];
+      list.push(span);
+      children.set(span.parent_span_id, list);
+    } else {
+      roots.push(span);
+    }
+  }
+
+  function pathOf(span) {
+    let best = typeof span.duration_ms === 'number' ? span.duration_ms : null;
+    let minStart = Infinity;
+    let maxEnd = -Infinity;
+    for (const kid of children.get(span.span_id) || []) {
+      const kidPath = pathOf(kid);
+      if (kidPath !== null && (best === null || kidPath > best)) best = kidPath;
+      if (typeof kid.start_ns === 'number' && typeof kid.end_ns === 'number') {
+        if (kid.start_ns < minStart) minStart = kid.start_ns;
+        if (kid.end_ns > maxEnd) maxEnd = kid.end_ns;
+      }
+    }
+    if (maxEnd >= minStart) {
+      const extent = (maxEnd - minStart) / 1e6;
+      if (best === null || extent > best) best = extent;
+    }
+    return best;
+  }
+
+  let root = null;
+  let bestPath = null;
+  for (const candidate of roots) {
+    const path = pathOf(candidate);
+    if (path !== null && (bestPath === null || path > bestPath)) {
+      bestPath = path;
+      root = candidate;
+    }
+  }
+  if (root === null) return [];
+
+  const chain = [root];
+  let cursor = root;
+  for (;;) {
+    const kids = children.get(cursor.span_id) || [];
+    let bestKid = null;
+    let bestKidPath = null;
+    for (const kid of kids) {
+      const kidPath = pathOf(kid);
+      if (kidPath !== null && (bestKidPath === null || kidPath > bestKidPath)) {
+        bestKidPath = kidPath;
+        bestKid = kid;
+      }
+    }
+    if (bestKid === null) break;
+    chain.push(bestKid);
+    cursor = bestKid;
+  }
+  return chain;
+}
+
+
+/** All resolvable record ids carried by a span (start first, end second). */
+function recordRefsOf(span) {
+  const refs = [];
+  if (typeof span.start_record_id === 'string' && span.start_record_id.length > 0) {
+    refs.push(span.start_record_id);
+  }
+  if (typeof span.end_record_id === 'string' && span.end_record_id.length > 0) {
+    refs.push(span.end_record_id);
+  }
+  return refs;
+}
+
+/**
+ * Fallback citation when the anomaly has no dedicated span set: the root
+ * span of the trace (the unit the detector ran over). Real ids only.
+ */
+function rootRefsOf(summary) {
+  for (const span of summary.spans) {
+    if (isPlainObject(span) && (span.parent_span_id === null || span.parent_span_id === undefined)) {
+      const refs = recordRefsOf(span);
+      if (refs.length > 0) return refs;
+    }
+  }
+  // No root — cite the first span that carries ids rather than nothing.
+  for (const span of summary.spans) {
+    if (!isPlainObject(span)) continue;
+    const refs = recordRefsOf(span);
+    if (refs.length > 0) return refs;
+  }
+  return [];
+}
+
+/** Record ids of source records matching a semantic name (event evidence). */
+function eventRefsOf(sourceRecords, semanticName) {
+  const refs = [];
+  for (const record of sourceRecords || []) {
+    if (isPlainObject(record) && record.semantic_name === semanticName &&
+        typeof record.record_id === 'string' && record.record_id.length > 0) {
+      refs.push(record.record_id);
+    }
+  }
+  return refs;
+}
+
+/** Confidence: measured signal ÷ the threshold that defines the anomaly —
+ * a statistical strength ratio (may exceed 1), never certainty. */
+function confidenceOf(value) {
+  if (!Number.isFinite(value) || value <= 0) return null;
+  return Math.round(value * 1000) / 1000;
+}
+
+function anomalyRecord(summary, fingerprint, kind, evidenceRefs, confidence) {
+  const refs = [...new Set(evidenceRefs)].sort();
+  if (refs.length === 0 || confidence === null) return null;
+  return {
+    record_type: 'derived',
+    semantic_name: 'anomaly.detected',
+    record_id: `anm-${fingerprint.hash}-${kind}`,
+    trace_id: typeof summary.trace_id === 'string' ? summary.trace_id : null,
+    importance: 'high',
+    privacy_class: 'internal',
+    origin: { derived_from: refs.slice() },
+    payload: {
+      detector: kind,
+      kind,
+      evidence_refs: refs,
+      confidence: { type: 'STATISTICAL', value: confidence },
+      cohort_key: fingerprint.hash,
+    },
+  };
+}
+
+/**
+ * @param {TraceSummary[]} summaries
+ * @param {object} [opts]
+ * @param {Map<string, object[]|null>|object} [opts.evidenceRecords] —
+ *   trace_id → source records, for event-level evidence (telemetry.dropped,
+ *   cache.*). Absent entries fall back to span-level ids.
+ * @returns {object[]} anomaly.detected derived-record skeletons
+ */
+export function detectAnomalies(summaries, opts = {}) {
+  if (!Array.isArray(summaries)) {
+    throw new TypeError('detectAnomalies: summaries must be an array of TraceSummary');
+  }
+  for (const summary of summaries) {
+    // A valid TraceSummary may carry spans: [] — event-only traces produce
+    // it and the event-level detectors (cache/drop/verification) still
+    // apply. What is malformed is an object that is not a TraceSummary at
+    // all: coverage and durations accompany every summarizeTrace result,
+    // so requiring them keeps genuine shape violations a TypeError.
+    if (
+      !isPlainObject(summary) ||
+      !Array.isArray(summary.spans) ||
+      !isPlainObject(summary.coverage) ||
+      !isPlainObject(summary.durations)
+    ) {
+      throw new TypeError(
+        'detectAnomalies: each summary must be a TraceSummary object with spans, coverage and durations'
+      );
+    }
+  }
+
+  const evidenceRecords = isPlainObject(opts.evidenceRecords) || opts.evidenceRecords instanceof Map
+    ? opts.evidenceRecords
+    : null;
+  const sourceRecordsOf = (traceId) => {
+    if (!evidenceRecords || typeof traceId !== 'string') return [];
+    if (evidenceRecords instanceof Map) return evidenceRecords.get(traceId) || [];
+    return evidenceRecords[traceId] || [];
+  };
+
+  // Fingerprint cohorts — the comparison group for relative detectors.
+  const bySummary = summaries.map((summary) => ({
+    summary,
+    fingerprint: computeFingerprint(summary),
+  }));
+  const cohortMembers = new Map();
+  for (const { fingerprint } of bySummary) {
+    cohortMembers.set(fingerprint.hash, (cohortMembers.get(fingerprint.hash) || 0) + 1);
+  }
+  // Cohort duration distribution (measured critical paths only — unmeasured
+  // traces are excluded from the baseline, never counted as 0).
+  const cohortDurations = new Map();
+  for (const { summary, fingerprint } of bySummary) {
+    if (typeof summary.critical_path_ms === 'number' && Number.isFinite(summary.critical_path_ms)) {
+      const list = cohortDurations.get(fingerprint.hash) || [];
+      list.push(summary.critical_path_ms);
+      cohortDurations.set(fingerprint.hash, list);
+    }
+  }
+  for (const list of cohortDurations.values()) list.sort((a, b) => a - b);
+
+  const anomalies = [];
+  const push = (record) => {
+    if (record) anomalies.push(record);
+  };
+
+  for (const { summary, fingerprint } of bySummary) {
+    const sourceRecords = sourceRecordsOf(summary.trace_id);
+
+    // --- duration-outlier (cohort-relative, honest-sample floor) ---
+    const durations = cohortDurations.get(fingerprint.hash) || [];
+    const members = cohortMembers.get(fingerprint.hash) || 0;
+    const median = members >= MIN_COHORT && durations.length >= MIN_COHORT
+      ? medianOf(durations)
+      : null;
+    const path = typeof summary.critical_path_ms === 'number' &&
+      Number.isFinite(summary.critical_path_ms)
+      ? summary.critical_path_ms
+      : null;
+    if (median !== null && path !== null && path > OUTLIER_FACTOR * median) {
+      const chainRefs = criticalPathSpans(summary).flatMap(recordRefsOf);
+      push(anomalyRecord(
+        summary,
+        fingerprint,
+        'duration-outlier',
+        chainRefs.length > 0 ? chainRefs : rootRefsOf(summary),
+        confidenceOf(path / (OUTLIER_FACTOR * median)),
+      ));
+    }
+
+    // --- retry-storm (≥N retried attempts of one operation) ---
+    const retriesByOp = new Map();
+    for (const span of summary.spans) {
+      if (!isPlainObject(span) || !(Number.isInteger(span.attempt_index) && span.attempt_index > 1)) continue;
+      const op = typeof span.operation === 'string' ? span.operation : 'unknown';
+      const list = retriesByOp.get(op) || [];
+      list.push(span);
+      retriesByOp.set(op, list);
+    }
+    for (const spans of retriesByOp.values()) {
+      if (spans.length >= RETRY_STORM_MIN) {
+        push(anomalyRecord(
+          summary,
+          fingerprint,
+          'retry-storm',
+          spans.flatMap(recordRefsOf),
+          confidenceOf(spans.length / RETRY_STORM_MIN),
+        ));
+        break; // one record per trace per detector — evidence is per-trace
+      }
+    }
+
+    // --- cache-miss-spike ---
+    const cache = isPlainObject(summary.cache) ? summary.cache : {};
+    const hits = num(cache.hits);
+    const misses = num(cache.misses);
+    const lookups = hits + misses;
+    if (lookups >= CACHE_MIN_LOOKUPS && misses > CACHE_MISS_FACTOR * hits) {
+      const refs = [...eventRefsOf(sourceRecords, 'cache.miss'), ...eventRefsOf(sourceRecords, 'cache.hit')];
+      push(anomalyRecord(
+        summary,
+        fingerprint,
+        'cache-miss-spike',
+        refs.length > 0 ? refs : rootRefsOf(summary),
+        confidenceOf(misses / Math.max(1, CACHE_MISS_FACTOR * hits)),
+      ));
+    }
+
+    // --- drop-rate-spike ---
+    const drops = isPlainObject(summary.drops) ? summary.drops : {};
+    if (num(drops.events) >= DROP_SPIKE_MIN) {
+      const refs = eventRefsOf(sourceRecords, 'telemetry.dropped');
+      push(anomalyRecord(
+        summary,
+        fingerprint,
+        'drop-rate-spike',
+        refs.length > 0 ? refs : rootRefsOf(summary),
+        confidenceOf(num(drops.events) / DROP_SPIKE_MIN),
+      ));
+    }
+
+    // --- verification-failure evidence: collect per cohort below ---
+  }
+
+  // --- verification-failure cluster (cohort-level) ---
+  // A structural cohort in which ≥ VERIFICATION_CLUSTER_MIN traces failed
+  // their verification gate is a cluster, not a fluke. Evidence is the
+  // failed verification record ids carried by summary.verification.
+  const cohorts = new Map();
+  for (const { summary, fingerprint } of bySummary) {
+    let cohort = cohorts.get(fingerprint.hash);
+    if (!cohort) {
+      cohort = { fingerprint, summaries: [] };
+      cohorts.set(fingerprint.hash, cohort);
+    }
+    cohort.summaries.push(summary);
+  }
+  for (const cohort of cohorts.values()) {
+    const failedSummaries = cohort.summaries.filter((s) => {
+      const v = isPlainObject(s.verification) ? s.verification : {};
+      return num(v.failed) > 0;
+    });
+    if (failedSummaries.length < VERIFICATION_CLUSTER_MIN) continue;
+    const refs = [];
+    for (const s of failedSummaries) {
+      const v = isPlainObject(s.verification) ? s.verification : {};
+      if (Array.isArray(v.failed_refs)) refs.push(...v.failed_refs.filter((r) => typeof r === 'string' && r.length > 0));
+    }
+    push(anomalyRecord(
+      { trace_id: null },
+      cohort.fingerprint,
+      'verification-failure-cluster',
+      refs,
+      confidenceOf(failedSummaries.length / VERIFICATION_CLUSTER_MIN),
+    ));
+  }
+
+  // Deterministic output order: trace_id, then kind.
+  anomalies.sort((a, b) => {
+    const at = String(a.trace_id);
+    const bt = String(b.trace_id);
+    if (at !== bt) return at < bt ? -1 : 1;
+    return a.payload.kind < b.payload.kind ? -1 : a.payload.kind > b.payload.kind ? 1 : 0;
+  });
+  return anomalies;
+}

package/src/core/observability/analytics/summary.js

@@ -8,7 +8,8 @@
  *
  * TraceSummary shape (SPEC §8):
  *   { trace_id, spans[], durations{}, critical_path_ms, retries, drops,
- *     cache{}, resource{}, telemetry_complete, coverage{}, metric_version }
+ *     cache{}, resource{}, verification{completed,failed,failed_refs[]},
+ *     telemetry_complete, coverage{}, metric_version }
  *
  * Rules that make this a flight recorder and not a dashboard:
  *   - critical_path_ms is CAUSAL: a span contributes its own measured
@@ -100,6 +101,7 @@ export function summarizeTrace(records) {
   const cache = { observed: false, hits: 0, misses: 0, hit_rate: null };
   const resource = { total_value: null, numeric_count: 0, unknown_count: 0, sources: {} };
   const retries = { model_attempts: 0, retries: 0, failed_spans: 0 };
+  const verification = { completed: 0, failed: 0, failed_refs: [] };
   const coverage = {
     records: ordered.length,
     records_with_trace: 0,
@@ -156,6 +158,18 @@ export function summarizeTrace(records) {
       }
     }
 
+    // Verification gate outcomes (event-level, not span pairs): outcome
+    // verdict + the record's own id for downstream evidence citation.
+    if (name === 'verification.completed') {
+      verification.completed += 1;
+      if (payload.outcome === 'failed') {
+        verification.failed += 1;
+        if (typeof record.record_id === 'string' && record.record_id.length > 0) {
+          verification.failed_refs.push(record.record_id);
+        }
+      }
+    }
+
     if (name === 'cache.hit') cache.hits += 1;
     if (name === 'cache.miss') cache.misses += 1;
     if (name === 'telemetry.dropped') {
@@ -290,6 +304,7 @@ export function summarizeTrace(records) {
     retries,
     drops,
     cache,
+    verification,
     resource,
     telemetry_complete,
     coverage,

package/src/core/observability/emit/config.js

@@ -2,7 +2,9 @@
  * config.js (TASK-007, SPEC §8 / §13) — observability stage gate.
  *
  *   resolveStage(config) → 'off' | 'shadow' | 'canary' | 'default'
- *
+ *   resolveSampling(config) → { policy_version, rates } | null   (DF2-FR02)
+ *   samplingConfigInvalid() → boolean — one-shot latch, true after a
+ *     malformed `observability.sampling` node was seen (once per boot).
  * Reads `observability.stage` from the project runtime config object.
  * Stages promote off → shadow → canary → default; the kill switch is
  * setting the value back to 'off'. Absent config, absent key, non-object
@@ -27,3 +29,69 @@ export function resolveStage(config = null) {
   const stage = isPlainObject(node) ? node.stage : undefined;
   return VALID_STAGES.has(stage) ? stage : 'off';
 }
+
+// --- DF2-FR02: sampling policy -------------------------------------------
+//
+// `observability.sampling = { policy_version, rates }` where rates keys are
+// the frozen IMPORTANCE_LEVELS vocabulary (low|normal|high|critical) with
+// values in [0, 1]; absent levels default to 1.0 (keep everything at that
+// level). resolveSampling mirrors resolveStage's fail-safe posture:
+// absent sampling node → null (feature off); ANY malformed part — wrong
+// type, unknown rate key, rate outside [0,1], missing/oversize
+// policy_version — → null AND latches the CONFIG_INVALID flag so the
+// recorder can report the configuration failure exactly once per boot
+// (telemetry.dropped{reason_code:'CONFIG_INVALID'}). Guessing at a partial
+// policy would silently drop data the operator never asked to drop, so a
+// bad config disables sampling wholesale.
+
+const SAMPLING_RATE_KEYS = new Set(['low', 'normal', 'high', 'critical']);
+const MAX_POLICY_VERSION_CHARS = 128;
+
+// Process-wide latch: a recorder boot is a process, so "once per boot" is a
+// module-level one-shot — the same granularity resolveStage operates at.
+let samplingConfigInvalidSeen = false;
+
+export function samplingConfigInvalid() {
+  return samplingConfigInvalidSeen;
+}
+
+// Test/secondary-consumer hook: the flag is a boot-scoped observation, not
+// stateful policy — clearing it re-arms the one-shot report.
+export function resetSamplingConfigInvalid() {
+  samplingConfigInvalidSeen = false;
+}
+
+export function resolveSampling(config = null) {
+  const node = isPlainObject(config) ? config.observability : undefined;
+  if (!isPlainObject(node)) return null;
+  const sampling = node.sampling;
+  if (sampling === undefined || sampling === null) return null; // absent: off
+
+  const invalid = () => {
+    samplingConfigInvalidSeen = true;
+    return null;
+  };
+
+  if (!isPlainObject(sampling)) return invalid();
+  const { policy_version, rates } = sampling;
+  if (
+    typeof policy_version !== 'string'
+    || policy_version.length === 0
+    || policy_version.length > MAX_POLICY_VERSION_CHARS
+  ) {
+    return invalid();
+  }
+  if (!isPlainObject(rates)) return invalid();
+  const out = {};
+  for (const [key, value] of Object.entries(rates)) {
+    if (!SAMPLING_RATE_KEYS.has(key)) return invalid();
+    if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) {
+      return invalid();
+    }
+    out[key] = value;
+  }
+  for (const level of SAMPLING_RATE_KEYS) {
+    if (out[level] === undefined) out[level] = 1;
+  }
+  return { policy_version, rates: out };
+}