@graphorin/observability 0.6.0 → 0.7.0

package/src/redaction/types.ts

@@ -0,0 +1,116 @@
+/**
+ * Public types for the {@link RedactionValidator} surface.
+ *
+ * @packageDocumentation
+ */
+
+import type { RedactionInput, RedactionOutput, RedactionValidator } from '@graphorin/core';
+
+import type { RedactionPattern } from './patterns.js';
+
+export type { RedactionInput, RedactionOutput, RedactionValidator } from '@graphorin/core';
+
+/**
+ * Optional sink invoked every time the validator drops a value or
+ * masks a pattern. Useful for emitting custom metrics, audit entries,
+ * or alert hooks. The callback receives only sanitized data - secret
+ * values are never forwarded.
+ *
+ * @stable
+ */
+export type RedactionViolationCallback = (violation: RedactionViolation) => void;
+
+/**
+ * Sanitized record describing a single redaction event. Never carries
+ * the secret value itself; only metadata that is safe to log.
+ *
+ * @stable
+ */
+export interface RedactionViolation {
+  readonly reason:
+    | 'sensitivity-tier-exceeded'
+    | 'pii-pattern-match'
+    | 'secret-pattern-match'
+    | 'unredacted-secret-value'
+    | 'invalid-input';
+  readonly attribute?: string;
+  readonly spanType?: string;
+  readonly origin?: string;
+  /** Pattern names that fired (if applicable). */
+  readonly patterns?: ReadonlyArray<string>;
+  /** Tier declared by the upstream caller. */
+  readonly declaredTier?: RedactionInput['tier'];
+}
+
+/**
+ * Counter exposed via {@link createRedactionValidator}. Implementations
+ * keep counters in-memory; downstream code can scrape them and convert
+ * to Prometheus metrics.
+ *
+ * @stable
+ */
+export interface RedactionCounters {
+  /** Total values dropped by the validator. */
+  droppedTotal: number;
+  /** Drops by reason; the four built-in reasons + any custom ones. */
+  droppedByReason: Readonly<Record<string, number>>;
+  /** Pattern-matching counters keyed by pattern name. */
+  matchesByPattern: Readonly<Record<string, number>>;
+}
+
+/**
+ * Configuration shape for {@link createRedactionValidator}.
+ *
+ * @stable
+ */
+export interface RedactionValidatorOptions {
+  /** Identifier reported via `validator.id`. Defaults to `'default'`. */
+  readonly id?: string;
+  /** Lowest tier that may pass through the validator. Default: `'public'`. */
+  readonly minTier?: RedactionInput['tier'];
+  /**
+   * When `true`, throw a {@link RedactionValidationError} on any drop.
+   * Useful in tests; production should keep this off so the validator
+   * silently drops + counts.
+   */
+  readonly failOnUnredactedSensitive?: boolean;
+  /**
+   * Pattern catalogue. Defaults to the 14 built-in default-on
+   * patterns. Custom patterns can extend or replace this list.
+   */
+  readonly patterns?: ReadonlyArray<RedactionPattern>;
+  /**
+   * Per-name allow-list. When provided, only patterns whose `name`
+   * appears here are evaluated. Empty array disables all pattern
+   * matching (tier filtering still applies).
+   */
+  readonly enabledPatterns?: ReadonlyArray<string>;
+  /**
+   * Per-name deny-list. Patterns listed here are skipped entirely.
+   * Applied **after** `enabledPatterns`.
+   */
+  readonly disabledPatterns?: ReadonlyArray<string>;
+  /**
+   * Optional sink invoked on every violation. Receives only sanitized
+   * data; never receives secret values.
+   */
+  readonly onViolation?: RedactionViolationCallback;
+  /**
+   * Optional pluggable additional check, fired after the pattern
+   * scan succeeds. The callback returns `null` to drop the value or a
+   * sanitized {@link RedactionOutput} to forward.
+   */
+  readonly customValidator?: (input: RedactionInput) => RedactionOutput | null;
+}
+
+/**
+ * Concrete validator returned by {@link createRedactionValidator}.
+ *
+ * @stable
+ */
+export interface RedactionValidatorInstance extends RedactionValidator {
+  /** Snapshot of internal counters. Returned object is a fresh copy. */
+  readonly counters: () => RedactionCounters;
+  /** Reset all counters back to zero. */
+  readonly resetCounters: () => void;
+}

package/src/redaction/validator.ts

@@ -0,0 +1,302 @@
+/**
+ * `createRedactionValidator(...)` - the building block for the
+ * mandatory `withValidation()` wrapper applied to every exporter.
+ *
+ * @packageDocumentation
+ */
+
+import { SENSITIVITY_ORDER, type Sensitivity } from '@graphorin/core';
+
+import { RedactionValidationError } from './errors.js';
+import { BUILT_IN_PATTERNS, type RedactionPattern } from './patterns.js';
+import type {
+  RedactionCounters,
+  RedactionInput,
+  RedactionOutput,
+  RedactionValidator,
+  RedactionValidatorInstance,
+  RedactionValidatorOptions,
+  RedactionViolation,
+} from './types.js';
+
+const TIER_INDEX = new Map<Sensitivity, number>(
+  SENSITIVITY_ORDER.map((tier, idx) => [tier, idx] as const),
+);
+
+function rank(tier: Sensitivity): number {
+  const idx = TIER_INDEX.get(tier);
+  return idx ?? 0;
+}
+
+interface CountersInternal {
+  droppedTotal: number;
+  droppedByReason: Map<string, number>;
+  matchesByPattern: Map<string, number>;
+}
+
+function makeCounters(): CountersInternal {
+  return {
+    droppedTotal: 0,
+    droppedByReason: new Map(),
+    matchesByPattern: new Map(),
+  };
+}
+
+function bump(map: Map<string, number>, key: string): void {
+  map.set(key, (map.get(key) ?? 0) + 1);
+}
+
+function freezeCounters(c: CountersInternal): RedactionCounters {
+  return {
+    droppedTotal: c.droppedTotal,
+    droppedByReason: Object.freeze(Object.fromEntries(c.droppedByReason)),
+    matchesByPattern: Object.freeze(Object.fromEntries(c.matchesByPattern)),
+  };
+}
+
+function selectPatterns(
+  patterns: ReadonlyArray<RedactionPattern>,
+  enabled?: ReadonlyArray<string>,
+  disabled?: ReadonlyArray<string>,
+): RedactionPattern[] {
+  const enabledSet = enabled === undefined ? null : new Set(enabled);
+  const disabledSet = new Set(disabled ?? []);
+  const out: RedactionPattern[] = [];
+  for (const p of patterns) {
+    if (enabledSet !== null && !enabledSet.has(p.name)) continue;
+    if (disabledSet.has(p.name)) continue;
+    out.push(p);
+  }
+  return out;
+}
+
+/**
+ * Walk every string discovered in `value` and apply `fn`. Returns
+ * `{ output, matched }` where `output` is the rewritten payload and
+ * `matched` is the de-duplicated list of pattern names that fired.
+ *
+ * The walker handles strings, plain objects, and arrays. Other JS
+ * primitives are passed through untouched. Cycles are detected via a
+ * `WeakSet` and broken with `'[Circular]'`.
+ */
+function walk(
+  value: unknown,
+  fn: (s: string, matched: Set<string>) => string,
+  matched: Set<string>,
+  seen: WeakSet<object>,
+): unknown {
+  if (typeof value === 'string') return fn(value, matched);
+  if (value === null || typeof value !== 'object') return value;
+  if (seen.has(value as object)) return '[Circular]';
+  seen.add(value as object);
+
+  if (Array.isArray(value)) {
+    const out = new Array<unknown>(value.length);
+    for (let i = 0; i < value.length; i++) {
+      out[i] = walk(value[i], fn, matched, seen);
+    }
+    return out;
+  }
+
+  const obj = value as Record<string, unknown>;
+  const out: Record<string, unknown> = {};
+  for (const key of Object.keys(obj)) {
+    out[key] = walk(obj[key], fn, matched, seen);
+  }
+  return out;
+}
+
+/**
+ * Apply every active pattern to `s`. Mutates `matched` with the names
+ * that fired and returns the rewritten string.
+ */
+function applyPatterns(
+  s: string,
+  patterns: ReadonlyArray<RedactionPattern>,
+  matched: Set<string>,
+): string {
+  let out = s;
+  for (const p of patterns) {
+    const mask = p.mask ?? `[REDACTED ${p.name}]`;
+    p.regex.lastIndex = 0;
+    if (p.verify === undefined) {
+      if (p.regex.test(out)) {
+        matched.add(p.name);
+        p.regex.lastIndex = 0;
+        out = out.replace(p.regex, mask);
+      }
+      continue;
+    }
+    // RP-21: per-match predicate - only mask hits the verifier accepts (e.g.
+    // Luhn-valid PANs), and only count the pattern as matched when one did.
+    const verify = p.verify;
+    p.regex.lastIndex = 0;
+    out = out.replace(p.regex, (m) => {
+      if (verify(m)) {
+        matched.add(p.name);
+        return mask;
+      }
+      return m;
+    });
+  }
+  return out;
+}
+
+function violationFor(
+  reason: RedactionViolation['reason'],
+  input: RedactionInput,
+  patterns?: ReadonlyArray<string>,
+): RedactionViolation {
+  const baseContext = input.context;
+  const base: RedactionViolation = {
+    reason,
+    declaredTier: input.tier,
+    ...(baseContext?.attribute === undefined ? {} : { attribute: baseContext.attribute }),
+    ...(baseContext?.spanType === undefined ? {} : { spanType: baseContext.spanType }),
+    ...(baseContext?.origin === undefined ? {} : { origin: baseContext.origin }),
+    ...(patterns === undefined || patterns.length === 0 ? {} : { patterns }),
+  };
+  return base;
+}
+
+/**
+ * Create a {@link RedactionValidator} configured against the supplied
+ * options. The result implements both the `RedactionValidator`
+ * contract from `@graphorin/core` and the
+ * {@link RedactionValidatorInstance} extension surface (counters +
+ * reset).
+ *
+ * @stable
+ */
+export function createRedactionValidator(
+  opts: RedactionValidatorOptions = {},
+): RedactionValidatorInstance {
+  const id = opts.id ?? 'default';
+  const minTier: Sensitivity = opts.minTier ?? 'public';
+  const failOnUnredacted = opts.failOnUnredactedSensitive === true;
+  const patterns = selectPatterns(
+    opts.patterns ?? BUILT_IN_PATTERNS,
+    opts.enabledPatterns,
+    opts.disabledPatterns,
+  );
+  const onViolation = opts.onViolation;
+  const custom = opts.customValidator;
+
+  let counters = makeCounters();
+
+  function recordViolation(violation: RedactionViolation): void {
+    counters.droppedTotal += 1;
+    bump(counters.droppedByReason, violation.reason);
+    for (const name of violation.patterns ?? []) {
+      bump(counters.matchesByPattern, name);
+    }
+    if (onViolation !== undefined) {
+      try {
+        onViolation(violation);
+      } catch {
+        // Listener errors must never break the exporter.
+      }
+    }
+  }
+
+  function dropOrThrow(_input: RedactionInput, violation: RedactionViolation): null {
+    recordViolation(violation);
+    if (failOnUnredacted) {
+      throw new RedactionValidationError(
+        `RedactionValidator dropped value (${violation.reason}) - set ` +
+          'failOnUnredactedSensitive: false in production to keep the ' +
+          'pipeline running.',
+        violation,
+      );
+    }
+    return null;
+  }
+
+  const validator: RedactionValidator = {
+    id,
+    minTier,
+    validate(input: RedactionInput): RedactionOutput | null {
+      if (input === null || input === undefined || typeof input !== 'object') {
+        return dropOrThrow(
+          input as RedactionInput,
+          violationFor('invalid-input', input as RedactionInput),
+        );
+      }
+
+      const declared = input.tier ?? 'internal';
+
+      if (rank(declared) > rank(minTier)) {
+        return dropOrThrow(input, violationFor('sensitivity-tier-exceeded', input));
+      }
+
+      const matched = new Set<string>();
+      const rewritten = walk(
+        input.value,
+        (s, m) => applyPatterns(s, patterns, m),
+        matched,
+        new WeakSet(),
+      );
+
+      if (matched.size > 0) {
+        const matchedList = [...matched];
+        for (const name of matchedList) {
+          bump(counters.matchesByPattern, name);
+        }
+        const containsSecret = matchedList.some(
+          (name) => patterns.find((p) => p.name === name)?.category === 'secret',
+        );
+
+        if (containsSecret) {
+          if (failOnUnredacted) {
+            const violation = violationFor('secret-pattern-match', input, matchedList);
+            counters.droppedTotal += 1;
+            bump(counters.droppedByReason, violation.reason);
+            if (onViolation !== undefined) {
+              try {
+                onViolation(violation);
+              } catch {
+                /* listener errors swallowed */
+              }
+            }
+            throw new RedactionValidationError(
+              `RedactionValidator detected secret pattern(s) in attribute (${matchedList.join(', ')}).`,
+              violation,
+            );
+          }
+        }
+      }
+
+      const out: RedactionOutput =
+        matched.size === 0
+          ? { value: rewritten, tier: declared }
+          : { value: rewritten, tier: declared, matched: [...matched] };
+
+      if (custom !== undefined) {
+        return custom({ ...input, value: rewritten });
+      }
+
+      return out;
+    },
+  };
+
+  const instance: RedactionValidatorInstance = {
+    ...validator,
+    counters: () => freezeCounters(counters),
+    resetCounters: () => {
+      counters = makeCounters();
+    },
+  };
+
+  return instance;
+}
+
+/**
+ * Quickly compute the relative ordering of two sensitivity tiers.
+ * Exposed because the tracer + replay layers need it without taking a
+ * full dependency on the validator implementation.
+ *
+ * @stable
+ */
+export function compareSensitivityTiers(a: Sensitivity, b: Sensitivity): number {
+  return rank(a) - rank(b);
+}

package/src/replay/config.ts

@@ -0,0 +1,58 @@
+/**
+ * Configuration shapes for the replay log. The shapes mirror the
+ * canonical `observability.replayLog.*` settings so consumer
+ * configuration files can use a single typed structure.
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Shape consumed by `observability.replayLog.*`.
+ *
+ * @stable
+ */
+export interface ReplayLogConfig {
+  /**
+   * Root directory for the JSONL trace files. Required when the
+   * replay log is enabled.
+   */
+  readonly path: string;
+  /**
+   * Retention window in days. `0` keeps every file forever.
+   *
+   * @default 30
+   */
+  readonly retentionDays?: number;
+  /**
+   * Auto-prune hint. `enabled` + `schedule` describe a daily prune of files
+   * older than `retentionDays`, but **no built-in scheduler consumes this
+   * yet** (RP-19) - it is a declarative intent. Until a host wires it to a
+   * trigger, callers must run `pruneTraces(...)` themselves; the default is
+   * therefore `enabled: false` so the option is not an inert default-on.
+   *
+   * @default { enabled: false, schedule: '0 4 * * *' }
+   */
+  readonly autoPrune?: {
+    readonly enabled: boolean;
+    readonly schedule?: string;
+  };
+  /**
+   * Encryption-at-rest toggle. `'off'` (default) writes plain JSONL;
+   * the opt-in `'aes256gcm'` mode hooks into the encryption-at-rest
+   * passphrase chain (Phase 16 deliverable).
+   *
+   * @default 'off'
+   */
+  readonly encryption?: 'off' | 'aes256gcm';
+}
+
+/**
+ * Default values for the replay log configuration.
+ *
+ * @stable
+ */
+export const DEFAULT_REPLAY_LOG_CONFIG: Omit<Required<ReplayLogConfig>, 'path'> = Object.freeze({
+  retentionDays: 30,
+  autoPrune: Object.freeze({ enabled: false, schedule: '0 4 * * *' }),
+  encryption: 'off' as const,
+});

package/src/replay/index.ts

@@ -0,0 +1,17 @@
+/**
+ * Sanitized-by-default replay surface.
+ *
+ * @packageDocumentation
+ */
+
+export { DEFAULT_REPLAY_LOG_CONFIG, type ReplayLogConfig } from './config.js';
+export { getTraceLog, type PruneTracesOptions, pruneTraces } from './log.js';
+export { createReplay, type Replay } from './replay.js';
+export type {
+  ReplayAuditBridge,
+  ReplayAuditEvent,
+  ReplayEvent,
+  ReplayMode,
+  ReplayOptions,
+  ReplayRunInput,
+} from './types.js';

package/src/replay/log.ts

@@ -0,0 +1,89 @@
+/**
+ * `getTraceLog(...)` / `pruneTraces(...)` - minimal helpers for reading
+ * back the JSONL files produced by {@link createJSONLExporter} and
+ * pruning old files based on retention policy.
+ *
+ * @packageDocumentation
+ */
+
+import { readdir, readFile, stat, unlink } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+
+import type { SpanRecord } from '../exporters/types.js';
+
+/**
+ * Read every span record from a JSONL trace log. Lines that fail to parse are
+ * skipped (the iterator keeps going); the generator only ever yields parsed
+ * {@link SpanRecord} values, never `null`.
+ *
+ * @stable
+ */
+export async function* getTraceLog(filePath: string): AsyncIterable<SpanRecord> {
+  const data = await readFile(filePath, 'utf8');
+  for (const line of data.split('\n')) {
+    if (line.trim() === '') continue;
+    try {
+      const parsed = JSON.parse(line) as SpanRecord;
+      yield parsed;
+    } catch {
+      // Skip malformed lines - replay must keep going.
+    }
+  }
+}
+
+/**
+ * Configuration shape for {@link pruneTraces}.
+ *
+ * @stable
+ */
+export interface PruneTracesOptions {
+  /** Root directory housing the JSONL files. */
+  readonly root: string;
+  /** Files older than `olderThanDays` are deleted. `0` keeps every file. */
+  readonly olderThanDays: number;
+  /**
+   * Wall clock used to compute the threshold. Defaults to `Date.now`.
+   *
+   * @internal
+   */
+  readonly now?: () => number;
+}
+
+/**
+ * Remove every JSONL file that is older than the configured retention
+ * window. Returns the deleted files for caller-side accounting.
+ *
+ * @stable
+ */
+export async function pruneTraces(opts: PruneTracesOptions): Promise<ReadonlyArray<string>> {
+  if (opts.olderThanDays <= 0) return [];
+
+  const now = opts.now ?? (() => Date.now());
+  const threshold = now() - opts.olderThanDays * 24 * 60 * 60 * 1000;
+  const root = resolve(opts.root);
+  const removed: string[] = [];
+
+  for await (const filePath of walkJsonl(root)) {
+    const info = await stat(filePath);
+    if (info.mtimeMs < threshold) {
+      await unlink(filePath);
+      removed.push(filePath);
+    }
+  }
+
+  return removed;
+}
+
+async function* walkJsonl(root: string): AsyncIterable<string> {
+  const entries = await readdir(root, { withFileTypes: true });
+  for (const entry of entries) {
+    const full = join(root, entry.name);
+    if (entry.isDirectory()) {
+      yield* walkJsonl(full);
+      continue;
+    }
+    if (entry.isFile() && full.endsWith('.jsonl')) {
+      yield full;
+    }
+  }
+}