@untangle-code/observability 0.0.0-stage → 0.1.1

package/src/profiling.ts

@@ -0,0 +1,171 @@
+/**
+ * Profiling collector for `--profile` (architecture.md §24.3): stage
+ * timings via the injected `Clock`, file counts, cache hit rates, embedding
+ * and Qdrant latency/bytes, a parser/language breakdown, and the process
+ * memory high water mark.
+ *
+ * Stage timing uses `Clock` exclusively (BUILD_PLAN determinism rule).
+ * The memory high water mark is the one place this module reads real
+ * process state (`process.memoryUsage()`), because that is exactly what
+ * architecture.md §24.3 asks for — it is not a timing or randomness
+ * decision, just a resource-usage sample.
+ */
+import type { Clock } from "@untangle-code/domain";
+
+import { type HistogramSnapshot, summarizeHistogram } from "./metrics.js";
+
+export interface StageTimingV1 {
+  stage: string;
+  startedAtIso: string;
+  endedAtIso: string;
+  durationMs: number;
+}
+
+export interface CacheHitRateV1 {
+  hits: number;
+  misses: number;
+  /** `hits / (hits + misses)`, or `null` when there were no accesses at all. */
+  rate: number | null;
+}
+
+export interface ParserBreakdownEntryV1 {
+  files: number;
+  failures: number;
+}
+
+export interface ProfileReportV1 {
+  stages: StageTimingV1[];
+  fileCounts: Record<string, number>;
+  cacheHitRates: Record<string, CacheHitRateV1>;
+  embeddingLatencyMs: HistogramSnapshot;
+  qdrantLatencyMs: HistogramSnapshot;
+  qdrantBytes: number;
+  parserBreakdown: Record<string, ParserBreakdownEntryV1>;
+  memoryHighWaterMarkBytes: number;
+}
+
+export interface StageHandle {
+  /** Ends the stage. Idempotent — a second call is a no-op. */
+  end(): void;
+}
+
+interface CacheCounters {
+  hits: number;
+  misses: number;
+}
+
+/** Reads the process's current resident set size. Isolated for testability. */
+export function sampleMemoryBytes(): number {
+  return process.memoryUsage().rss;
+}
+
+/** Collects one `--profile` run's worth of measurements. */
+export class ProfileCollector {
+  private readonly clock: Clock;
+  private readonly readMemoryBytes: () => number;
+  private readonly stages: StageTimingV1[] = [];
+  private readonly fileCounts: Record<string, number> = {};
+  private readonly cacheCounters: Record<string, CacheCounters> = {};
+  private readonly embeddingLatencies: number[] = [];
+  private readonly qdrantLatencies: number[] = [];
+  private readonly parserBreakdown: Record<string, ParserBreakdownEntryV1> = {};
+  private qdrantBytesTotal = 0;
+  private memoryHighWaterMarkBytes: number;
+
+  public constructor(deps: { clock: Clock; readMemoryBytes?: () => number }) {
+    this.clock = deps.clock;
+    this.readMemoryBytes = deps.readMemoryBytes ?? sampleMemoryBytes;
+    this.memoryHighWaterMarkBytes = this.readMemoryBytes();
+  }
+
+  /** Starts timing `stage`. Returns a handle whose `end()` records the duration. */
+  public beginStage(stage: string): StageHandle {
+    const startedAtIso = this.clock.nowIso();
+    const startedAtMs = this.clock.nowEpochMs();
+    this.sampleMemory();
+    let ended = false;
+    return {
+      end: (): void => {
+        if (ended) {
+          return;
+        }
+        ended = true;
+        const endedAtMs = this.clock.nowEpochMs();
+        this.stages.push({
+          stage,
+          startedAtIso,
+          endedAtIso: this.clock.nowIso(),
+          durationMs: endedAtMs - startedAtMs,
+        });
+        this.sampleMemory();
+      },
+    };
+  }
+
+  public recordFileCount(category: string, delta = 1): void {
+    this.fileCounts[category] = (this.fileCounts[category] ?? 0) + delta;
+  }
+
+  public recordCacheAccess(name: string, hit: boolean): void {
+    const counters = this.cacheCounters[name] ?? { hits: 0, misses: 0 };
+    if (hit) {
+      counters.hits += 1;
+    } else {
+      counters.misses += 1;
+    }
+    this.cacheCounters[name] = counters;
+  }
+
+  public recordEmbeddingLatency(ms: number): void {
+    this.embeddingLatencies.push(ms);
+    this.sampleMemory();
+  }
+
+  public recordQdrantLatency(ms: number): void {
+    this.qdrantLatencies.push(ms);
+    this.sampleMemory();
+  }
+
+  public addQdrantBytes(bytes: number): void {
+    this.qdrantBytesTotal += bytes;
+  }
+
+  public recordParserOutcome(language: string, outcome: "success" | "failure"): void {
+    const entry = this.parserBreakdown[language] ?? { files: 0, failures: 0 };
+    entry.files += 1;
+    if (outcome === "failure") {
+      entry.failures += 1;
+    }
+    this.parserBreakdown[language] = entry;
+  }
+
+  /** Re-samples the memory high water mark on demand (also sampled automatically around stages/latencies). */
+  public sampleMemory(): void {
+    const current = this.readMemoryBytes();
+    if (current > this.memoryHighWaterMarkBytes) {
+      this.memoryHighWaterMarkBytes = current;
+    }
+  }
+
+  public snapshot(): ProfileReportV1 {
+    const cacheHitRates: Record<string, CacheHitRateV1> = {};
+    for (const [name, counters] of Object.entries(this.cacheCounters)) {
+      const total = counters.hits + counters.misses;
+      cacheHitRates[name] = {
+        hits: counters.hits,
+        misses: counters.misses,
+        rate: total === 0 ? null : counters.hits / total,
+      };
+    }
+    return {
+      stages: [...this.stages],
+      fileCounts: { ...this.fileCounts },
+      cacheHitRates,
+      embeddingLatencyMs: summarizeHistogram(this.embeddingLatencies),
+      qdrantLatencyMs: summarizeHistogram(this.qdrantLatencies),
+      qdrantBytes: this.qdrantBytesTotal,
+      parserBreakdown: { ...this.parserBreakdown },
+      memoryHighWaterMarkBytes: this.memoryHighWaterMarkBytes,
+    };
+  }
+}

package/src/redact.ts

@@ -0,0 +1,144 @@
+/**
+ * Redaction (spec.md §24, §23.2): keeps secrets and full source text out of
+ * logs, status output and crash reports by default.
+ *
+ * `redact()` is a pure function — no I/O, no logging decisions of its own.
+ * The logger (`logger.ts`) applies it to every event's `context` before
+ * writing.
+ */
+
+const MASK = "[REDACTED]";
+const SOURCE_TEXT_SUPPRESSED = "[SOURCE_TEXT_SUPPRESSED]";
+
+/**
+ * Key names spec.md §24 calls out by name, plus the common synonyms and
+ * variants ("api_key", "apiKey", "auth", …) adapters realistically use.
+ * Matched case-insensitively against the whole key.
+ */
+const SECRET_KEY_PATTERN =
+  /(api[-_]?key|access[-_]?key|secret[-_]?key|private[-_]?key|token|password|passwd|pwd|secret|authorization|auth|cookie|credential)/i;
+
+/**
+ * Field names that carry full source text / code (spec.md §23.2: "vollständige
+ * Quelltexte standardmäßig redigieren"). Mirrors the content-bearing fields of
+ * domain's `CodeChunkPayloadV1`/`CodeChunkDraft` plus generic synonyms, so a
+ * logged chunk, diff or search hit does not dump raw code into logs by
+ * default. Matched case-insensitively against the whole key.
+ */
+const SOURCE_TEXT_KEYS = new Set([
+  "content",
+  "text",
+  "sourcetext",
+  "source",
+  "sourcecode",
+  "code",
+  "snippet",
+  "prepareddensetext",
+  "bm25text",
+  "diff",
+  "patch",
+  "raw",
+  "rawtext",
+]);
+
+export interface RedactOptions {
+  /**
+   * Opt-in escape hatch for interactive debugging (spec.md §24: only on
+   * explicit debug opt-in). When `false` (default), values under a
+   * {@link SOURCE_TEXT_KEYS} key are replaced instead of logged verbatim.
+   */
+  includeSourceText?: boolean;
+}
+
+/**
+ * Recursively redacts `value`: keys that look like secrets are masked
+ * regardless of their value's shape, keys that carry source text are
+ * suppressed unless `includeSourceText` is set, and string values that look
+ * like a secret themselves (bearer tokens, JWTs, common vendor key prefixes)
+ * are masked even under an innocuous key name.
+ *
+ * Never throws: unsupported value kinds (bigint, function, symbol) are
+ * stringified rather than propagating a serialization error into a logging
+ * call.
+ */
+export function redact(value: unknown, options: RedactOptions = {}): unknown {
+  return redactValue(value, options, new WeakSet<object>());
+}
+
+function redactValue(value: unknown, options: RedactOptions, seen: WeakSet<object>): unknown {
+  if (value === null || value === undefined) {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    if (seen.has(value)) {
+      return "[CIRCULAR]";
+    }
+    seen.add(value);
+    return value.map((entry) => redactValue(entry, options, seen));
+  }
+  if (value instanceof Date) {
+    return value.toISOString();
+  }
+  if (typeof value === "object") {
+    if (seen.has(value)) {
+      return "[CIRCULAR]";
+    }
+    seen.add(value);
+    const out: Record<string, unknown> = {};
+    for (const [key, entryValue] of Object.entries(value as Record<string, unknown>)) {
+      out[key] = redactEntry(key, entryValue, options, seen);
+    }
+    return out;
+  }
+  if (typeof value === "bigint") {
+    return value.toString();
+  }
+  if (typeof value === "function" || typeof value === "symbol") {
+    return `[${typeof value}]`;
+  }
+  if (typeof value === "string" && looksLikeSecretValue(value)) {
+    return MASK;
+  }
+  return value;
+}
+
+function redactEntry(
+  key: string,
+  entryValue: unknown,
+  options: RedactOptions,
+  seen: WeakSet<object>,
+): unknown {
+  const lowerKey = key.toLowerCase();
+  if (SECRET_KEY_PATTERN.test(lowerKey)) {
+    return MASK;
+  }
+  if (!options.includeSourceText && SOURCE_TEXT_KEYS.has(lowerKey)) {
+    return SOURCE_TEXT_SUPPRESSED;
+  }
+  return redactValue(entryValue, options, seen);
+}
+
+/**
+ * Heuristic for "a value that looks like a key" (spec.md §24): bearer
+ * headers, JWT-shaped triples, and common vendor secret prefixes
+ * (`sk-`, `ghp_`, `xox…`). Conservative on purpose — false negatives are
+ * expected, false positives on ordinary prose are not.
+ */
+function looksLikeSecretValue(value: string): boolean {
+  if (/^bearer\s+\S+/i.test(value)) {
+    return true;
+  }
+  if (/^[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}$/.test(value)) {
+    return true;
+  }
+  if (/^(sk|pk|rk)-[A-Za-z0-9_-]{16,}$/i.test(value)) {
+    return true;
+  }
+  if (/^(ghp|gho|ghu|ghs|github_pat)_[A-Za-z0-9_]{16,}$/i.test(value)) {
+    return true;
+  }
+  if (/^xox[baprs]-[A-Za-z0-9-]{10,}$/i.test(value)) {
+    return true;
+  }
+  return false;
+}