codeharness 0.21.1 → 0.22.1

package/dist/modules/observability/index.d.ts

@@ -96,6 +96,8 @@ interface StaticCoverageState {
     readonly lastScanTimestamp: string;
     /** Historical coverage entries */
     readonly history: readonly CoverageHistoryEntry[];
+    /** Cached observability gaps from last analysis run (persisted by saveCoverageResult) */
+    readonly gaps?: readonly ObservabilityGap[];
 }
 /** Coverage targets configuration */
 interface CoverageTargets {
@@ -153,6 +155,17 @@ interface RuntimeCoverageState {
     /** Whether telemetry was detected at all */
     readonly telemetryDetected: boolean;
 }
+/** Result of the observability coverage gate check */
+interface ObservabilityCoverageGateResult {
+    /** Whether the gate passed (all required checks met their targets) */
+    readonly passed: boolean;
+    /** Static coverage check result */
+    readonly staticResult: CoverageTargetResult;
+    /** Runtime coverage check result, or null if no runtime data exists */
+    readonly runtimeResult: CoverageTargetResult | null;
+    /** List of observability gaps for actionable feedback */
+    readonly gapSummary: ObservabilityGap[];
+}
 /** Trend comparison between latest and previous coverage entries */
 interface CoverageTrend {
     /** Latest coverage percentage */
@@ -177,6 +190,50 @@ interface CoverageTargetResult {
     /** Gap: target - current (0 if met) */
     readonly gap: number;
 }
+/** Configuration for standalone runtime validation (audit mode) */
+interface RuntimeValidationConfig {
+    /** Test command to run (default: 'npm test') */
+    readonly testCommand: string;
+    /** OTLP endpoint for telemetry export */
+    readonly otlpEndpoint: string;
+    /** Query endpoint for the observability backend (VictoriaLogs) */
+    readonly queryEndpoint: string;
+    /** Timeout for the entire validation process in milliseconds (default: 120000) */
+    readonly timeoutMs: number;
+}
+/** A single module's telemetry status from runtime validation */
+interface ModuleTelemetryEntry {
+    /** Module name (derived from source directory) */
+    readonly moduleName: string;
+    /** Whether telemetry was detected for this module */
+    readonly telemetryDetected: boolean;
+    /** Number of telemetry events emitted by this module */
+    readonly eventCount: number;
+}
+/** Result of a standalone runtime validation run */
+interface RuntimeValidationResult {
+    /** Per-module telemetry entries */
+    readonly entries: readonly ModuleTelemetryEntry[];
+    /** Total number of modules in the project */
+    readonly totalModules: number;
+    /** Number of modules that emitted telemetry */
+    readonly modulesWithTelemetry: number;
+    /** Runtime coverage percentage: modulesWithTelemetry / totalModules * 100 */
+    readonly coveragePercent: number;
+    /** Whether validation was skipped (e.g., backend unreachable) */
+    readonly skipped: boolean;
+    /** Reason validation was skipped, if applicable */
+    readonly skipReason?: string;
+}
+/** A telemetry event returned from the observability backend query */
+interface TelemetryEvent {
+    /** ISO 8601 timestamp of the event */
+    readonly timestamp: string;
+    /** Log message or event description */
+    readonly message: string;
+    /** Source file or service attribute */
+    readonly source: string;
+}
 
 /**
  * Discriminated union Result type for consistent error handling.
@@ -214,38 +271,15 @@ type Result<T> = Ok<T> | Fail;
  */
 declare function analyze(projectDir: string, config?: AnalyzerConfig): Result<AnalyzerResult>;
 
-/**
- * Coverage state persistence — tracks observability coverage in sprint-state.json.
- *
- * Implements Story 1.3: read/write/trend/target comparison for static analysis coverage.
- * Only the `static` section is managed here; `runtime` is deferred to Epic 2.
- */
+/** Coverage state persistence — tracks observability coverage in sprint-state.json. */
 
-/**
- * Save coverage result from an analysis run into sprint-state.json.
- *
- * Updates `observability.static.coveragePercent`, `observability.static.lastScanTimestamp`,
- * and appends to `observability.static.history`. Preserves all other state fields.
- */
+/** Save coverage result from an analysis run into sprint-state.json. */
 declare function saveCoverageResult(projectDir: string, result: AnalyzerResult): Result<void>;
-/**
- * Read the observability coverage state from sprint-state.json.
- *
- * Returns the typed `ObservabilityCoverageState` from the `observability` key.
- * If no observability section exists, returns default state.
- */
+/** Read the observability coverage state from sprint-state.json. */
 declare function readCoverageState(projectDir: string): Result<ObservabilityCoverageState>;
-/**
- * Get coverage trend by comparing latest vs previous history entries.
- */
+/** Get coverage trend by comparing latest vs previous history entries. */
 declare function getCoverageTrend(projectDir: string): Result<CoverageTrend>;
-/**
- * Check current coverage against a target threshold.
- *
- * @param projectDir - Project root directory
- * @param target - Target percentage (default 80%)
- * @returns Structured result indicating whether coverage meets the target
- */
+/** Check current coverage against a target threshold. */
 declare function checkCoverageTarget(projectDir: string, target?: number): Result<CoverageTargetResult>;
 
 /** Per-AC observability gap presence */
@@ -292,4 +326,42 @@ declare function computeRuntimeCoverage(gapResults: ObservabilityGapResult): Run
  */
 declare function saveRuntimeCoverage(projectDir: string, result: RuntimeCoverageResult): Result<void>;
 
-export { type AnalyzerConfig, type AnalyzerResult, type AnalyzerSummary, type CoverageHistoryEntry, type CoverageTargetResult, type CoverageTargets, type CoverageTrend, type GapSeverity, type ObservabilityCoverageState, type ObservabilityGap, type RuntimeCoverageEntry, type RuntimeCoverageResult, type RuntimeCoverageState, type StaticCoverageState, analyze, checkCoverageTarget, computeRuntimeCoverage, getCoverageTrend, readCoverageState, saveCoverageResult, saveRuntimeCoverage };
+/**
+ * Observability coverage gate — checks if coverage meets targets for commit gating.
+ *
+ * Implements Story 2.2: reads cached coverage from sprint-state.json and compares
+ * against targets. Does NOT re-run Semgrep — uses cached results for performance.
+ * Static coverage is always required; runtime is only checked when data exists.
+ */
+
+/**
+ * Check observability coverage against targets for commit gating.
+ *
+ * Reads static and runtime coverage from sprint-state.json. Static coverage
+ * is always checked. Runtime coverage is only checked when runtime data exists
+ * in the state file (i.e., after at least one verification run).
+ *
+ * @param projectDir - Project root directory
+ * @param overrides - Optional target overrides
+ * @returns Gate result with pass/fail, per-dimension results, and gap summary
+ */
+declare function checkObservabilityCoverageGate(projectDir: string, overrides?: {
+    staticTarget?: number;
+    runtimeTarget?: number;
+}): Result<ObservabilityCoverageGateResult>;
+
+/**
+ * Standalone runtime validator — validates telemetry outside Docker verification.
+ * Implements Story 2.3: run tests with OTLP, query backend, report module coverage.
+ */
+
+/** Run tests with OTLP enabled, query backend for events, compute module coverage. */
+declare function validateRuntime(projectDir: string, config?: Partial<RuntimeValidationConfig>): Promise<Result<RuntimeValidationResult>>;
+/** Check if the observability backend is reachable (2xx within 3s). */
+declare function checkBackendHealth(queryEndpoint: string): Promise<boolean>;
+/** Query VictoriaLogs for telemetry events within a time window. */
+declare function queryTelemetryEvents(queryEndpoint: string, startTime: string, endTime: string): Promise<Result<TelemetryEvent[]>>;
+/** Map telemetry events to project modules by matching source paths. */
+declare function mapEventsToModules(events: TelemetryEvent[], projectDir: string, modules?: string[]): ModuleTelemetryEntry[];
+
+export { type AnalyzerConfig, type AnalyzerResult, type AnalyzerSummary, type CoverageHistoryEntry, type CoverageTargetResult, type CoverageTargets, type CoverageTrend, type GapSeverity, type ModuleTelemetryEntry, type ObservabilityCoverageGateResult, type ObservabilityCoverageState, type ObservabilityGap, type RuntimeCoverageEntry, type RuntimeCoverageResult, type RuntimeCoverageState, type RuntimeValidationConfig, type RuntimeValidationResult, type StaticCoverageState, type TelemetryEvent, analyze, checkBackendHealth, checkCoverageTarget, checkObservabilityCoverageGate, computeRuntimeCoverage, getCoverageTrend, mapEventsToModules, queryTelemetryEvents, readCoverageState, saveCoverageResult, saveRuntimeCoverage, validateRuntime };

package/dist/modules/observability/index.js

@@ -195,11 +195,13 @@ function saveCoverageResult(projectDir, result) {
   const updatedStatic = {
     coveragePercent: result.summary.coveragePercent,
     lastScanTimestamp: timestamp,
-    history: trimmedHistory
+    history: trimmedHistory,
+    gaps: result.gaps.map(({ file, line, type, description, severity }) => ({ file, line, type, description, severity }))
   };
   const updatedObservability = {
     static: updatedStatic,
-    targets: existing.targets
+    targets: existing.targets,
+    ...existing.runtime ? { runtime: existing.runtime } : {}
   };
   const updatedState = {
     ...state,
@@ -275,18 +277,45 @@ function extractCoverageState(state) {
   }
   const staticSection = obs.static;
   const targets = obs.targets;
-  return {
+  const runtimeSection = obs.runtime;
+  const runtime = runtimeSection && typeof runtimeSection.coveragePercent === "number" ? {
+    coveragePercent: runtimeSection.coveragePercent,
+    lastValidationTimestamp: typeof runtimeSection.lastValidationTimestamp === "string" ? runtimeSection.lastValidationTimestamp : "",
+    modulesWithTelemetry: typeof runtimeSection.modulesWithTelemetry === "number" ? runtimeSection.modulesWithTelemetry : 0,
+    totalModules: typeof runtimeSection.totalModules === "number" ? runtimeSection.totalModules : 0,
+    telemetryDetected: typeof runtimeSection.telemetryDetected === "boolean" ? runtimeSection.telemetryDetected : false
+  } : void 0;
+  const parsedGaps = parseGapArray(staticSection?.gaps);
+  const result = {
     static: {
       coveragePercent: typeof staticSection?.coveragePercent === "number" ? staticSection.coveragePercent : 0,
       lastScanTimestamp: typeof staticSection?.lastScanTimestamp === "string" ? staticSection.lastScanTimestamp : "",
       history: Array.isArray(staticSection?.history) ? staticSection.history.filter(
         (entry) => typeof entry === "object" && entry !== null && typeof entry.coveragePercent === "number" && typeof entry.timestamp === "string"
-      ) : []
+      ) : [],
+      ...parsedGaps.length > 0 ? { gaps: parsedGaps } : {}
     },
     targets: {
-      staticTarget: typeof targets?.staticTarget === "number" ? targets.staticTarget : DEFAULT_STATIC_TARGET
-    }
+      staticTarget: typeof targets?.staticTarget === "number" ? targets.staticTarget : DEFAULT_STATIC_TARGET,
+      ...typeof targets?.runtimeTarget === "number" ? { runtimeTarget: targets.runtimeTarget } : {}
+    },
+    ...runtime ? { runtime } : {}
   };
+  return result;
+}
+function parseGapArray(raw) {
+  if (!Array.isArray(raw)) return [];
+  return raw.filter((g) => {
+    if (typeof g !== "object" || g === null) return false;
+    const r = g;
+    return typeof r.file === "string" && typeof r.line === "number" && typeof r.type === "string" && typeof r.description === "string";
+  }).map((g) => ({
+    file: g.file,
+    line: g.line,
+    type: g.type,
+    description: g.description,
+    severity: g.severity === "error" || g.severity === "warning" ? g.severity : "info"
+  }));
 }
 
 // src/modules/observability/runtime-coverage.ts
@@ -355,12 +384,216 @@ function saveRuntimeCoverage(projectDir, result) {
     return fail(`Failed to write ${STATE_FILE2}: ${msg}`);
   }
 }
+
+// src/modules/observability/coverage-gate.ts
+var DEFAULT_STATIC_TARGET2 = 80;
+var DEFAULT_RUNTIME_TARGET = 60;
+function checkObservabilityCoverageGate(projectDir, overrides) {
+  if (!projectDir || typeof projectDir !== "string") {
+    return fail("projectDir is required and must be a non-empty string");
+  }
+  const stateResult = readCoverageState(projectDir);
+  if (!stateResult.success) {
+    return fail(stateResult.error);
+  }
+  const state = stateResult.data;
+  const staticTarget = overrides?.staticTarget ?? state.targets.staticTarget ?? DEFAULT_STATIC_TARGET2;
+  const runtimeTarget = overrides?.runtimeTarget ?? state.targets.runtimeTarget ?? DEFAULT_RUNTIME_TARGET;
+  const staticCurrent = state.static.coveragePercent;
+  const staticMet = staticCurrent >= staticTarget;
+  const staticResult = {
+    met: staticMet,
+    current: staticCurrent,
+    target: staticTarget,
+    gap: staticMet ? 0 : staticTarget - staticCurrent
+  };
+  let runtimeResult = null;
+  if (state.runtime) {
+    const runtimeCurrent = state.runtime.coveragePercent;
+    const runtimeMet = runtimeCurrent >= runtimeTarget;
+    runtimeResult = {
+      met: runtimeMet,
+      current: runtimeCurrent,
+      target: runtimeTarget,
+      gap: runtimeMet ? 0 : runtimeTarget - runtimeCurrent
+    };
+  }
+  const passed = staticResult.met && (runtimeResult === null || runtimeResult.met);
+  const gapSummary = state.static.gaps ? [...state.static.gaps] : [];
+  return ok({ passed, staticResult, runtimeResult, gapSummary });
+}
+
+// src/modules/observability/runtime-validator.ts
+import { execSync } from "child_process";
+import { readdirSync, statSync } from "fs";
+import { join as join4 } from "path";
+var DEFAULT_CONFIG = {
+  testCommand: "npm test",
+  otlpEndpoint: "http://localhost:4318",
+  queryEndpoint: "http://localhost:9428",
+  timeoutMs: 12e4
+};
+var HEALTH_TIMEOUT_MS = 3e3;
+var QUERY_TIMEOUT_MS = 3e4;
+async function validateRuntime(projectDir, config) {
+  if (!projectDir || typeof projectDir !== "string") {
+    return fail("projectDir is required and must be a non-empty string");
+  }
+  const cfg = { ...DEFAULT_CONFIG, ...config };
+  if (/[;&|`$(){}!#]/.test(cfg.testCommand)) {
+    return fail(`testCommand contains disallowed shell metacharacters: ${cfg.testCommand}`);
+  }
+  const healthy = await checkBackendHealth(cfg.queryEndpoint);
+  if (!healthy) {
+    const modules2 = discoverModules(projectDir);
+    const entries2 = modules2.map((m) => ({
+      moduleName: m,
+      telemetryDetected: false,
+      eventCount: 0
+    }));
+    return ok({
+      entries: entries2,
+      totalModules: modules2.length,
+      modulesWithTelemetry: 0,
+      coveragePercent: 0,
+      skipped: true,
+      skipReason: "runtime validation skipped -- observability stack not available"
+    });
+  }
+  const startTime = (/* @__PURE__ */ new Date()).toISOString();
+  try {
+    execSync(cfg.testCommand, {
+      cwd: projectDir,
+      timeout: cfg.timeoutMs,
+      env: {
+        ...process.env,
+        OTEL_EXPORTER_OTLP_ENDPOINT: cfg.otlpEndpoint
+      },
+      stdio: "pipe"
+    });
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return fail(`Test command failed: ${msg}`);
+  }
+  const endTime = (/* @__PURE__ */ new Date()).toISOString();
+  const eventsResult = await queryTelemetryEvents(cfg.queryEndpoint, startTime, endTime);
+  if (!eventsResult.success) {
+    return fail(eventsResult.error);
+  }
+  const modules = discoverModules(projectDir);
+  const entries = mapEventsToModules(eventsResult.data, projectDir, modules);
+  const modulesWithTelemetry = entries.filter((e) => e.telemetryDetected).length;
+  const totalModules = entries.length;
+  const coveragePercent = totalModules === 0 ? 0 : modulesWithTelemetry / totalModules * 100;
+  return ok({
+    entries,
+    totalModules,
+    modulesWithTelemetry,
+    coveragePercent,
+    skipped: false
+  });
+}
+async function checkBackendHealth(queryEndpoint) {
+  try {
+    const response = await fetch(`${queryEndpoint}/health`, {
+      signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS)
+    });
+    return response.ok;
+  } catch {
+    return false;
+  }
+}
+async function queryTelemetryEvents(queryEndpoint, startTime, endTime) {
+  let url;
+  try {
+    url = new URL("/select/logsql/query", queryEndpoint);
+  } catch {
+    return fail(`Invalid queryEndpoint URL: ${queryEndpoint}`);
+  }
+  url.searchParams.set("query", "*");
+  url.searchParams.set("start", startTime);
+  url.searchParams.set("end", endTime);
+  url.searchParams.set("limit", "1000");
+  try {
+    const response = await fetch(url.toString(), {
+      signal: AbortSignal.timeout(QUERY_TIMEOUT_MS)
+    });
+    if (!response.ok) {
+      return fail(`VictoriaLogs returned ${response.status}`);
+    }
+    const text = await response.text();
+    const events = parseLogEvents(text);
+    return ok(events);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return fail(`Failed to query telemetry events: ${msg}`);
+  }
+}
+function mapEventsToModules(events, projectDir, modules) {
+  void projectDir;
+  const moduleList = modules ?? [];
+  const moduleCounts = /* @__PURE__ */ new Map();
+  for (const mod of moduleList) {
+    moduleCounts.set(mod, 0);
+  }
+  for (const event of events) {
+    for (const mod of moduleList) {
+      if (event.source.includes(mod) || event.message.includes(mod)) {
+        moduleCounts.set(mod, (moduleCounts.get(mod) ?? 0) + 1);
+      }
+    }
+  }
+  return moduleList.map((mod) => {
+    const count = moduleCounts.get(mod) ?? 0;
+    return {
+      moduleName: mod,
+      telemetryDetected: count > 0,
+      eventCount: count
+    };
+  });
+}
+function discoverModules(projectDir) {
+  const srcDir = join4(projectDir, "src");
+  try {
+    return readdirSync(srcDir).filter((name) => {
+      try {
+        return statSync(join4(srcDir, name)).isDirectory();
+      } catch {
+        return false;
+      }
+    });
+  } catch {
+    return [];
+  }
+}
+function parseLogEvents(text) {
+  if (!text.trim()) return [];
+  const lines = text.trim().split("\n");
+  const events = [];
+  for (const line of lines) {
+    try {
+      const raw = JSON.parse(line);
+      events.push({
+        timestamp: String(raw._time ?? raw.timestamp ?? ""),
+        message: String(raw._msg ?? raw.message ?? ""),
+        source: String(raw.source ?? raw._source ?? raw.service ?? "")
+      });
+    } catch {
+    }
+  }
+  return events;
+}
 export {
   analyze,
+  checkBackendHealth,
   checkCoverageTarget,
+  checkObservabilityCoverageGate,
   computeRuntimeCoverage,
   getCoverageTrend,
+  mapEventsToModules,
+  queryTelemetryEvents,
   readCoverageState,
   saveCoverageResult,
-  saveRuntimeCoverage
+  saveRuntimeCoverage,
+  validateRuntime
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeharness",
-  "version": "0.21.1",
+  "version": "0.22.1",
   "type": "module",
   "description": "CLI for codeharness — makes autonomous coding agents produce software that actually works",
   "bin": {

package/ralph/ralph.sh

@@ -782,7 +782,12 @@ execute_iteration() {
         fi
 
         local has_errors="false"
-        if grep -v '"[^"]*error[^"]*":' "$output_file" 2>/dev/null | \
+        # Only check non-JSON lines for errors. Stream-json output is NDJSON
+        # (one JSON object per line), so any line starting with '{' is Claude
+        # content — which naturally contains words like "error" and "Exception"
+        # in code reviews, test output, and discussion. Grepping those produces
+        # false positives that trip the circuit breaker.
+        if grep -v '^[[:space:]]*{' "$output_file" 2>/dev/null | \
            grep -qE '(^Error:|^ERROR:|^error:|\]: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)'; then
             has_errors="true"
             log_status "WARN" "Errors detected in output"