codeharness 0.22.0 → 0.22.2

package/dist/modules/observability/index.d.ts

@@ -190,6 +190,50 @@ interface CoverageTargetResult {
     /** Gap: target - current (0 if met) */
     readonly gap: number;
 }
+/** Configuration for standalone runtime validation (audit mode) */
+interface RuntimeValidationConfig {
+    /** Test command to run (default: 'npm test') */
+    readonly testCommand: string;
+    /** OTLP endpoint for telemetry export */
+    readonly otlpEndpoint: string;
+    /** Query endpoint for the observability backend (VictoriaLogs) */
+    readonly queryEndpoint: string;
+    /** Timeout for the entire validation process in milliseconds (default: 120000) */
+    readonly timeoutMs: number;
+}
+/** A single module's telemetry status from runtime validation */
+interface ModuleTelemetryEntry {
+    /** Module name (derived from source directory) */
+    readonly moduleName: string;
+    /** Whether telemetry was detected for this module */
+    readonly telemetryDetected: boolean;
+    /** Number of telemetry events emitted by this module */
+    readonly eventCount: number;
+}
+/** Result of a standalone runtime validation run */
+interface RuntimeValidationResult {
+    /** Per-module telemetry entries */
+    readonly entries: readonly ModuleTelemetryEntry[];
+    /** Total number of modules in the project */
+    readonly totalModules: number;
+    /** Number of modules that emitted telemetry */
+    readonly modulesWithTelemetry: number;
+    /** Runtime coverage percentage: modulesWithTelemetry / totalModules * 100 */
+    readonly coveragePercent: number;
+    /** Whether validation was skipped (e.g., backend unreachable) */
+    readonly skipped: boolean;
+    /** Reason validation was skipped, if applicable */
+    readonly skipReason?: string;
+}
+/** A telemetry event returned from the observability backend query */
+interface TelemetryEvent {
+    /** ISO 8601 timestamp of the event */
+    readonly timestamp: string;
+    /** Log message or event description */
+    readonly message: string;
+    /** Source file or service attribute */
+    readonly source: string;
+}
 
 /**
  * Discriminated union Result type for consistent error handling.
@@ -306,4 +350,18 @@ declare function checkObservabilityCoverageGate(projectDir: string, overrides?:
     runtimeTarget?: number;
 }): Result<ObservabilityCoverageGateResult>;
 
-export { type AnalyzerConfig, type AnalyzerResult, type AnalyzerSummary, type CoverageHistoryEntry, type CoverageTargetResult, type CoverageTargets, type CoverageTrend, type GapSeverity, type ObservabilityCoverageGateResult, type ObservabilityCoverageState, type ObservabilityGap, type RuntimeCoverageEntry, type RuntimeCoverageResult, type RuntimeCoverageState, type StaticCoverageState, analyze, checkCoverageTarget, checkObservabilityCoverageGate, computeRuntimeCoverage, getCoverageTrend, readCoverageState, saveCoverageResult, saveRuntimeCoverage };
+/**
+ * Standalone runtime validator — validates telemetry outside Docker verification.
+ * Implements Story 2.3: run tests with OTLP, query backend, report module coverage.
+ */
+
+/** Run tests with OTLP enabled, query backend for events, compute module coverage. */
+declare function validateRuntime(projectDir: string, config?: Partial<RuntimeValidationConfig>): Promise<Result<RuntimeValidationResult>>;
+/** Check if the observability backend is reachable (2xx within 3s). */
+declare function checkBackendHealth(queryEndpoint: string): Promise<boolean>;
+/** Query VictoriaLogs for telemetry events within a time window. */
+declare function queryTelemetryEvents(queryEndpoint: string, startTime: string, endTime: string): Promise<Result<TelemetryEvent[]>>;
+/** Map telemetry events to project modules by matching source paths. */
+declare function mapEventsToModules(events: TelemetryEvent[], projectDir: string, modules?: string[]): ModuleTelemetryEntry[];
+
+export { type AnalyzerConfig, type AnalyzerResult, type AnalyzerSummary, type CoverageHistoryEntry, type CoverageTargetResult, type CoverageTargets, type CoverageTrend, type GapSeverity, type ModuleTelemetryEntry, type ObservabilityCoverageGateResult, type ObservabilityCoverageState, type ObservabilityGap, type RuntimeCoverageEntry, type RuntimeCoverageResult, type RuntimeCoverageState, type RuntimeValidationConfig, type RuntimeValidationResult, type StaticCoverageState, type TelemetryEvent, analyze, checkBackendHealth, checkCoverageTarget, checkObservabilityCoverageGate, computeRuntimeCoverage, getCoverageTrend, mapEventsToModules, queryTelemetryEvents, readCoverageState, saveCoverageResult, saveRuntimeCoverage, validateRuntime };

package/dist/modules/observability/index.js

@@ -422,13 +422,178 @@ function checkObservabilityCoverageGate(projectDir, overrides) {
   const gapSummary = state.static.gaps ? [...state.static.gaps] : [];
   return ok({ passed, staticResult, runtimeResult, gapSummary });
 }
+
+// src/modules/observability/runtime-validator.ts
+import { execSync } from "child_process";
+import { readdirSync, statSync } from "fs";
+import { join as join4 } from "path";
+var DEFAULT_CONFIG = {
+  testCommand: "npm test",
+  otlpEndpoint: "http://localhost:4318",
+  queryEndpoint: "http://localhost:9428",
+  timeoutMs: 12e4
+};
+var HEALTH_TIMEOUT_MS = 3e3;
+var QUERY_TIMEOUT_MS = 3e4;
+async function validateRuntime(projectDir, config) {
+  if (!projectDir || typeof projectDir !== "string") {
+    return fail("projectDir is required and must be a non-empty string");
+  }
+  const cfg = { ...DEFAULT_CONFIG, ...config };
+  if (/[;&|`$(){}!#]/.test(cfg.testCommand)) {
+    return fail(`testCommand contains disallowed shell metacharacters: ${cfg.testCommand}`);
+  }
+  const healthy = await checkBackendHealth(cfg.queryEndpoint);
+  if (!healthy) {
+    const modules2 = discoverModules(projectDir);
+    const entries2 = modules2.map((m) => ({
+      moduleName: m,
+      telemetryDetected: false,
+      eventCount: 0
+    }));
+    return ok({
+      entries: entries2,
+      totalModules: modules2.length,
+      modulesWithTelemetry: 0,
+      coveragePercent: 0,
+      skipped: true,
+      skipReason: "runtime validation skipped -- observability stack not available"
+    });
+  }
+  const startTime = (/* @__PURE__ */ new Date()).toISOString();
+  try {
+    execSync(cfg.testCommand, {
+      cwd: projectDir,
+      timeout: cfg.timeoutMs,
+      env: {
+        ...process.env,
+        OTEL_EXPORTER_OTLP_ENDPOINT: cfg.otlpEndpoint
+      },
+      stdio: "pipe"
+    });
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return fail(`Test command failed: ${msg}`);
+  }
+  const endTime = (/* @__PURE__ */ new Date()).toISOString();
+  const eventsResult = await queryTelemetryEvents(cfg.queryEndpoint, startTime, endTime);
+  if (!eventsResult.success) {
+    return fail(eventsResult.error);
+  }
+  const modules = discoverModules(projectDir);
+  const entries = mapEventsToModules(eventsResult.data, projectDir, modules);
+  const modulesWithTelemetry = entries.filter((e) => e.telemetryDetected).length;
+  const totalModules = entries.length;
+  const coveragePercent = totalModules === 0 ? 0 : modulesWithTelemetry / totalModules * 100;
+  return ok({
+    entries,
+    totalModules,
+    modulesWithTelemetry,
+    coveragePercent,
+    skipped: false
+  });
+}
+async function checkBackendHealth(queryEndpoint) {
+  try {
+    const response = await fetch(`${queryEndpoint}/health`, {
+      signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS)
+    });
+    return response.ok;
+  } catch {
+    return false;
+  }
+}
+async function queryTelemetryEvents(queryEndpoint, startTime, endTime) {
+  let url;
+  try {
+    url = new URL("/select/logsql/query", queryEndpoint);
+  } catch {
+    return fail(`Invalid queryEndpoint URL: ${queryEndpoint}`);
+  }
+  url.searchParams.set("query", "*");
+  url.searchParams.set("start", startTime);
+  url.searchParams.set("end", endTime);
+  url.searchParams.set("limit", "1000");
+  try {
+    const response = await fetch(url.toString(), {
+      signal: AbortSignal.timeout(QUERY_TIMEOUT_MS)
+    });
+    if (!response.ok) {
+      return fail(`VictoriaLogs returned ${response.status}`);
+    }
+    const text = await response.text();
+    const events = parseLogEvents(text);
+    return ok(events);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return fail(`Failed to query telemetry events: ${msg}`);
+  }
+}
+function mapEventsToModules(events, projectDir, modules) {
+  void projectDir;
+  const moduleList = modules ?? [];
+  const moduleCounts = /* @__PURE__ */ new Map();
+  for (const mod of moduleList) {
+    moduleCounts.set(mod, 0);
+  }
+  for (const event of events) {
+    for (const mod of moduleList) {
+      if (event.source.includes(mod) || event.message.includes(mod)) {
+        moduleCounts.set(mod, (moduleCounts.get(mod) ?? 0) + 1);
+      }
+    }
+  }
+  return moduleList.map((mod) => {
+    const count = moduleCounts.get(mod) ?? 0;
+    return {
+      moduleName: mod,
+      telemetryDetected: count > 0,
+      eventCount: count
+    };
+  });
+}
+function discoverModules(projectDir) {
+  const srcDir = join4(projectDir, "src");
+  try {
+    return readdirSync(srcDir).filter((name) => {
+      try {
+        return statSync(join4(srcDir, name)).isDirectory();
+      } catch {
+        return false;
+      }
+    });
+  } catch {
+    return [];
+  }
+}
+function parseLogEvents(text) {
+  if (!text.trim()) return [];
+  const lines = text.trim().split("\n");
+  const events = [];
+  for (const line of lines) {
+    try {
+      const raw = JSON.parse(line);
+      events.push({
+        timestamp: String(raw._time ?? raw.timestamp ?? ""),
+        message: String(raw._msg ?? raw.message ?? ""),
+        source: String(raw.source ?? raw._source ?? raw.service ?? "")
+      });
+    } catch {
+    }
+  }
+  return events;
+}
 export {
   analyze,
+  checkBackendHealth,
   checkCoverageTarget,
   checkObservabilityCoverageGate,
   computeRuntimeCoverage,
   getCoverageTrend,
+  mapEventsToModules,
+  queryTelemetryEvents,
   readCoverageState,
   saveCoverageResult,
-  saveRuntimeCoverage
+  saveRuntimeCoverage,
+  validateRuntime
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeharness",
-  "version": "0.22.0",
+  "version": "0.22.2",
   "type": "module",
   "description": "CLI for codeharness — makes autonomous coding agents produce software that actually works",
   "bin": {

package/ralph/ralph.sh

@@ -782,7 +782,12 @@ execute_iteration() {
         fi
 
         local has_errors="false"
-        if grep -v '"[^"]*error[^"]*":' "$output_file" 2>/dev/null | \
+        # Only check non-JSON lines for errors. Stream-json output is NDJSON
+        # (one JSON object per line), so any line starting with '{' is Claude
+        # content — which naturally contains words like "error" and "Exception"
+        # in code reviews, test output, and discussion. Grepping those produces
+        # false positives that trip the circuit breaker.
+        if grep -v '^[[:space:]]*{' "$output_file" 2>/dev/null | \
            grep -qE '(^Error:|^ERROR:|^error:|\]: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)'; then
             has_errors="true"
             log_status "WARN" "Errors detected in output"