harnery 0.35.0 → 0.37.0

package/src/core/semantic/routes.ts

@@ -0,0 +1,30 @@
+import type { SemanticConfiguredModel, SemanticHarness } from "./contract.ts";
+
+export interface SemanticReaderRoute {
+  harness: SemanticHarness;
+  binary: string;
+  configured_model: SemanticConfiguredModel;
+  invocation_model_id: string;
+}
+
+/** Pure route data shared by the adapters, CLI status, and read-only Codec path. */
+export const SEMANTIC_READER_ROUTES: Record<SemanticHarness, SemanticReaderRoute> = {
+  "claude-code": {
+    harness: "claude-code",
+    binary: "claude",
+    configured_model: "haiku-4.5",
+    invocation_model_id: "claude-haiku-4-5-20251001",
+  },
+  codex: {
+    harness: "codex",
+    binary: "codex",
+    configured_model: "gpt-5.6-luna",
+    invocation_model_id: "gpt-5.6-luna",
+  },
+  cursor: {
+    harness: "cursor",
+    binary: "cursor-agent",
+    configured_model: "composer-2.5",
+    invocation_model_id: "composer-2.5",
+  },
+};

package/src/core/semantic/scheduler.ts

@@ -1,14 +1,17 @@
 import type { SemanticAgentReadModelV2, SemanticEvidenceV1 } from "./contract.ts";
 import type { SemanticCallReceipt, SemanticPendingItem } from "./storage.ts";
 
-export const SEMANTIC_HARD_CALLS_PER_HOUR = 60;
+export const SEMANTIC_HARD_CALLS_PER_HOUR = 120;
 export const SEMANTIC_INVALID_RETRY_COOLDOWN_MS = 5 * 60_000;
-export const SEMANTIC_MIN_GENERATION_CALL_INTERVAL_MS = 30_000;
+export const SEMANTIC_MIN_PRIORITY_GENERATION_CALL_INTERVAL_MS = 2 * 60_000;
+export const SEMANTIC_MIN_AMBIENT_GENERATION_CALL_INTERVAL_MS = 5 * 60_000;
+export const SEMANTIC_MIN_GENERATION_CALL_INTERVAL_MS =
+  SEMANTIC_MIN_PRIORITY_GENERATION_CALL_INTERVAL_MS;
 
 export function semanticPriorityBand(evidence: SemanticEvidenceV1): 1 | 2 {
   if (
     evidence.attention ||
-    evidence.lifecycle ||
+    (evidence.lifecycle && evidence.lifecycle.state !== "active") ||
     evidence.recent.some((item) => item.kind === "terminal")
   ) {
     return 1;
@@ -16,6 +19,12 @@ export function semanticPriorityBand(evidence: SemanticEvidenceV1): 1 | 2 {
   return 2;
 }
 
+export function semanticPendingCallIntervalMs(pending: SemanticPendingItem): number {
+  return pending.band === 1
+    ? SEMANTIC_MIN_PRIORITY_GENERATION_CALL_INTERVAL_MS
+    : SEMANTIC_MIN_AMBIENT_GENERATION_CALL_INTERVAL_MS;
+}
+
 export function enqueueSemanticPending(
   pending: readonly SemanticPendingItem[],
   evidence: SemanticEvidenceV1,
@@ -35,6 +44,7 @@ export function enqueueSemanticPending(
 export function selectSemanticPending(
   pending: readonly SemanticPendingItem[],
   lastFirstBandGenerationId?: string,
+  callHistory: readonly SemanticCallReceipt[] = [],
 ): SemanticPendingItem | undefined {
   const sorted = [...pending].sort(
     (left, right) =>
@@ -42,14 +52,17 @@ export function selectSemanticPending(
       left.pending_since.localeCompare(right.pending_since) ||
       left.generation_id.localeCompare(right.generation_id),
   );
-  const firstBand = sorted.filter((item) => item.band === 1);
+  const recentlyCalled = new Set(callHistory.map((call) => call.generation_id));
+  const uncovered = sorted.filter((item) => !recentlyCalled.has(item.generation_id));
+  const eligible = uncovered.length > 0 ? uncovered : sorted;
+  const firstBand = eligible.filter((item) => item.band === 1);
   if (firstBand.length > 0) {
     const lastIndex = firstBand.findIndex(
       (item) => item.generation_id === lastFirstBandGenerationId,
     );
     return firstBand[lastIndex >= 0 ? (lastIndex + 1) % firstBand.length : 0];
   }
-  return sorted[0];
+  return eligible[0];
 }
 
 export function activeSemanticCallHistory(
@@ -92,6 +105,37 @@ export function semanticGenerationCallEligible(
   return newest === undefined || nowMs - newest >= Math.max(0, minimumIntervalMs);
 }
 
+export function semanticPendingPassDue(input: {
+  pending: readonly SemanticPendingItem[];
+  callHistory: readonly SemanticCallReceipt[];
+  nowMs: number;
+  debounceMs: number;
+  callsPerHour?: number;
+  minimumGenerationCallIntervalMs?: number;
+}): boolean {
+  const matured = input.pending.filter((item) => {
+    const pendingSince = Date.parse(item.pending_since);
+    return (
+      !Number.isFinite(pendingSince) || input.nowMs - pendingSince >= Math.max(0, input.debounceMs)
+    );
+  });
+  if (matured.length === 0) return false;
+
+  const cap = semanticRateCap(input.callHistory, input.nowMs, input.callsPerHour);
+  // One pass is still required at the cap so runSemanticOnce can publish
+  // deferred receipts and remove the matured items from the pending queue.
+  if (cap.available === 0) return true;
+
+  return matured.some((item) =>
+    semanticGenerationCallEligible(
+      input.callHistory,
+      item.generation_id,
+      input.nowMs,
+      input.minimumGenerationCallIntervalMs ?? semanticPendingCallIntervalMs(item),
+    ),
+  );
+}
+
 export function semanticDocumentEligible(
   document: SemanticAgentReadModelV2 | undefined,
   evidence: SemanticEvidenceV1,

package/src/core/semantic/service-status.ts

@@ -2,7 +2,19 @@ import { existsSync, readFileSync, statSync } from "node:fs";
 import { hostname } from "node:os";
 import { resolve } from "node:path";
 
-import { readSemanticManifest, semanticPaths } from "./storage.ts";
+import type { SemanticHarness } from "./contract.ts";
+import { SEMANTIC_READER_ROUTES } from "./routes.ts";
+import {
+  activeSemanticCallHistory,
+  SEMANTIC_HARD_CALLS_PER_HOUR,
+  semanticRateCap,
+} from "./scheduler.ts";
+import { readSemanticManifest, type SemanticReaderResolution, semanticPaths } from "./storage.ts";
+import {
+  aggregateSemanticUsage,
+  emptySemanticUsageAggregate,
+  type SemanticUsageAggregate,
+} from "./usage.ts";
 
 export const SEMANTIC_SERVICE_STATUS_SCHEMA_VERSION = 1 as const;
 const FOREIGN_STATUS_STALE_MS = 2 * 60_000;
@@ -24,6 +36,7 @@ export interface SemanticServiceStatusRecord {
   pass_count: number;
   model_calls: number;
   cache_hits: number;
+  process_usage?: SemanticUsageAggregate;
   last_sweep_at?: string;
   last_pass_at?: string;
   last_error_code?: SemanticServiceErrorCode;
@@ -36,6 +49,23 @@ export interface SemanticServiceStatus {
   record?: SemanticServiceStatusRecord;
   newest_successful_pass?: string;
   pending_count: number;
+  rolling_calls?: {
+    used: number;
+    limit: number;
+    available: number;
+    eligible_after?: string;
+  };
+  routes?: Array<{
+    harness: keyof typeof SEMANTIC_READER_ROUTES;
+    configured_model: string;
+    invocation_model_id: string;
+    resolved_model_id?: string;
+    model_attestation?: "verified" | "requested-only";
+    available?: boolean;
+    reason_code?: string;
+  }>;
+  rolling_usage?: SemanticUsageAggregate;
+  process_usage?: SemanticUsageAggregate;
 }
 
 /** Read-only service health. This module cannot spawn, signal, or write. */
@@ -43,12 +73,46 @@ export function readSemanticServiceStatus(coordRootRaw: string): SemanticService
   const coordRoot = resolve(coordRootRaw);
   const record = readStatusRecord(coordRoot);
   const manifest = safeManifest(coordRoot);
+  const now = Date.now();
+  const history = activeSemanticCallHistory(manifest?.call_history ?? [], now);
+  const limit = Math.min(
+    SEMANTIC_HARD_CALLS_PER_HOUR,
+    Math.max(1, Math.floor(record?.calls_per_hour ?? SEMANTIC_HARD_CALLS_PER_HOUR)),
+  );
+  const cap = semanticRateCap(history, now, limit);
+  const rollingUsage = aggregateSemanticUsage(
+    history.map((call) => ({
+      source_harness: call.source_harness,
+      configured_model: call.configured_model,
+      resolved_model_id: call.resolved_model_id,
+      model_attestation: call.model_attestation,
+      action: call.outcome,
+      model_call: true,
+      usage: call.usage,
+      invalid_reason_codes: call.invalid_reason_codes,
+    })),
+  );
+  const processUsage = record?.process_usage ?? legacyProcessUsage(record?.model_calls ?? 0);
+  const shared = {
+    ...(manifest?.newest_successful_pass
+      ? { newest_successful_pass: manifest.newest_successful_pass }
+      : {}),
+    pending_count: manifest?.pending.length ?? 0,
+    rolling_calls: {
+      used: history.length,
+      limit,
+      available: cap.available,
+      ...(cap.eligible_after ? { eligible_after: cap.eligible_after } : {}),
+    },
+    routes: semanticStatusRoutes(manifest?.adapter_resolutions),
+    rolling_usage: rollingUsage,
+    process_usage: processUsage,
+  };
   if (!record) {
     return {
       running: false,
       stale: false,
-      newest_successful_pass: manifest?.newest_successful_pass,
-      pending_count: manifest?.pending.length ?? 0,
+      ...shared,
     };
   }
   const running = statusOwnerIsLive(record);
@@ -56,11 +120,41 @@ export function readSemanticServiceStatus(coordRootRaw: string): SemanticService
     running,
     stale: !running && record.state !== "stopped",
     record,
-    newest_successful_pass: manifest?.newest_successful_pass,
-    pending_count: manifest?.pending.length ?? 0,
+    ...shared,
   };
 }
 
+function semanticStatusRoutes(
+  resolutions: Partial<Record<SemanticHarness, SemanticReaderResolution>> | undefined,
+): NonNullable<SemanticServiceStatus["routes"]> {
+  return (Object.keys(SEMANTIC_READER_ROUTES) as Array<keyof typeof SEMANTIC_READER_ROUTES>).map(
+    (harness) => {
+      const route = SEMANTIC_READER_ROUTES[harness];
+      const resolution = resolutions?.[harness];
+      return {
+        harness,
+        configured_model: route.configured_model,
+        invocation_model_id: route.invocation_model_id,
+        ...(resolution?.resolved_model_id
+          ? { resolved_model_id: resolution.resolved_model_id }
+          : {}),
+        ...(resolution?.model_attestation
+          ? { model_attestation: resolution.model_attestation }
+          : {}),
+        ...(resolution ? { available: resolution.available } : {}),
+        ...(resolution?.reason_code ? { reason_code: resolution.reason_code } : {}),
+      };
+    },
+  );
+}
+
+function legacyProcessUsage(modelCalls: number): SemanticUsageAggregate {
+  const aggregate = emptySemanticUsageAggregate();
+  aggregate.call_count = modelCalls;
+  aggregate.unreported_calls = modelCalls;
+  return aggregate;
+}
+
 function safeManifest(coordRoot: string) {
   try {
     return readSemanticManifest(coordRoot);

package/src/core/semantic/service.ts

@@ -16,15 +16,23 @@ import {
   writeFileSync,
 } from "node:fs";
 import { hostname } from "node:os";
-import { dirname, resolve } from "node:path";
+import { dirname, join, resolve } from "node:path";
+import { coordEnv } from "../../lib/env.ts";
 import {
   type ReadLedgerV3SinceResult,
   readEventV3ControlState,
   readLedgerV3Since,
 } from "../events/v3/index.ts";
 import { eventV3ActiveWatchPath } from "../events/v3/reader.ts";
+import {
+  closeProcessLoggers,
+  encodeBoundedLogObjectArray,
+  legacyLogFields,
+  processLogger,
+} from "../storage/logger.ts";
 import type { SemanticHarness } from "./contract.ts";
 import { runSemanticOnce, type SemanticOnceReport } from "./once.ts";
+import { semanticPendingPassDue } from "./scheduler.ts";
 import {
   readSemanticServiceStatus,
   SEMANTIC_SERVICE_STATUS_SCHEMA_VERSION,
@@ -32,12 +40,18 @@ import {
   type SemanticServiceStatus,
   type SemanticServiceStatusRecord,
 } from "./service-status.ts";
+import { semanticSoakReadings } from "./soak.ts";
 import {
   listSemanticAgentDocuments,
   readSemanticManifest,
   semanticPaths,
   writeSemanticManifest,
 } from "./storage.ts";
+import {
+  aggregateSemanticUsage,
+  emptySemanticUsageAggregate,
+  mergeSemanticUsageAggregates,
+} from "./usage.ts";
 
 export {
   readSemanticServiceStatus,
@@ -55,6 +69,20 @@ const FOREIGN_STATUS_STALE_MS = 2 * 60_000;
 const MAX_FILE_BYTES = 512 * 1024;
 const MAX_LOG_BYTES = 512 * 1024;
 const REPEATED_ERROR_LOG_INTERVAL_MS = 60_000;
+const SEMANTIC_READINGS_FIELD_MAX_BYTES = 4_096;
+const SEMANTIC_READING_FIELDS = [
+  "subject_id",
+  "generated_at",
+  "source_harness",
+  "configured_model",
+  "resolved_model_id",
+  "model_attestation",
+  "origin",
+  "phase",
+  "phase_confidence",
+  "expression_cue",
+  "expression_confidence",
+] as const;
 
 interface SemanticServiceLease {
   pid: number;
@@ -112,11 +140,12 @@ export async function spawnSemanticService(
   }
   const paths = semanticPaths(coordRoot);
   mkdirSync(paths.root, { recursive: true, mode: 0o700 });
-  const logFd = openSync(paths.log, "a", 0o600);
-  chmodSync(paths.log, 0o600);
+  const sharedLogs = coordEnv("SHARED_LOGS") !== "0";
+  const logFd = sharedLogs ? undefined : openSync(paths.log, "a", 0o600);
+  if (logFd !== undefined) chmodSync(paths.log, 0o600);
   const harnBin = new URL("../../../bin/harn", import.meta.url).pathname;
   if (!existsSync(harnBin)) {
-    closeSync(logFd);
+    if (logFd !== undefined) closeSync(logFd);
     throw new Error(`cannot find harn executable at ${harnBin}`);
   }
   const args = ["semantic", "service", "daemon", "--root", coordRoot];
@@ -127,7 +156,7 @@ export async function spawnSemanticService(
   const child = spawn(harnBin, args, {
     cwd: coordRoot,
     detached: true,
-    stdio: ["ignore", logFd, logFd],
+    stdio: logFd === undefined ? ["ignore", "ignore", "ignore"] : ["ignore", logFd, logFd],
     env: {
       ...process.env,
       HARNERY_COORD_ROOT_OVERRIDE: coordRoot,
@@ -137,7 +166,7 @@ export async function spawnSemanticService(
   child.once("error", (error) => {
     spawnError = error;
   });
-  closeSync(logFd);
+  if (logFd !== undefined) closeSync(logFd);
   child.unref();
   const deadline = Date.now() + 5_000;
   while (Date.now() < deadline) {
@@ -147,7 +176,10 @@ export async function spawnSemanticService(
     if (status.running && status.record?.pid === child.pid) return status;
     if (child.exitCode !== null) break;
   }
-  throw new Error(`semantic service failed to start; inspect ${paths.log}`);
+  const diagnosticPath = sharedLogs
+    ? join(coordRoot, ".harnery", "logs", "semantic-service", "active.jsonl")
+    : paths.log;
+  throw new Error(`semantic service failed to start; inspect ${diagnosticPath}`);
 }
 
 /**
@@ -243,6 +275,7 @@ export async function runSemanticServiceDaemon(
     pass_count: 0,
     model_calls: 0,
     cache_hits: 0,
+    process_usage: emptySemanticUsageAggregate(),
   };
   let stopRequested = false;
   let dirtySince: number | undefined;
@@ -261,7 +294,7 @@ export async function runSemanticServiceDaemon(
   process.on("SIGTERM", requestStop);
   status.state = "running";
   writeStatus();
-  appendSemanticServiceLog(coordRoot, { event: "service_started" });
+  appendSemanticServiceDiagnostic(coordRoot, { event: "service_started" });
   const heartbeat = setInterval(() => {
     try {
       writeStatus();
@@ -283,14 +316,22 @@ export async function runSemanticServiceDaemon(
         if (read.events.length > 0 || !before?.cursor) {
           dirtySince ??= sweepAt.getTime();
         }
-        const hasPending = (before?.pending.length ?? 0) > 0;
+        const hasPendingPassDue = before
+          ? semanticPendingPassDue({
+              pending: before.pending,
+              callHistory: before.call_history,
+              nowMs: sweepAt.getTime(),
+              debounceMs,
+              ...(input.callsPerHour !== undefined ? { callsPerHour: input.callsPerHour } : {}),
+            })
+          : false;
         const hasEligibleDeferred = listSemanticAgentDocuments(coordRoot).some(
           (document) =>
             document.reader_outcome === "deferred" &&
             Date.parse(document.receipt.eligible_after) <= sweepAt.getTime(),
         );
         if (
-          hasPending ||
+          hasPendingPassDue ||
           hasEligibleDeferred ||
           (dirtySince !== undefined && sweepAt.getTime() - dirtySince >= debounceMs)
         ) {
@@ -303,6 +344,10 @@ export async function runSemanticServiceDaemon(
           status.pass_count += 1;
           status.model_calls += report.model_calls;
           status.cache_hits += report.cache_hits;
+          status.process_usage = mergeSemanticUsageAggregates(
+            status.process_usage ?? emptySemanticUsageAggregate(),
+            aggregateSemanticUsage(report.outcomes),
+          );
           status.last_pass_at = report.completed_at;
           status.last_error_code = undefined;
           lastLoggedErrorCode = undefined;
@@ -324,9 +369,11 @@ export async function runSemanticServiceDaemon(
             invalid: report.outcomes.filter((outcome) => outcome.action === "invalid").length,
             deferred: report.outcomes.filter((outcome) => outcome.action === "deferred").length,
             harness_metrics: semanticHarnessMetrics(report),
+            usage: aggregateSemanticUsage(report.outcomes),
+            semantic_readings: semanticSoakReadings(coordRoot, report.outcomes),
           };
           if (report.model_calls > 0 || report.cache_hits > 0 || logEntry.unavailable > 0) {
-            appendSemanticServiceLog(coordRoot, logEntry);
+            appendSemanticServiceDiagnostic(coordRoot, logEntry);
           }
         }
       } catch (error) {
@@ -335,7 +382,7 @@ export async function runSemanticServiceDaemon(
           status.last_error_code !== lastLoggedErrorCode ||
           sweepAt.getTime() - lastLoggedErrorAt >= REPEATED_ERROR_LOG_INTERVAL_MS
         ) {
-          appendSemanticServiceLog(coordRoot, {
+          appendSemanticServiceDiagnostic(coordRoot, {
             event: "sweep_error",
             reason_code: status.last_error_code,
           });
@@ -358,13 +405,18 @@ export async function runSemanticServiceDaemon(
     status.state = "stopped";
     status.stopped_at = now().toISOString();
     writeStatus();
-    appendSemanticServiceLog(coordRoot, {
+    appendSemanticServiceDiagnostic(coordRoot, {
       event: "service_stopped",
       sweeps: status.sweep_count,
       passes: status.pass_count,
       model_calls: status.model_calls,
+      usage: status.process_usage,
     });
-    release();
+    try {
+      await closeProcessLoggers();
+    } finally {
+      release();
+    }
   }
   return status;
 }
@@ -534,7 +586,56 @@ async function waitForLedgerWake(path: string, milliseconds: number): Promise<vo
   });
 }
 
-function appendSemanticServiceLog(coordRoot: string, entry: Record<string, unknown>): void {
+export function appendSemanticServiceDiagnostic(
+  coordRoot: string,
+  entry: Record<string, unknown>,
+): "shared" | "legacy" {
+  const prepared = prepareSemanticServiceDiagnostic(entry);
+  if (coordEnv("SHARED_LOGS") !== "0") {
+    try {
+      const logger = processLogger(coordRoot, "semantic-service");
+      if (prepared.event.includes("error")) {
+        logger.error(prepared.event, legacyLogFields(prepared.fields));
+      } else logger.info(prepared.event, legacyLogFields(prepared.fields));
+      return "shared";
+    } catch {
+      // Bootstrap failures retain the bounded legacy writer as the process fallback.
+    }
+  }
+  appendSemanticServiceLegacyLog(coordRoot, prepared.legacyEntry);
+  return "legacy";
+}
+
+function prepareSemanticServiceDiagnostic(entry: Record<string, unknown>): {
+  event: string;
+  fields: Record<string, unknown>;
+  legacyEntry: Record<string, unknown>;
+} {
+  const event = typeof entry.event === "string" ? entry.event : "service_event";
+  const fields = { ...entry };
+  delete fields.event;
+  const legacyEntry: Record<string, unknown> = { ...entry, event };
+  const semanticReadings = fields.semantic_readings;
+  if (semanticReadings === undefined) return { event, fields, legacyEntry };
+  if (!Array.isArray(semanticReadings)) throw new Error("semantic_readings must be an array");
+  const bounded = encodeBoundedLogObjectArray(semanticReadings, {
+    max_bytes: SEMANTIC_READINGS_FIELD_MAX_BYTES,
+    allowed_fields: SEMANTIC_READING_FIELDS,
+  });
+  fields.semantic_readings = bounded.json;
+  fields.semantic_readings_count = bounded.included;
+  legacyEntry.semantic_readings = JSON.parse(bounded.json) as unknown[];
+  legacyEntry.semantic_readings_count = bounded.included;
+  if (bounded.truncated) {
+    fields.semantic_readings_truncated = true;
+    fields.semantic_readings_omitted = bounded.omitted;
+    legacyEntry.semantic_readings_truncated = true;
+    legacyEntry.semantic_readings_omitted = bounded.omitted;
+  }
+  return { event, fields, legacyEntry };
+}
+
+function appendSemanticServiceLegacyLog(coordRoot: string, entry: Record<string, unknown>): void {
   const path = semanticPaths(coordRoot).log;
   mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
   appendFileSync(