@opengeni/api-router 0.30.3 → 1.0.0

package/src/sandbox/auth-callout.ts

@@ -12,15 +12,15 @@
 //      resolveEnrollmentSigningSecret) — an invalid/expired/forged bearer is denied;
 //   3. confirms the enrollment is still ACTIVE in the DB at the exact credential
 //      generation (a revoked or re-enrolled machine denies an old bearer);
-//   4. signs a NATS user JWT granting pub/sub ONLY `agent.<ws>.>` + `_INBOX.>`
-//      (deny-all-else by an allow-list) and returns it inside a signed
+//   4. atomically claims the daemon instance named in CONNECT;
+//   5. signs a NATS user JWT granting pub/sub ONLY that exact process subtree +
+//      `_INBOX.>` (deny-all-else by an allow-list) and returns it inside a signed
 //      authorization-response JWT.
 //
-// That per-subject scope is the per-workspace ISOLATION: workspace A's agent is
-// cryptographically incapable of pub/sub on `agent.B.>`. nats-server enforces the
-// signed permission set; the boundary does not rely on subject naming alone (the M4
-// transport test proved the CODE constructs scoped subjects; THIS proves the SERVER
-// refuses cross-workspace access).
+// That exact scope is both the tenancy boundary and routing fence: workspace A
+// cannot reach B, and an old/duplicate process cannot subscribe to its successor's
+// work. nats-server enforces the signed permission set; the boundary does not rely
+// on naming alone.
 //
 // SECURITY (§18): the bearer value + the account signing seed are NEVER logged. A
 // validation failure → a DENIAL response (the server refuses the connection); a
@@ -28,34 +28,75 @@
 // on its callout timeout). The bearer's `exp` caps the minted credential's life so a
 // revoked/expired enrollment cannot outlive its bearer.
 
+import { createHash } from "node:crypto";
+import { createLogThrottle, type LogThrottle } from "@opengeni/observability";
 import {
   resolveEnrollmentSigningSecret,
   type NatsCalloutConfig,
   type Settings,
 } from "@opengeni/config";
 import { verifyEnrollmentBearer } from "@opengeni/contracts";
-import { getEnrollment, type Database } from "@opengeni/db";
+import { claimEnrollmentConnection, getEnrollment, type Database } from "@opengeni/db";
 import {
   createResponderConnection,
   decodeAuthRequest,
   mintAuthResponse,
   mintUserJwt,
+  parseAgentConnectionName,
   workspaceAgentPermissions,
   type ResponderConnection,
 } from "@opengeni/events";
 import type { Observability } from "@opengeni/observability";
-import { observabilityEventLogger } from "../observability";
+import { observabilityEventBusOptions } from "../observability";
+import { AGENT_CONNECTION_LEASE_MS } from "./connection-authority";
 
 /** The NATS subject nats-server publishes authorization requests on (ADR-26). */
 export const AUTH_CALLOUT_SUBJECT = "$SYS.REQ.USER.AUTH";
 /** Keep live NATS credentials short-lived while never outliving the bearer. */
 export const NATS_USER_JWT_TTL_SECONDS = 5 * 60;
-
 export interface AuthCalloutDeps {
   db: Database;
   settings: Settings;
   callout: NatsCalloutConfig;
   observability?: Observability;
+  /** Bounds repeated denial warnings; defaults to one process-wide throttle. */
+  denialWarningThrottle?: LogThrottle;
+}
+
+/** A stale or revoked machine re-dials on its reconnect backoff forever, so an
+ * unthrottled denial warning repeats every few seconds per machine. Log each
+ * distinct machine's first denial, then at most once per interval with the
+ * count of denials it hid. The denial decision itself is never throttled. */
+export const AUTH_CALLOUT_DENIAL_WARNING_INTERVAL_MS = 60_000;
+const processDenialWarningThrottle = createLogThrottle({
+  intervalMs: AUTH_CALLOUT_DENIAL_WARNING_INTERVAL_MS,
+  maxKeys: 1_024,
+});
+
+/** Closed denial vocabulary. The public log projection drops identifiers, so
+ * `reason` is what tells an operator which throttle key a count belongs to. */
+type AuthCalloutDenialReason = "invalid_bearer" | "inactive_enrollment" | "duplicate_runner";
+
+function warnDenial(
+  deps: AuthCalloutDeps,
+  key: string,
+  reason: AuthCalloutDenialReason,
+  message: string,
+  attributes: Record<string, string | number> = {},
+): void {
+  const admission = (deps.denialWarningThrottle ?? processDenialWarningThrottle).admit(key);
+  if (!admission) return;
+  deps.observability?.warn?.(message, {
+    ...attributes,
+    reason,
+    ...(admission.suppressedCount > 0 ? { suppressedCount: admission.suppressedCount } : {}),
+  });
+}
+
+/** A process-local dedupe key for a rejected bearer. The digest is never
+ * logged, and the bearer value never leaves this function. */
+function rejectedBearerKey(bearer: string): string {
+  return `invalid-bearer:${createHash("sha256").update(bearer).digest("hex").slice(0, 32)}`;
 }
 
 /**
@@ -107,7 +148,12 @@ export async function handleAuthorizationRequest(
   const claims = await verifyEnrollmentBearer(secret, bearer);
   if (!claims) {
     // Invalid signature / malformed / expired bearer. NEVER log the bearer value.
-    deps.observability?.warn?.("auth-callout: rejected an invalid enrollment bearer", {});
+    warnDenial(
+      deps,
+      rejectedBearerKey(bearer),
+      "invalid_bearer",
+      "auth-callout: rejected an invalid enrollment bearer",
+    );
     return deny("invalid or expired enrollment bearer");
   }
 
@@ -115,10 +161,13 @@ export async function handleAuthorizationRequest(
   // still-unexpired bearer (the revoke path flips status; this re-checks at connect).
   const enrollment = await getEnrollment(deps.db, claims.workspaceId, claims.enrollmentId);
   if (!enrollment || enrollment.status !== "active") {
-    deps.observability?.warn?.("auth-callout: denied a revoked or unknown enrollment", {
-      workspaceId: claims.workspaceId,
-      agentId: claims.agentId,
-    });
+    warnDenial(
+      deps,
+      `inactive:${claims.workspaceId}:${claims.agentId}`,
+      "inactive_enrollment",
+      "auth-callout: denied a revoked or unknown enrollment",
+      { workspaceId: claims.workspaceId, agentId: claims.agentId },
+    );
     return deny("enrollment is not active");
   }
 
@@ -138,9 +187,36 @@ export async function handleAuthorizationRequest(
     return deny("enrollment credential generation mismatch");
   }
 
-  // GRANT: a user JWT scoped to ONLY this workspace's agent subtree + the reply
-  // inbox. This allow-list IS the per-workspace isolation boundary.
-  const permissions = workspaceAgentPermissions(claims.workspaceId);
+  const connectionInstanceId = parseAgentConnectionName(decoded.name);
+  if (!connectionInstanceId) {
+    return deny("missing or invalid agent connection instance");
+  }
+  const claim = await claimEnrollmentConnection(deps.db, {
+    workspaceId: claims.workspaceId,
+    enrollmentId: claims.enrollmentId,
+    credentialGeneration: claims.credentialGeneration,
+    connectionInstanceId,
+    leaseMs: AGENT_CONNECTION_LEASE_MS,
+  });
+  if (!claim.claimed) {
+    warnDenial(
+      deps,
+      `duplicate:${claims.workspaceId}:${claims.agentId}`,
+      "duplicate_runner",
+      "auth-callout: denied a duplicate live runner",
+      { workspaceId: claims.workspaceId, agentId: claims.agentId },
+    );
+    return deny("another runner instance currently owns this enrollment");
+  }
+
+  // GRANT: the operational subject includes the exact claimed process instance.
+  // A previous socket may remain open until its short JWT expires, but no control
+  // request is routed to its old subject after a successor claims authority.
+  const permissions = workspaceAgentPermissions(
+    claims.workspaceId,
+    claims.agentId,
+    connectionInstanceId,
+  );
   const nowSeconds = Math.floor(Date.now() / 1000);
   const userJwt = mintUserJwt({
     userPublicKey: decoded.userNkey,
@@ -165,6 +241,7 @@ export async function handleAuthorizationRequest(
   deps.observability?.info?.("auth-callout: granted a workspace-scoped NATS credential", {
     workspaceId: claims.workspaceId,
     agentId: claims.agentId,
+    connectionGeneration: claim.connectionGeneration,
   });
   return Buffer.from(response, "utf8");
 }
@@ -188,7 +265,7 @@ export async function startAuthCalloutResponder(
     (bytes) => handleAuthorizationRequest(deps, bytes),
     {
       name: "opengeni-auth-callout",
-      ...(deps.observability ? { logger: observabilityEventLogger(deps.observability) } : {}),
+      ...(deps.observability ? observabilityEventBusOptions(deps.observability) : {}),
     },
   );
   deps.observability?.info?.("OpenGeni NATS auth-callout responder started", {