@triflux/core 10.43.0 → 10.45.0

package/hub/router.mjs

@@ -9,6 +9,7 @@ import {
   resolveHardCeilingMs,
 } from "./lib/timeout-defaults.mjs";
 import { uuidv7 } from "./lib/uuidv7.mjs";
+import { normalizeRoleKey } from "./role-contract.mjs";
 
 const ASSIGN_PENDING_STATUSES = new Set(["queued", "running"]);
 const ROLE_TOPICS = new Set(["cto"]);
@@ -16,6 +17,258 @@ const ROLE_SOURCE_RANK = Object.freeze({
   explicit: 2,
   "topic-fallback": 1,
 });
+const TRANSPORT_LOCATOR_SCHEMA_VERSION = "tfx.transport-locator.v1";
+const TRANSPORT_KINDS = new Set(["claude-uds", "tmux"]);
+const TRANSPORT_CAPABILITIES = new Set([
+  "probe",
+  "wake",
+  "activate",
+  "interrupt",
+]);
+const HOST_ID_RE = /^hst_[A-Za-z0-9_-]+$/u;
+
+function registrationError(code, message = code) {
+  const error = new Error(message);
+  error.code = code;
+  return error;
+}
+
+function registrationString(value, code, { maxLength = 256 } = {}) {
+  if (typeof value !== "string" || !value.trim() || value.length > maxLength) {
+    throw registrationError(code);
+  }
+  return value.trim();
+}
+
+function registrationLogicalRef(value, code, { maxLength = 128 } = {}) {
+  const normalized = registrationString(value, code, { maxLength });
+  if (!/^[A-Za-z0-9._:-]+$/u.test(normalized)) {
+    throw registrationError(code);
+  }
+  return normalized;
+}
+
+function assertExactFields(value, allowed, code) {
+  if (
+    value === null ||
+    typeof value !== "object" ||
+    Array.isArray(value) ||
+    Object.keys(value).some((field) => !allowed.has(field))
+  ) {
+    throw registrationError(code);
+  }
+}
+
+function normalizeTransportLocator(
+  value,
+  { cli, hostId, now, leaseExpiresMs },
+) {
+  assertExactFields(
+    value,
+    new Set([
+      "schema_version",
+      "transport_id",
+      "kind",
+      "host_id",
+      "capabilities",
+      "priority",
+      "expires_at_ms",
+      "locator",
+    ]),
+    "TRANSPORT_LOCATOR_INVALID",
+  );
+  if (value.schema_version !== TRANSPORT_LOCATOR_SCHEMA_VERSION) {
+    throw registrationError("TRANSPORT_LOCATOR_SCHEMA_UNSUPPORTED");
+  }
+  const transportId = registrationLogicalRef(
+    value.transport_id,
+    "TRANSPORT_ID_INVALID",
+    { maxLength: 128 },
+  );
+  if (!TRANSPORT_KINDS.has(value.kind)) {
+    throw registrationError("TRANSPORT_KIND_UNSUPPORTED");
+  }
+  if (cli === "codex" && value.kind !== "tmux") {
+    throw registrationError("CODEX_TRANSPORT_UNSUPPORTED");
+  }
+  const locatorHostId = registrationString(value.host_id, "HOST_ID_INVALID", {
+    maxLength: 128,
+  });
+  if (!HOST_ID_RE.test(locatorHostId) || (hostId && locatorHostId !== hostId)) {
+    throw registrationError("HOST_ID_INVALID");
+  }
+  if (!Array.isArray(value.capabilities)) {
+    throw registrationError("TRANSPORT_CAPABILITIES_INVALID");
+  }
+  const capabilities = uniqueStrings(value.capabilities);
+  if (
+    capabilities.length !== value.capabilities.length ||
+    capabilities.some((capability) => !TRANSPORT_CAPABILITIES.has(capability))
+  ) {
+    throw registrationError("TRANSPORT_CAPABILITIES_INVALID");
+  }
+  if (!Number.isSafeInteger(value.priority)) {
+    throw registrationError("TRANSPORT_PRIORITY_INVALID");
+  }
+  if (
+    !Number.isSafeInteger(value.expires_at_ms) ||
+    value.expires_at_ms <= now ||
+    value.expires_at_ms > leaseExpiresMs
+  ) {
+    throw registrationError("TRANSPORT_EXPIRY_INVALID");
+  }
+
+  let locator;
+  if (value.kind === "claude-uds") {
+    assertExactFields(
+      value.locator,
+      new Set(["session_id", "short", "bridge_id"]),
+      "TRANSPORT_LOCATOR_INVALID",
+    );
+    const sessionId = value.locator.session_id;
+    const short = value.locator.short;
+    if (!sessionId && !short) {
+      throw registrationError("CLAUDE_UDS_SESSION_REQUIRED");
+    }
+    locator = {
+      ...(sessionId
+        ? {
+            session_id: registrationLogicalRef(
+              sessionId,
+              "CLAUDE_UDS_SESSION_INVALID",
+            ),
+          }
+        : {}),
+      ...(short
+        ? {
+            short: registrationLogicalRef(short, "CLAUDE_UDS_SHORT_INVALID", {
+              maxLength: 128,
+            }),
+          }
+        : {}),
+      bridge_id: registrationLogicalRef(
+        value.locator.bridge_id,
+        "CLAUDE_UDS_BRIDGE_INVALID",
+        { maxLength: 128 },
+      ),
+    };
+  } else {
+    assertExactFields(
+      value.locator,
+      new Set(["session_name", "mux_server_id"]),
+      "TRANSPORT_LOCATOR_INVALID",
+    );
+    locator = {
+      session_name: registrationLogicalRef(
+        value.locator.session_name,
+        "TMUX_SESSION_INVALID",
+        { maxLength: 128 },
+      ),
+      mux_server_id: registrationLogicalRef(
+        value.locator.mux_server_id,
+        "TMUX_SERVER_INVALID",
+        { maxLength: 128 },
+      ),
+    };
+  }
+
+  return {
+    schema_version: TRANSPORT_LOCATOR_SCHEMA_VERSION,
+    transport_id: transportId,
+    kind: value.kind,
+    host_id: locatorHostId,
+    capabilities,
+    priority: value.priority,
+    expires_at_ms: value.expires_at_ms,
+    locator,
+  };
+}
+
+export function normalizeRegistrationIdentity(args = {}, now = Date.now()) {
+  const hasProjectId = Object.hasOwn(args, "project_id");
+  const hasSessionId = Object.hasOwn(args, "session_id");
+  const hasHostId = Object.hasOwn(args, "host_id");
+  const hasLocatorJson = Object.hasOwn(args, "transport_locators_json");
+  const hasLocators = Object.hasOwn(args, "transport_locators");
+  if (
+    !hasProjectId &&
+    !hasSessionId &&
+    !hasHostId &&
+    !hasLocatorJson &&
+    !hasLocators
+  ) {
+    return null;
+  }
+
+  const projectId = hasProjectId
+    ? normalizeRoleKey({ project_id: args.project_id, role_kind: "cto" })
+        .project_id
+    : null;
+  const sessionId = hasSessionId
+    ? registrationString(args.session_id, "SESSION_ID_INVALID")
+    : null;
+  const hostId = hasHostId
+    ? registrationString(args.host_id, "HOST_ID_INVALID", { maxLength: 128 })
+    : null;
+  if (hostId && !HOST_ID_RE.test(hostId)) {
+    throw registrationError("HOST_ID_INVALID");
+  }
+
+  let rawLocators = [];
+  if (hasLocatorJson) {
+    if (Array.isArray(args.transport_locators_json)) {
+      rawLocators = args.transport_locators_json;
+    } else if (typeof args.transport_locators_json === "string") {
+      try {
+        rawLocators = JSON.parse(args.transport_locators_json);
+      } catch {
+        throw registrationError("TRANSPORT_LOCATORS_JSON_INVALID");
+      }
+    } else {
+      throw registrationError("TRANSPORT_LOCATORS_JSON_INVALID");
+    }
+  } else if (hasLocators) {
+    rawLocators = args.transport_locators;
+  }
+  if (!Array.isArray(rawLocators)) {
+    throw registrationError("TRANSPORT_LOCATORS_INVALID");
+  }
+
+  const ttlMs = Number(args.heartbeat_ttl_ms ?? 30000);
+  if (rawLocators.length > 0 && (!Number.isFinite(ttlMs) || ttlMs <= 0)) {
+    throw registrationError("HEARTBEAT_TTL_INVALID");
+  }
+  const leaseExpiresMs = now + ttlMs;
+  const normalizedLocators = rawLocators
+    .map((locator) =>
+      normalizeTransportLocator(locator, {
+        cli: args.cli,
+        hostId,
+        now,
+        leaseExpiresMs,
+      }),
+    )
+    .sort(
+      (left, right) =>
+        right.priority - left.priority ||
+        left.transport_id.localeCompare(right.transport_id),
+    );
+  const resolvedHostId = hostId || normalizedLocators[0]?.host_id || null;
+  if (
+    resolvedHostId &&
+    normalizedLocators.some((locator) => locator.host_id !== resolvedHostId)
+  ) {
+    throw registrationError("HOST_ID_INVALID");
+  }
+
+  return {
+    project_id: projectId,
+    session_id: sessionId,
+    host_id: resolvedHostId,
+    transport_locators_json:
+      hasLocatorJson || hasLocators ? JSON.stringify(normalizedLocators) : null,
+  };
+}
 
 function uniqueStrings(values = []) {
   return Array.from(
@@ -123,10 +376,89 @@ export function createRouter(store) {
   const queuesByAgent = new Map();
   const liveMessages = new Map();
   const roleStates = new Map();
+  const hubInstanceId = uuidv7();
+  const storeFingerprint =
+    store.fingerprint || `${store.type || "unknown"}:unknown`;
+  const updateRegistrationIdentity = store.db?.prepare
+    ? store.db.prepare(`
+        UPDATE agents SET
+          project_id=COALESCE(@project_id, project_id),
+          session_id=COALESCE(@session_id, session_id),
+          host_id=COALESCE(@host_id, host_id),
+          transport_locators_json=COALESCE(
+            @transport_locators_json,
+            transport_locators_json
+          )
+        WHERE agent_id=@agent_id`)
+    : null;
   const MAX_LATENCY_SAMPLES = 100;
   let latencyIdx = 0;
   const deliveryLatencies = new Array(MAX_LATENCY_SAMPLES).fill(0);
 
+  function hubIdentity() {
+    return {
+      hub_instance_id: hubInstanceId,
+      store_fingerprint: storeFingerprint,
+    };
+  }
+
+  function validateCallerIdentity(input = {}) {
+    if (
+      input.hub_instance_id != null &&
+      input.hub_instance_id !== hubInstanceId
+    ) {
+      return {
+        ok: false,
+        error: {
+          code: "STALE_HUB_INSTANCE",
+          message: "caller is bound to a previous hub instance",
+        },
+        data: hubIdentity(),
+      };
+    }
+    if (
+      input.store_fingerprint != null &&
+      input.store_fingerprint !== storeFingerprint
+    ) {
+      return {
+        ok: false,
+        error: {
+          code: "STORE_FINGERPRINT_MISMATCH",
+          message: "caller store fingerprint does not match this hub",
+        },
+        data: hubIdentity(),
+      };
+    }
+    return null;
+  }
+
+  function isControlPlaneRegistration(args = {}) {
+    const metadata = args.metadata || {};
+    const values = [
+      ...(Array.isArray(args.capabilities) ? args.capabilities : []),
+      ...(Array.isArray(args.topics) ? args.topics : []),
+      metadata.role,
+      ...(Array.isArray(metadata.roles) ? metadata.roles : []),
+    ]
+      .flat()
+      .map((value) => String(value || "").toLowerCase());
+    return (
+      metadata.control_plane === true ||
+      values.some((value) => value === "cto" || value === "role:cto")
+    );
+  }
+
+  function staleOwnerFailure(agentId, result) {
+    return {
+      ok: false,
+      error: {
+        code: "STALE_PRESENCE_OWNER",
+        message: `stale presence owner: ${agentId}`,
+      },
+      data: { ...result, ...hubIdentity() },
+    };
+  }
+
   function ensureAgentQueue(agentId) {
     let queue = queuesByAgent.get(agentId);
     if (!queue) {
@@ -855,10 +1187,29 @@ export function createRouter(store) {
     deliveryEmitter,
 
     registerAgent(args) {
+      const identityFailure = validateCallerIdentity(args);
+      if (identityFailure) return identityFailure;
+      if (isControlPlaneRegistration(args) && store.type !== "sqlite") {
+        return {
+          ok: false,
+          error: {
+            code: "CONTROL_PLANE_STORE_DEGRADED",
+            message: "control-plane presence requires durable SQLite storage",
+          },
+          data: { ...hubIdentity(), store_type: store.type || "unknown" },
+        };
+      }
+      const identity = normalizeRegistrationIdentity(args);
       const result = store.registerAgent(args);
       if (!registerPresenceIsEffective(result)) {
         return registerPresenceFailure(args.agent_id, result);
       }
+      if (identity && updateRegistrationIdentity) {
+        updateRegistrationIdentity.run({
+          agent_id: args.agent_id,
+          ...identity,
+        });
+      }
       upsertRuntimeTopics(args.agent_id, args.topics || [], { replace: true });
       refreshRoleCandidateForAgent(args.agent_id);
       for (const roleName of ROLE_TOPICS) {
@@ -867,11 +1218,23 @@ export function createRouter(store) {
           transferBacklog: true,
         });
       }
-      return { ok: true, data: { ...result, hub_pid: process.pid } };
+      return {
+        ok: true,
+        data: { ...result, hub_pid: process.pid, ...hubIdentity() },
+      };
     },
 
-    refreshAgentLease(agentId, ttlMs = 30000) {
-      const result = store.refreshLease(agentId, ttlMs);
+    refreshAgentLease(
+      agentId,
+      ttlMs = 30000,
+      presenceGeneration,
+      callerIdentity = {},
+    ) {
+      const identityFailure = validateCallerIdentity(callerIdentity);
+      if (identityFailure) return identityFailure;
+      const result = store.refreshLease(agentId, ttlMs, presenceGeneration);
+      if (result?.effective?.updated === false)
+        return staleOwnerFailure(agentId, result);
       refreshRoleCandidateForAgent(agentId);
       for (const roleName of ROLE_TOPICS) {
         ensureRoleLeader(roleName, {
@@ -879,7 +1242,7 @@ export function createRouter(store) {
           transferBacklog: true,
         });
       }
-      return result;
+      return { ok: true, data: result };
     },
 
     subscribeAgent(agentId, topics, { replace = false } = {}) {
@@ -898,11 +1261,15 @@ export function createRouter(store) {
       return listRuntimeTopics(agentId);
     },
 
-    updateAgentStatus(agentId, status) {
+    updateAgentStatus(agentId, status, presenceGeneration) {
       if (status === "offline") {
         runtimeTopics.delete(agentId);
       }
-      const updated = store.updateAgentStatus(agentId, status);
+      const updated = store.updateAgentStatus(
+        agentId,
+        status,
+        presenceGeneration,
+      );
       refreshRoleCandidateForAgent(agentId);
       for (const roleName of ROLE_TOPICS) {
         ensureRoleLeader(roleName, {
@@ -913,6 +1280,39 @@ export function createRouter(store) {
       return updated;
     },
 
+    transitionAgentStatus(
+      agentId,
+      status,
+      presenceGeneration,
+      callerIdentity = {},
+    ) {
+      const identityFailure = validateCallerIdentity(callerIdentity);
+      if (identityFailure) return identityFailure;
+      const updated = store.updateAgentStatus(
+        agentId,
+        status,
+        presenceGeneration,
+      );
+      if (!updated) return staleOwnerFailure(agentId, { status });
+      if (status === "offline") runtimeTopics.delete(agentId);
+      refreshRoleCandidateForAgent(agentId);
+      for (const roleName of ROLE_TOPICS) {
+        ensureRoleLeader(roleName, {
+          reason: `status:${status}`,
+          transferBacklog: true,
+        });
+      }
+      return {
+        ok: true,
+        data: {
+          agent_id: agentId,
+          status,
+          presence_generation: presenceGeneration,
+          ...hubIdentity(),
+        },
+      };
+    },
+
     route(msg) {
       const recipients = uniqueStrings(resolveRecipients(msg));
       if (!recipients.length) {
@@ -1097,7 +1497,14 @@ export function createRouter(store) {
       agent_id,
       reason = "manual",
       requested_by = "manual",
+      hub_instance_id,
+      store_fingerprint,
     } = {}) {
+      const identityFailure = validateCallerIdentity({
+        hub_instance_id,
+        store_fingerprint,
+      });
+      if (identityFailure) return identityFailure;
       const roleName = normalizeRoleName(role);
       if (!roleName) {
         return {
@@ -1168,6 +1575,7 @@ export function createRouter(store) {
           leader_epoch: snapshot.leader_epoch,
           ...transfer,
           status: snapshot,
+          ...hubIdentity(),
         },
       };
     },
@@ -1375,18 +1783,15 @@ export function createRouter(store) {
       return scheduleAssignRetry(job, reason, job.error, requested_by);
     },
 
-    sweepExpired() {
-      const now = Date.now();
-      let expired = 0;
-      const deadLettered = [];
+    sweepExpired({ now_ms = Date.now(), source = "manual" } = {}) {
+      const result = store.sweepExpired({ now_ms });
       for (const [messageId, record] of Array.from(liveMessages.entries())) {
-        if (record.message.expires_at_ms > now) continue;
-        store.moveToDeadLetter(messageId, "ttl_expired", null);
-        deadLettered.push(record.message);
+        if (record.message.expires_at_ms > now_ms) continue;
         removeMessage(messageId);
-        expired += 1;
       }
-      for (const message of deadLettered) {
+      for (const transition of result.transitions) {
+        if (!transition.dlq_inserted) continue;
+        const { message } = transition;
         if (
           !["handoff", "request"].includes(message.type) ||
           message.payload?.kind === "assign.job"
@@ -1411,12 +1816,15 @@ export function createRouter(store) {
             original_to: message.to_agent,
             reason: "ttl_expired",
             task: message.payload?.task ?? null,
+            expired_at_ms: message.expires_at_ms,
+            failed_at_ms: transition.failed_at_ms,
+            recovery_source: source,
           },
           trace_id: message.trace_id,
           correlation_id: message.correlation_id,
         });
       }
-      return { messages: expired };
+      return result;
     },
 
     sweepTimedOutAssigns() {
@@ -1437,19 +1845,49 @@ export function createRouter(store) {
       return { timed_out, retried };
     },
 
-    startSweeper() {
+    startSweeper({
+      run_immediately = false,
+      interval_ms = 10000,
+      onSweepError,
+    } = {}) {
       if (sweepTimer) return;
-      sweepTimer = setInterval(() => {
+      const reportSweepError = (source, error, failures) => {
+        if (typeof onSweepError === "function") {
+          onSweepError({ source, error, consecutive_failures: failures });
+        }
+      };
+      let sweepFailures = 0;
+      let assignFailures = 0;
+      const runSweep = () => {
+        try {
+          router.sweepExpired({ source: "periodic" });
+          sweepFailures = 0;
+        } catch (error) {
+          sweepFailures += 1;
+          reportSweepError("periodic", error, sweepFailures);
+        }
         try {
-          router.sweepExpired();
           router.sweepTimedOutAssigns();
-        } catch {}
-      }, 10000);
+          assignFailures = 0;
+        } catch (error) {
+          assignFailures += 1;
+          reportSweepError("assign_timeout", error, assignFailures);
+        }
+      };
+      if (run_immediately) runSweep();
+      sweepTimer = setInterval(() => {
+        runSweep();
+      }, interval_ms);
+      let staleFailures = 0;
       staleTimer = setInterval(() => {
         try {
           store.sweepStaleAgents();
           router.reelectStaleRoles();
-        } catch {}
+          staleFailures = 0;
+        } catch (error) {
+          staleFailures += 1;
+          reportSweepError("stale_agents", error, staleFailures);
+        }
       }, 120000);
       sweepTimer.unref();
       staleTimer.unref();
@@ -1505,8 +1943,19 @@ export function createRouter(store) {
 
     getStatus(
       scope = "hub",
-      { agent_id, trace_id, include_metrics = true } = {},
+      {
+        agent_id,
+        trace_id,
+        include_metrics = true,
+        hub_instance_id,
+        store_fingerprint,
+      } = {},
     ) {
+      const identityFailure = validateCallerIdentity({
+        hub_instance_id,
+        store_fingerprint,
+      });
+      if (identityFailure) return identityFailure;
       const data = {};
 
       if (scope === "hub" || scope === "queue") {
@@ -1515,6 +1964,7 @@ export function createRouter(store) {
           uptime_ms: (process.uptime() * 1000) | 0,
           realtime_transport: "named-pipe",
           audit_store: store.type || "sqlite",
+          ...hubIdentity(),
         };
         data.roles = {
           cto: getRoleSnapshot("cto"),
@@ -1568,7 +2018,7 @@ export function createRouter(store) {
         data.trace = store.getMessagesByTrace(trace_id);
       }
 
-      return { ok: true, data };
+      return { ok: true, data: { ...data, ...hubIdentity() } };
     },
   };
 

package/hub/team/claude-daemon-control.mjs

@@ -321,10 +321,12 @@ export async function probeClaudeDaemonCandidates({
   short,
   sessionId,
   timeoutMs = 6000,
+  tmpRoot = "/tmp",
 } = {}) {
   const candidates = await buildClaudeDaemonDiscoveryCandidates({
     configDir,
     env,
+    tmpRoot,
   });
   const callerProvenance = detectCallerProvenance(env);
   const targetRequested = Boolean(short || sessionId);

package/hub/team/intervention.mjs

@@ -5,11 +5,11 @@ import { statSync } from "node:fs";
 import { readdir, readlink } from "node:fs/promises";
 import { promisify } from "node:util";
 import { isPidAlive } from "../lib/process-utils.mjs";
-import { killProcess } from "../platform.mjs";
 import {
   resolveHardCeilingMs,
   resolveStallInterventionMs as resolveStallThresholdMs,
 } from "../lib/timeout-defaults.mjs";
+import { killProcess } from "../platform.mjs";
 
 export { resolveHardCeilingMs, resolveStallThresholdMs };