agent-relay-server 0.120.4 → 0.120.6

package/src/db/delivery.ts

@@ -474,13 +474,95 @@ export function resolveQueuedDirectMessages(agentId: string): Message[] {
  * message that died without ever reaching its recipient — as a health signal.
  */
 export function countExpiredUndeliveredMessages(now: number = Date.now(), windowMs = 86_400_000): number {
-  return (getDb().query(`
-    SELECT COUNT(*) as c FROM messages
+  return summarizeExpiredUndeliveredMessages(now, windowMs).unexplainedCount;
+}
+
+interface ExpiredUndeliveredRow {
+  id: number;
+  toTarget: string;
+  kind: string;
+  subject?: string | null;
+  policyValue?: string | null;
+  policyStatus?: string | null;
+  policyAgentId?: string | null;
+  policyLastError?: string | null;
+}
+
+export interface ExpiredUndeliveredMessagesSummary {
+  totalCount: number;
+  unexplainedCount: number;
+  explainedManagedPolicyCount: number;
+  subjects: NonNullable<HealthCheck["subjects"]>;
+}
+
+function managedPolicyExpiryExplanation(row: ExpiredUndeliveredRow): string | undefined {
+  if (!row.toTarget.startsWith("policy:")) return undefined;
+  if (typeof row.policyValue !== "string") return undefined;
+  const policy = parseJson<Record<string, unknown>>(row.policyValue, {});
+  if (policy.enabled === false) return "managed policy is disabled";
+  if (row.policyStatus === "backoff" || row.policyStatus === "failed") {
+    return `managed policy is ${row.policyStatus}${row.policyLastError ? `: ${row.policyLastError}` : ""}`;
+  }
+  return undefined;
+}
+
+/**
+ * Classify queued-message expiries that happened within the trailing window.
+ * Channel/direct/unknown expiries remain a silent-loss health warning. Managed
+ * policy expiries with an explicit unavailable policy state are explained by the
+ * policy health surface instead, so they stay visible without degrading global
+ * health as generic message loss (#998).
+ */
+export function summarizeExpiredUndeliveredMessages(now: number = Date.now(), windowMs = 86_400_000): ExpiredUndeliveredMessagesSummary {
+  const rows = getDb().query(`
+    SELECT
+      m.id,
+      m.to_target AS toTarget,
+      m.kind,
+      m.subject,
+      c.value AS policyValue,
+      s.status AS policyStatus,
+      s.agent_id AS policyAgentId,
+      s.last_error AS policyLastError
+    FROM messages m
+    LEFT JOIN config c
+      ON c.namespace = 'spawn-policy'
+     AND c.key = substr(m.to_target, 8)
+    LEFT JOIN managed_agent_state s
+      ON s.policy_name = substr(m.to_target, 8)
     WHERE delivery_status = 'dead'
       AND delivery_poison_reason = 'queued message expired'
       AND delivery_updated_at IS NOT NULL
       AND delivery_updated_at >= ?
-  `).get(now - windowMs) as { c: number }).c;
+  `).all(now - windowMs) as ExpiredUndeliveredRow[];
+
+  let unexplainedCount = 0;
+  let explainedManagedPolicyCount = 0;
+  const explainedByTarget = new Map<string, { count: number; detail: string }>();
+
+  for (const row of rows) {
+    const explanation = managedPolicyExpiryExplanation(row);
+    if (!explanation) {
+      unexplainedCount += 1;
+      continue;
+    }
+    explainedManagedPolicyCount += 1;
+    const existing = explainedByTarget.get(row.toTarget);
+    if (existing) existing.count += 1;
+    else explainedByTarget.set(row.toTarget, { count: 1, detail: explanation });
+  }
+
+  return {
+    totalCount: rows.length,
+    unexplainedCount,
+    explainedManagedPolicyCount,
+    subjects: [...explainedByTarget.entries()].slice(0, 10).map(([target, item]) => ({
+      id: target,
+      label: target,
+      status: "explained",
+      detail: `${item.count} expired message(s); ${item.detail}`,
+    })),
+  };
 }
 
 export function expireQueuedMessages(now: number = Date.now()): Message[] {

package/src/db/index.ts

@@ -39,6 +39,8 @@ export * from "./blackboard.ts";
 export * from "./projects.ts";
 export * from "./project-entries.ts";
 export * from "./provisioning-registry.ts";
+export * from "./provisioning-resolve.ts";
+export * from "./provisioning-callmux.ts";
 export * from "./provider-quotas.ts";
 export * from "./provider-quota-leases.ts";
 export * from "./scheduled-timers.ts";

package/src/db/migrations.ts

@@ -22,7 +22,8 @@ import { ensureDriveLeaseSchema } from "./drive-leases.ts";
 import { ensureContinuationArchiveSearchSchema } from "./continuation-archives.ts";
 import { ensureCrashReportsSchema } from "./crash-reports.ts";
 import { ensureTerminalEventsSchema } from "./terminal-events.ts";
-import { seedBundledProvisioningAssets, seedSharedCallmuxProvisioningAssets } from "./provisioning-registry.ts";
+import { seedBundledProvisioningAssets } from "./provisioning-registry.ts";
+import { seedSharedCallmuxProvisioningAssets } from "./provisioning-callmux.ts";
 import { provisioningAssetsColumnNames, provisioningAssetsNeedsRebuild, rebuildProvisioningAssetsTable } from "./provisioning-assets-migration.ts";
 import { normalizeReactionEmoji } from "./mappers.ts";
 import { normalizeProviderQuotaAccountKeys } from "./provider-quotas.ts";

package/src/db/provisioning-callmux.ts

@@ -0,0 +1,180 @@
+// Shared-callmux registry seeding (#849/#871) — split out of provisioning-registry.ts
+// under the file-size ratchet (epic #291). Reads the host callmux source config,
+// scrubs secrets into env placeholders (literal non-secret values survive only via
+// the per-server configEnv allowlist), and seeds/updates the relay-owned registry
+// rows for the shared callmux MCP servers. The validation invariants the scrub must
+// satisfy (assertConfigEnv / assertCallmuxServerPlaceholders + the allowlist) stay
+// in provisioning-registry.ts beside the definition validator that enforces them.
+
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { errMessage, isRecord } from "agent-relay-sdk";
+import type { ProvisioningAsset, ProvisioningMcpServer } from "agent-relay-sdk";
+import { getDb, ValidationError } from "./connection.ts";
+import { parseJson } from "../utils";
+import {
+  isEnvPlaceholder,
+  SHARED_CALLMUX_CONFIG_ENV_ALLOWLIST,
+  upsertProvisioningAsset,
+} from "./provisioning-registry.ts";
+import { stringRecord } from "./provisioning-resolve.ts";
+
+const HOST_CALLMUX_SOURCE_TYPE = "host-callmux";
+export const SHARED_CALLMUX_SEED_SERVER_NAMES = ["tokenlean", "github", "vent", "reasoning", "scribe", "searxng", "qmd"] as const;
+export const SHARED_CALLMUX_SOURCE_CONFIG_ENV = "AGENT_RELAY_SHARED_CALLMUX_SOURCE_CONFIG";
+// Servers seeded from the host callmux config but NOT eligible for the shared listener (deno not provisioned, etc.).
+const SHARED_CALLMUX_SEED_NON_LISTENER_NAMES: ReadonlySet<string> = new Set(["scribe"]);
+
+export interface SharedCallmuxSeedResult {
+  sourceConfigPath: string;
+  seeded: string[];
+  skipped: Array<{ name: string; reason: string }>;
+}
+
+export function sharedCallmuxSourceConfigPath(env: Record<string, string | undefined> = process.env): string {
+  return env[SHARED_CALLMUX_SOURCE_CONFIG_ENV] || env.CALLMUX_CONFIG || join(homedir(), ".config", "callmux", "config.json");
+}
+
+function cloneJsonRecord(value: Record<string, unknown>): Record<string, unknown> {
+  return JSON.parse(JSON.stringify(value)) as Record<string, unknown>;
+}
+
+function scrubEnvPlaceholders(serverName: string, value: unknown): { env: Record<string, string> | undefined; configEnv: Record<string, string> | undefined } {
+  if (!isRecord(value)) return { env: undefined, configEnv: undefined };
+  const env: Record<string, string> = {};
+  const configEnv: Record<string, string> = {};
+  for (const [key, raw] of Object.entries(value)) {
+    if (typeof raw !== "string") continue;
+    if (isEnvPlaceholder(raw)) {
+      env[key] = raw;
+    } else if (SHARED_CALLMUX_CONFIG_ENV_ALLOWLIST[serverName]?.includes(key)) {
+      configEnv[key] = raw;
+    } else {
+      env[key] = `$${key}`;
+    }
+  }
+  return {
+    env: Object.keys(env).length ? env : undefined,
+    configEnv: Object.keys(configEnv).length ? configEnv : undefined,
+  };
+}
+
+function envPlaceholderForLiteral(value: string, env: Record<string, string | undefined>): string | undefined {
+  for (const [key, candidate] of Object.entries(env)) {
+    if (!candidate || candidate !== value || !/^[A-Z_][A-Z0-9_]*$/.test(key)) continue;
+    return `$${key}`;
+  }
+  return undefined;
+}
+
+function envPlaceholderName(value: string): string | undefined {
+  const match = value.match(/^\$?\{?([A-Z_][A-Z0-9_]*)\}?$/);
+  return match?.[1];
+}
+
+function scrubHeaderPlaceholders(value: unknown, env: Record<string, string | undefined>): Record<string, string> | undefined {
+  if (!isRecord(value)) return undefined;
+  const headers: Record<string, string> = {};
+  for (const [key, raw] of Object.entries(value)) {
+    if (typeof raw !== "string") continue;
+    if (!/authorization|token|secret|key|api/i.test(key) || isEnvPlaceholder(raw) || /^Bearer \$\{[A-Z_][A-Z0-9_]*\}$/.test(raw)) {
+      headers[key] = raw;
+      continue;
+    }
+    const bearer = raw.match(/^Bearer\s+(.+)$/i);
+    const bearerValue = bearer?.[1];
+    const bearerPlaceholderName = bearerValue ? envPlaceholderName(bearerValue) : undefined;
+    if (bearerPlaceholderName) {
+      headers[key] = `Bearer \${${bearerPlaceholderName}}`;
+      continue;
+    }
+    const placeholder = envPlaceholderForLiteral(bearerValue ?? raw, env);
+    if (placeholder) headers[key] = bearer ? `Bearer ${placeholder.startsWith("${") ? placeholder : `\${${placeholder.slice(1)}}`}` : placeholder;
+  }
+  return Object.keys(headers).length ? headers : undefined;
+}
+
+function scrubCallmuxServerSecrets(serverName: string, server: Record<string, unknown>, env: Record<string, string | undefined>): Record<string, unknown> {
+  const scrubbed = cloneJsonRecord(server);
+  const { env: scrubbedEnv, configEnv } = scrubEnvPlaceholders(serverName, scrubbed.env);
+  const scrubbedHeaders = scrubHeaderPlaceholders(scrubbed.headers, env);
+  if (scrubbedEnv !== undefined) scrubbed.env = scrubbedEnv;
+  else delete scrubbed.env;
+  if (configEnv !== undefined) scrubbed.configEnv = configEnv;
+  else delete scrubbed.configEnv;
+  if (scrubbedHeaders) scrubbed.headers = scrubbedHeaders;
+  else delete scrubbed.headers;
+  return scrubbed;
+}
+
+function mcpServerFromCallmuxServer(server: Record<string, unknown>): ProvisioningMcpServer {
+  const projected: ProvisioningMcpServer = {};
+  if (typeof server.command === "string") {
+    projected.type = "stdio";
+    projected.command = server.command;
+    if (Array.isArray(server.args)) projected.args = server.args.filter((arg): arg is string => typeof arg === "string");
+    if (isRecord(server.env)) projected.env = stringRecord(server.env);
+    if (isRecord(server.configEnv)) projected.configEnv = stringRecord(server.configEnv);
+  } else if (typeof server.url === "string") {
+    projected.type = server.transport === "sse" ? "sse" : "http";
+    projected.url = server.url;
+    if (isRecord(server.headers)) projected.headers = stringRecord(server.headers);
+    if (isRecord(server.configEnv)) projected.configEnv = stringRecord(server.configEnv);
+  }
+  return projected;
+}
+
+function updateSharedCallmuxAssetMetadata(asset: ProvisioningAsset, sourceConfigPath: string, callmuxServer: Record<string, unknown>, sharedListenerEligible = true): void {
+  if (!asset.capabilityId) throw new ValidationError("shared callmux seed missing capability id");
+  const now = Date.now();
+  const provenance = {
+    importedBy: "agent-relay-shared-callmux-seed",
+    sharedListenerEligible,
+    callmux: { sourceConfigPath, serverName: asset.name },
+  };
+  getDb().query(`
+    UPDATE provisioning_assets
+    SET source_type = ?, source_ref = ?, provenance = ?, updated_at = ?
+    WHERE id = ?
+  `).run(HOST_CALLMUX_SOURCE_TYPE, sourceConfigPath, JSON.stringify(provenance), now, asset.id);
+  getDb().query("UPDATE provisioning_capabilities SET title = ?, description = ?, updated_at = ? WHERE id = ?")
+    .run(asset.name, `Host callmux MCP server ${asset.name}`, now, asset.capabilityId);
+}
+
+export function seedSharedCallmuxProvisioningAssets(
+  input: { sourceConfigPath?: string; env?: Record<string, string | undefined> } = {},
+): SharedCallmuxSeedResult {
+  const sourceConfigPath = input.sourceConfigPath ?? sharedCallmuxSourceConfigPath(input.env);
+  const result: SharedCallmuxSeedResult = { sourceConfigPath, seeded: [], skipped: [] };
+  if (!existsSync(sourceConfigPath)) return result;
+  const source = parseJson(readFileSync(sourceConfigPath, "utf8"), {}) as unknown;
+  const sourceServers = isRecord(source) && isRecord(source.servers) ? source.servers : {};
+  for (const name of SHARED_CALLMUX_SEED_SERVER_NAMES) {
+    const rawServer = sourceServers[name];
+    if (!isRecord(rawServer)) {
+      result.skipped.push({ name, reason: "missing" });
+      continue;
+    }
+    const callmuxServer = scrubCallmuxServerSecrets(name, rawServer, input.env ?? process.env);
+    const server = mcpServerFromCallmuxServer(callmuxServer);
+    try {
+      const sharedListenerEligible = !SHARED_CALLMUX_SEED_NON_LISTENER_NAMES.has(name);
+      const asset = upsertProvisioningAsset({
+        kind: "mcp",
+        name,
+        definition: {
+          kind: "mcp",
+          name,
+          server,
+          metadata: { sharedListenerEligible, callmuxServer },
+        },
+      });
+      updateSharedCallmuxAssetMetadata(asset, sourceConfigPath, callmuxServer, sharedListenerEligible);
+      result.seeded.push(name);
+    } catch (err) {
+      result.skipped.push({ name, reason: errMessage(err) });
+    }
+  }
+  return result;
+}