@desplega.ai/agent-swarm 1.161.0 → 1.162.0

package/src/telemetry-identity.ts

@@ -0,0 +1,129 @@
+/**
+ * API-server side of the telemetry identity envelope: the parts that need the
+ * DB. `src/telemetry.ts` stays DB-free so workers can import it.
+ *
+ * - `context.org_domain`: the email DOMAIN of the org's admin (never the email)
+ * - `context.user_role`: `admin` when the user's grants include everything, else `member`
+ *
+ * This file must not import `./be/db` (`be/db.ts` imports the trigger module
+ * that imports this one). It reads through the DB client and lazy-imports RBAC.
+ */
+import { getDbClient } from "./be/db/runtime";
+import { setTelemetryOrgDomain } from "./telemetry";
+import { emailDomain, type TelemetryUserRole } from "./telemetry-context";
+
+const ROLE_TTL_MS = 60 * 60_000;
+const ROLE_CACHE_LIMIT = 1_000;
+const DOMAIN_RECOMPUTE_DEBOUNCE_MS = 2_000;
+
+const roleCache = new Map<string, { role: TelemetryUserRole; at: number }>();
+const roleLoads = new Map<string, Promise<TelemetryUserRole | undefined>>();
+
+async function loadUserRole(userId: string): Promise<TelemetryUserRole | undefined> {
+  try {
+    // Lazy: be/rbac-roles imports be/db, which (transitively) imports this file.
+    const { getUserGrant } = await import("./be/rbac-roles");
+    const grant = await getUserGrant(userId);
+    const role: TelemetryUserRole = grant.grantsAll ? "admin" : "member";
+    if (roleCache.size >= ROLE_CACHE_LIMIT) {
+      const oldest = roleCache.keys().next().value;
+      if (oldest !== undefined) roleCache.delete(oldest);
+    }
+    roleCache.set(userId, { role, at: Date.now() });
+    return role;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Load (or refresh) a user's role. Concurrent calls for one user share one lookup. */
+export function primeUserRole(userId: string): Promise<TelemetryUserRole | undefined> {
+  const hit = roleCache.get(userId);
+  if (hit && Date.now() - hit.at < ROLE_TTL_MS) return Promise.resolve(hit.role);
+  let load = roleLoads.get(userId);
+  if (!load) {
+    load = loadUserRole(userId).finally(() => roleLoads.delete(userId));
+    roleLoads.set(userId, load);
+  }
+  return load;
+}
+
+/**
+ * Synchronous role lookup for `track()`. On a miss it starts a background load
+ * and returns undefined, so only the first event of a user can carry a null
+ * `user_role`. Emitters that are already async call `primeUserRole` first.
+ */
+export function cachedUserRole(userId: string): TelemetryUserRole | undefined {
+  const hit = roleCache.get(userId);
+  if (!hit || Date.now() - hit.at >= ROLE_TTL_MS) void primeUserRole(userId);
+  return hit?.role;
+}
+
+/**
+ * The email domain of the org's earliest-created active admin, falling back to
+ * the earliest active user with an email. A candidate whose address has no
+ * valid hostname (`admin@localhost`) is skipped. Undefined when none qualifies.
+ */
+export async function computeOrgDomain(): Promise<string | undefined> {
+  const rows = await getDbClient().query<{ email: string }>(
+    `SELECT u.email AS email
+       FROM users u
+      WHERE u.status = 'active' AND u.email IS NOT NULL AND TRIM(u.email) <> ''
+      ORDER BY EXISTS (
+                 SELECT 1 FROM principal_roles pr JOIN roles r ON r.id = pr.roleId
+                  WHERE pr.principalType = 'user' AND pr.principalId = u.id AND r.grantsAll = 1
+               ) DESC,
+               u.createdAt ASC, u.id ASC
+      LIMIT 20`,
+  );
+  for (const row of rows) {
+    const domain = emailDomain(row.email);
+    if (domain) return domain;
+  }
+  return undefined;
+}
+
+let persistDomain: ((domain: string | undefined) => Promise<void>) | null = null;
+let lastPersistedDomain: string | undefined;
+let recomputeTimer: ReturnType<typeof setTimeout> | null = null;
+
+/** Register how to persist `telemetry_org_domain`, so workers can read it. Set by the ticker. */
+export function configureOrgDomainPersistence(
+  persist: ((domain: string | undefined) => Promise<void>) | null,
+  alreadyStored?: string,
+): void {
+  persistDomain = persist;
+  lastPersistedDomain = alreadyStored;
+}
+
+/** Recompute the org domain, publish it to `track()`, and persist it when it changed. */
+export async function recomputeOrgDomain(): Promise<string | undefined> {
+  const domain = await computeOrgDomain();
+  setTelemetryOrgDomain(domain);
+  if (persistDomain && domain !== lastPersistedDomain) {
+    await persistDomain(domain);
+    lastPersistedDomain = domain;
+  }
+  return domain;
+}
+
+/** Debounced `recomputeOrgDomain` for user create, update and delete. Never throws. */
+export function scheduleOrgDomainRecompute(): void {
+  if (!persistDomain) return; // telemetry not started (or opted out)
+  if (recomputeTimer) clearTimeout(recomputeTimer);
+  recomputeTimer = setTimeout(() => {
+    recomputeTimer = null;
+    recomputeOrgDomain().catch(() => {});
+  }, DOMAIN_RECOMPUTE_DEBOUNCE_MS);
+  recomputeTimer.unref?.();
+}
+
+/** Test-only: clear module state. */
+export function _resetTelemetryIdentityForTests(): void {
+  roleCache.clear();
+  roleLoads.clear();
+  persistDomain = null;
+  lastPersistedDomain = undefined;
+  if (recomputeTimer) clearTimeout(recomputeTimer);
+  recomputeTimer = null;
+}

package/src/telemetry-snapshot.ts

@@ -0,0 +1,211 @@
+/**
+ * The daily `org.snapshot` event and the hourly telemetry ticker.
+ *
+ * A user count is org state, not a fact about an event, so it travels in one
+ * row per install per day instead of in `context` on every event. The snapshot
+ * carries counts only: no names, no emails, no user IDs.
+ *
+ * API server only. Workers never import this file.
+ */
+
+import { getDbClient } from "./be/db/runtime";
+import { isTelemetryReady, setTelemetryRoleResolver, telemetry } from "./telemetry";
+import type { PropsFor } from "./telemetry-contract";
+import {
+  cachedUserRole,
+  configureOrgDomainPersistence,
+  recomputeOrgDomain,
+} from "./telemetry-identity";
+import { scrubSecrets } from "./utils/secret-scrubber";
+
+export type OrgSnapshot = PropsFor<"org.snapshot">;
+
+export const LAST_SNAPSHOT_KEY = "telemetry_last_snapshot_at";
+const ORG_DOMAIN_KEY = "telemetry_org_domain";
+const FIRST_SNAPSHOT_DELAY_MS = 60_000;
+const TICK_INTERVAL_MS = 60 * 60_000;
+const SNAPSHOT_INTERVAL_MS = 24 * 60 * 60_000;
+const ACTIVE_WINDOW_MS = 7 * 24 * 60 * 60_000;
+
+/** Identity kinds that get their own `users_linked_*` counter. Every other kind sums into `users_linked_other`. */
+const NAMED_LINK_KINDS = ["slack", "github", "gitlab", "linear", "jira"] as const;
+
+type CountRow = { n: number };
+
+async function count(sql: string, params: unknown[] = []): Promise<number> {
+  const row = await getDbClient().get<CountRow>(sql, params as never[]);
+  return Number(row?.n ?? 0);
+}
+
+/** Read-only aggregates over the user, token, identity, task and agent tables. */
+export async function collectOrgSnapshot(now = Date.now()): Promise<OrgSnapshot> {
+  const linked = async (kind: string) =>
+    count(
+      `SELECT COUNT(DISTINCT x.userId) AS n
+         FROM user_external_ids x JOIN users u ON u.id = x.userId
+        WHERE u.status = 'active' AND x.kind = ?`,
+      [kind],
+    );
+  const since = new Date(now - ACTIVE_WINDOW_MS).toISOString();
+  const [
+    usersTotal,
+    usersSuspended,
+    usersAdmin,
+    usersWithLogin,
+    slack,
+    github,
+    gitlab,
+    linear,
+    jira,
+    other,
+    usersActive7d,
+    agentsTotal,
+  ] = await Promise.all([
+    count(`SELECT COUNT(*) AS n FROM users WHERE status = 'active'`),
+    count(`SELECT COUNT(*) AS n FROM users WHERE status = 'suspended'`),
+    count(
+      `SELECT COUNT(DISTINCT u.id) AS n
+         FROM users u
+         JOIN principal_roles pr ON pr.principalType = 'user' AND pr.principalId = u.id
+         JOIN roles r ON r.id = pr.roleId
+        WHERE u.status = 'active' AND r.grantsAll = 1`,
+    ),
+    count(
+      `SELECT COUNT(*) AS n
+         FROM users u
+        WHERE u.status = 'active'
+          AND ((u.email IS NOT NULL AND TRIM(u.email) <> '')
+               OR EXISTS (SELECT 1 FROM user_tokens t WHERE t.userId = u.id AND t.revokedAt IS NULL))`,
+    ),
+    linked("slack"),
+    linked("github"),
+    linked("gitlab"),
+    linked("linear"),
+    linked("jira"),
+    count(
+      `SELECT COUNT(DISTINCT x.userId) AS n
+         FROM user_external_ids x JOIN users u ON u.id = x.userId
+        WHERE u.status = 'active' AND x.kind NOT IN (${NAMED_LINK_KINDS.map(() => "?").join(", ")})`,
+      [...NAMED_LINK_KINDS],
+    ),
+    count(
+      `SELECT COUNT(DISTINCT requestedByUserId) AS n
+         FROM agent_tasks
+        WHERE requestedByUserId IS NOT NULL AND createdAt >= ?`,
+      [since],
+    ),
+    count(`SELECT COUNT(*) AS n FROM agents`),
+  ]);
+  return {
+    users_total: usersTotal,
+    users_suspended: usersSuspended,
+    users_admin: usersAdmin,
+    users_with_login: usersWithLogin,
+    users_linked_slack: slack,
+    users_linked_github: github,
+    users_linked_gitlab: gitlab,
+    users_linked_linear: linear,
+    users_linked_jira: jira,
+    users_linked_other: other,
+    users_active_7d: usersActive7d,
+    agents_total: agentsTotal,
+  };
+}
+
+export interface SnapshotDeps {
+  getConfig: (key: string) => Promise<string | undefined> | string | undefined;
+  setConfig: (key: string, value: string) => Promise<void> | void;
+  now?: () => number;
+}
+
+let inFlight: Promise<boolean> | null = null;
+
+/**
+ * Send one `org.snapshot` when 24 h have passed since the last one.
+ * `swarm_config.telemetry_last_snapshot_at` records the send time, so a restart
+ * does not send twice in a day. Returns true when an event was sent. Never
+ * throws. Sends nothing (and records nothing) when telemetry is off or not yet
+ * identified, so the next tick retries.
+ */
+export function emitOrgSnapshotIfDue(deps: SnapshotDeps): Promise<boolean> {
+  if (inFlight) return inFlight;
+  inFlight = (async () => {
+    try {
+      if (!isTelemetryReady()) return false;
+      const now = (deps.now ?? Date.now)();
+      const lastRaw = await deps.getConfig(LAST_SNAPSHOT_KEY);
+      const last = lastRaw ? Date.parse(lastRaw) : Number.NaN;
+      if (Number.isFinite(last) && now - last < SNAPSHOT_INTERVAL_MS) return false;
+      const snapshot = await collectOrgSnapshot(now);
+      // Record first: a crash between the two steps loses one snapshot, where the
+      // other order would send a second one on every restart loop.
+      await deps.setConfig(LAST_SNAPSHOT_KEY, new Date(now).toISOString());
+      telemetry.org("snapshot", snapshot);
+      return true;
+    } catch (err) {
+      console.error(
+        "[telemetry] org.snapshot failed:",
+        scrubSecrets(err instanceof Error ? err.message : String(err)),
+      );
+      return false;
+    } finally {
+      inFlight = null;
+    }
+  })();
+  return inFlight;
+}
+
+let tickTimer: ReturnType<typeof setInterval> | null = null;
+let firstTimer: ReturnType<typeof setTimeout> | null = null;
+
+/**
+ * Start the API server's telemetry upkeep: the role resolver for
+ * `context.user_role`, the org-domain recompute (at boot, on user changes, then
+ * hourly) and the daily `org.snapshot` (60 s after boot, then whenever the
+ * hourly tick finds 24 h have passed). One timer, no new cadence per feature.
+ * A no-op when telemetry is off.
+ */
+export function startTelemetryTicker(deps: SnapshotDeps): void {
+  if (!isTelemetryReady()) return;
+  stopTelemetryTicker();
+  setTelemetryRoleResolver(cachedUserRole);
+
+  const tick = async () => {
+    try {
+      await recomputeOrgDomain();
+    } catch (err) {
+      console.error(
+        "[telemetry] org domain recompute failed:",
+        scrubSecrets(err instanceof Error ? err.message : String(err)),
+      );
+    }
+    await emitOrgSnapshotIfDue(deps);
+  };
+
+  Promise.resolve(deps.getConfig(ORG_DOMAIN_KEY))
+    .then((stored) => {
+      configureOrgDomainPersistence(async (domain) => {
+        // An empty value clears a stale domain, so workers stop sending it.
+        await deps.setConfig(ORG_DOMAIN_KEY, domain ?? "");
+      }, stored || undefined);
+      return recomputeOrgDomain();
+    })
+    .catch(() => {});
+
+  firstTimer = setTimeout(() => {
+    firstTimer = null;
+    void emitOrgSnapshotIfDue(deps);
+  }, FIRST_SNAPSHOT_DELAY_MS);
+  firstTimer.unref?.();
+  tickTimer = setInterval(() => void tick(), TICK_INTERVAL_MS);
+  tickTimer.unref?.();
+}
+
+export function stopTelemetryTicker(): void {
+  if (tickTimer) clearInterval(tickTimer);
+  if (firstTimer) clearTimeout(firstTimer);
+  tickTimer = null;
+  firstTimer = null;
+  setTelemetryRoleResolver(null);
+  configureOrgDomainPersistence(null);
+}

package/src/telemetry-trigger.ts

@@ -0,0 +1,125 @@
+/**
+ * `trigger_surface` for task and session telemetry: the surface that started
+ * the whole chain of work, resolved at the root of the `parentTaskId` chain.
+ *
+ * A Slack ask that the lead splits into 5 worker tasks (`send-task` writes
+ * `source: "mcp"`, follow-ups write `system`) is 6 Slack tasks, not 1 Slack
+ * task and 5 MCP ones. The raw per-task value is kept as `task_source`.
+ *
+ * API server only: it reads `agent_tasks`. Workers receive the resolved value
+ * from the poll response.
+ */
+
+import { getDbClient } from "./be/db/runtime";
+import { isTelemetryEnabled, telemetry } from "./telemetry";
+import { mapTriggerSurface } from "./telemetry-context";
+import type { PropsFor, TriggerSurface } from "./telemetry-contract";
+import { primeUserRole } from "./telemetry-identity";
+import { scrubSecrets } from "./utils/secret-scrubber";
+
+const MAX_CHAIN_DEPTH = 50;
+const CACHE_LIMIT = 10_000;
+
+/** Task ID -> root surface. A task's root is fixed at creation, because `parentTaskId` is set once. */
+const rootSurfaceCache = new Map<string, TriggerSurface>();
+
+function remember(taskId: string, surface: TriggerSurface): void {
+  rootSurfaceCache.delete(taskId); // re-insert so the Map order is least-recently-used first
+  rootSurfaceCache.set(taskId, surface);
+  if (rootSurfaceCache.size > CACHE_LIMIT) {
+    const oldest = rootSurfaceCache.keys().next().value;
+    if (oldest !== undefined) rootSurfaceCache.delete(oldest);
+  }
+}
+
+/**
+ * The surface of the root task above `taskId`.
+ *
+ * One recursive CTE walks `parentTaskId` (depth capped at 50) and the deepest
+ * row wins, so an orphan whose parent row was deleted reports the deepest row
+ * that still exists. When the task itself is gone, `fallbackSource` (the
+ * caller's own task source) is used and nothing is cached.
+ */
+export async function resolveTriggerSurface(
+  taskId: string,
+  fallbackSource?: string | null,
+): Promise<TriggerSurface> {
+  const cached = rootSurfaceCache.get(taskId);
+  if (cached) {
+    remember(taskId, cached);
+    return cached;
+  }
+  const row = await getDbClient().get<{ source: string | null }>(
+    `WITH RECURSIVE chain(id, parentTaskId, source, depth) AS (
+       SELECT id, parentTaskId, source, 0 FROM agent_tasks WHERE id = ?
+       UNION ALL
+       SELECT t.id, t.parentTaskId, t.source, c.depth + 1
+         FROM agent_tasks t JOIN chain c ON t.id = c.parentTaskId
+        WHERE c.depth < ?
+     )
+     SELECT source FROM chain ORDER BY depth DESC LIMIT 1`,
+    [taskId, MAX_CHAIN_DEPTH],
+  );
+  if (!row) return mapTriggerSurface(fallbackSource);
+  const surface = mapTriggerSurface(row.source);
+  remember(taskId, surface);
+  return surface;
+}
+
+export type TaskTelemetryEvent =
+  | "created"
+  | "started"
+  | "claimed"
+  | "completed"
+  | "failed"
+  | "cancelled"
+  | "superseded";
+
+/**
+ * A task lifecycle event as call sites build it: the catalog properties minus
+ * the two derived ones, plus the task's own `source`. `emitTaskTelemetry`
+ * renames `source` to `task_source` and adds `trigger_surface`, in one place.
+ */
+export type TaskTelemetryInput<S extends TaskTelemetryEvent = TaskTelemetryEvent> =
+  S extends TaskTelemetryEvent
+    ? Omit<PropsFor<`task.${S}`>, "task_source" | "trigger_surface"> & {
+        source?: string | null;
+      }
+    : never;
+
+/**
+ * Send a `task.*` lifecycle event with `trigger_surface` and `task_source`.
+ * Every lifecycle event of a task resolves the same task ID, so `completed`
+ * reports exactly the `trigger_surface` its `created` reported. Never throws.
+ */
+export async function emitTaskTelemetry<S extends TaskTelemetryEvent>(
+  event: S,
+  input: TaskTelemetryInput<S>,
+  actorUserId?: string | null,
+): Promise<void> {
+  if (!isTelemetryEnabled()) return;
+  try {
+    const { source, ...rest } = input as TaskTelemetryInput & { taskId: string };
+    const triggerSurface = await resolveTriggerSurface(rest.taskId, source);
+    if (actorUserId) await primeUserRole(actorUserId);
+    telemetry.taskEvent(
+      event,
+      {
+        ...rest,
+        task_source: mapTriggerSurface(source),
+        trigger_surface: triggerSurface,
+      } as never,
+      actorUserId ? { userId: actorUserId } : undefined,
+    );
+  } catch (err) {
+    console.error(
+      "[telemetry] task event failed:",
+      scrubSecrets(err instanceof Error ? err.message : String(err)),
+    );
+  }
+}
+
+/** Test-only: clear the memo so a test can re-shape the task tree. */
+export function _resetTriggerSurfaceCacheForTests(): void {
+  rootSurfaceCache.clear();
+}

package/src/telemetry-types.typetest.ts

@@ -0,0 +1,40 @@
+/**
+ * Compile-time regression guard for `track()`'s catalog bound.
+ *
+ * Never imported and never run. `tsconfig.json` excludes `src/tests`, so a
+ * `@ts-expect-error` in a `*.test.ts` file is never checked; this file sits
+ * inside the program that `bun run tsc:check` compiles (CI: "Lint and Type
+ * Check"). Every `@ts-expect-error` below is an assertion: once `track()` stops
+ * rejecting the line under it, tsc reports an unused directive and CI fails.
+ */
+import { track } from "./telemetry";
+
+const taskCreated = {
+  taskId: "task-1",
+  task_source: "mcp",
+  trigger_surface: "mcp",
+  hasParent: false,
+  has_repo: false,
+  priority: 50,
+} as const;
+
+export function _trackRejectsWhatTheCatalogDoesNot(): void {
+  // Control: a catalogued event with catalogued properties compiles. If the
+  // contract types break, this line fails instead of the assertions passing.
+  track({ event: "task.created", properties: taskCreated });
+
+  track({
+    // @ts-expect-error "not.in.catalog" is not a catalogued agent-swarm event
+    event: "not.in.catalog",
+    properties: {},
+  });
+
+  track({
+    event: "task.created",
+    properties: {
+      ...taskCreated,
+      // @ts-expect-error "bogus" is not a property of task.created
+      bogus: 1,
+    },
+  });
+}