@wopr-network/platform-core 1.35.0 → 1.35.2

package/src/fleet/fleet-notification-listener.test.ts

@@ -0,0 +1,322 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { INotificationPreferencesRepository, NotificationPrefs } from "../email/notification-repository-types.js";
+import type { NotificationService } from "../email/notification-service.js";
+import type { BotFleetEvent } from "./fleet-event-emitter.js";
+import { FleetEventEmitter } from "./fleet-event-emitter.js";
+import { initFleetNotificationListener } from "./fleet-notification-listener.js";
+
+vi.mock("../config/logger.js", () => ({
+  logger: {
+    info: vi.fn(),
+    error: vi.fn(),
+    warn: vi.fn(),
+    debug: vi.fn(),
+  },
+}));
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+const DEBOUNCE_MS = 100;
+
+function makePrefs(overrides: Partial<NotificationPrefs> = {}): NotificationPrefs {
+  return {
+    billing_low_balance: true,
+    billing_receipts: true,
+    billing_auto_topup: true,
+    agent_channel_disconnect: true,
+    agent_status_changes: false,
+    account_role_changes: true,
+    account_team_invites: true,
+    fleet_updates: true,
+    ...overrides,
+  };
+}
+
+function makePrefsRepo(prefs: NotificationPrefs = makePrefs()): INotificationPreferencesRepository {
+  return {
+    get: vi.fn().mockResolvedValue(prefs),
+    update: vi.fn().mockResolvedValue(undefined),
+  } as unknown as INotificationPreferencesRepository;
+}
+
+function makeNotificationService(): NotificationService {
+  return {
+    notifyFleetUpdateComplete: vi.fn(),
+    notifyFleetUpdateAvailable: vi.fn(),
+    notifyLowBalance: vi.fn(),
+  } as unknown as NotificationService;
+}
+
+function botUpdated(tenantId: string, version = "v1.2.0"): BotFleetEvent {
+  return {
+    type: "bot.updated",
+    botId: `bot-${Math.random().toString(36).slice(2, 6)}`,
+    tenantId,
+    timestamp: new Date().toISOString(),
+    version,
+  };
+}
+
+function botUpdateFailed(tenantId: string, version = "v1.2.0"): BotFleetEvent {
+  return {
+    type: "bot.update_failed",
+    botId: `bot-${Math.random().toString(36).slice(2, 6)}`,
+    tenantId,
+    timestamp: new Date().toISOString(),
+    version,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("initFleetNotificationListener", () => {
+  let emitter: FleetEventEmitter;
+
+  beforeEach(() => {
+    vi.useFakeTimers();
+    emitter = new FleetEventEmitter();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("ignores non-bot events (node events)", async () => {
+    const notificationService = makeNotificationService();
+    const preferences = makePrefsRepo();
+
+    initFleetNotificationListener({
+      eventEmitter: emitter,
+      notificationService,
+      preferences,
+      resolveEmail: vi.fn().mockResolvedValue("user@example.com"),
+      debounceMs: DEBOUNCE_MS,
+    });
+
+    emitter.emit({
+      type: "node.provisioned",
+      nodeId: "node-1",
+      timestamp: new Date().toISOString(),
+    });
+
+    await vi.advanceTimersByTimeAsync(DEBOUNCE_MS + 50);
+
+    expect(notificationService.notifyFleetUpdateComplete).not.toHaveBeenCalled();
+  });
+
+  it("ignores non-update bot events (bot.started, bot.stopped)", async () => {
+    const notificationService = makeNotificationService();
+    const preferences = makePrefsRepo();
+
+    initFleetNotificationListener({
+      eventEmitter: emitter,
+      notificationService,
+      preferences,
+      resolveEmail: vi.fn().mockResolvedValue("user@example.com"),
+      debounceMs: DEBOUNCE_MS,
+    });
+
+    emitter.emit({
+      type: "bot.started",
+      botId: "bot-1",
+      tenantId: "t1",
+      timestamp: new Date().toISOString(),
+    });
+
+    emitter.emit({
+      type: "bot.stopped",
+      botId: "bot-2",
+      tenantId: "t1",
+      timestamp: new Date().toISOString(),
+    });
+
+    await vi.advanceTimersByTimeAsync(DEBOUNCE_MS + 50);
+
+    expect(notificationService.notifyFleetUpdateComplete).not.toHaveBeenCalled();
+  });
+
+  it("aggregates multiple bot.updated events per tenant into one notification", async () => {
+    const notificationService = makeNotificationService();
+    const preferences = makePrefsRepo();
+    const resolveEmail = vi.fn().mockResolvedValue("owner@example.com");
+
+    initFleetNotificationListener({
+      eventEmitter: emitter,
+      notificationService,
+      preferences,
+      resolveEmail,
+      debounceMs: DEBOUNCE_MS,
+    });
+
+    // Emit 3 successful updates for the same tenant
+    emitter.emit(botUpdated("t1", "v2.0.0"));
+    emitter.emit(botUpdated("t1", "v2.0.0"));
+    emitter.emit(botUpdated("t1", "v2.0.0"));
+
+    // No notification yet — debounce hasn't fired
+    expect(notificationService.notifyFleetUpdateComplete).not.toHaveBeenCalled();
+
+    await vi.advanceTimersByTimeAsync(DEBOUNCE_MS + 50);
+
+    expect(notificationService.notifyFleetUpdateComplete).toHaveBeenCalledOnce();
+    expect(notificationService.notifyFleetUpdateComplete).toHaveBeenCalledWith(
+      "t1",
+      "owner@example.com",
+      "v2.0.0",
+      3, // succeeded
+      0, // failed
+    );
+  });
+
+  it("sends summary with correct succeeded/failed counts after debounce", async () => {
+    const notificationService = makeNotificationService();
+    const preferences = makePrefsRepo();
+    const resolveEmail = vi.fn().mockResolvedValue("owner@example.com");
+
+    initFleetNotificationListener({
+      eventEmitter: emitter,
+      notificationService,
+      preferences,
+      resolveEmail,
+      debounceMs: DEBOUNCE_MS,
+    });
+
+    emitter.emit(botUpdated("t1", "v3.0.0"));
+    emitter.emit(botUpdated("t1", "v3.0.0"));
+    emitter.emit(botUpdateFailed("t1", "v3.0.0"));
+
+    await vi.advanceTimersByTimeAsync(DEBOUNCE_MS + 50);
+
+    expect(notificationService.notifyFleetUpdateComplete).toHaveBeenCalledWith(
+      "t1",
+      "owner@example.com",
+      "v3.0.0",
+      2, // succeeded
+      1, // failed
+    );
+  });
+
+  it("updates version from subsequent events", async () => {
+    const notificationService = makeNotificationService();
+    const preferences = makePrefsRepo();
+    const resolveEmail = vi.fn().mockResolvedValue("owner@example.com");
+
+    initFleetNotificationListener({
+      eventEmitter: emitter,
+      notificationService,
+      preferences,
+      resolveEmail,
+      debounceMs: DEBOUNCE_MS,
+    });
+
+    // First event has v1.0.0, subsequent event changes to v1.1.0
+    emitter.emit(botUpdated("t1", "v1.0.0"));
+    emitter.emit(botUpdated("t1", "v1.1.0"));
+
+    await vi.advanceTimersByTimeAsync(DEBOUNCE_MS + 50);
+
+    expect(notificationService.notifyFleetUpdateComplete).toHaveBeenCalledWith(
+      "t1",
+      "owner@example.com",
+      "v1.1.0", // updated to latest version
+      2,
+      0,
+    );
+  });
+
+  it("checks fleet_updates preference — skips if disabled", async () => {
+    const notificationService = makeNotificationService();
+    const preferences = makePrefsRepo(makePrefs({ fleet_updates: false }));
+    const resolveEmail = vi.fn().mockResolvedValue("owner@example.com");
+
+    initFleetNotificationListener({
+      eventEmitter: emitter,
+      notificationService,
+      preferences,
+      resolveEmail,
+      debounceMs: DEBOUNCE_MS,
+    });
+
+    emitter.emit(botUpdated("t1"));
+
+    await vi.advanceTimersByTimeAsync(DEBOUNCE_MS + 50);
+
+    expect(preferences.get).toHaveBeenCalledWith("t1");
+    expect(notificationService.notifyFleetUpdateComplete).not.toHaveBeenCalled();
+  });
+
+  it("skips if email resolver returns null", async () => {
+    const notificationService = makeNotificationService();
+    const preferences = makePrefsRepo();
+    const resolveEmail = vi.fn().mockResolvedValue(null);
+
+    initFleetNotificationListener({
+      eventEmitter: emitter,
+      notificationService,
+      preferences,
+      resolveEmail,
+      debounceMs: DEBOUNCE_MS,
+    });
+
+    emitter.emit(botUpdated("t1"));
+
+    await vi.advanceTimersByTimeAsync(DEBOUNCE_MS + 50);
+
+    expect(resolveEmail).toHaveBeenCalledWith("t1");
+    expect(notificationService.notifyFleetUpdateComplete).not.toHaveBeenCalled();
+  });
+
+  it("async shutdown flushes pending notifications", async () => {
+    const notificationService = makeNotificationService();
+    const preferences = makePrefsRepo();
+    const resolveEmail = vi.fn().mockResolvedValue("owner@example.com");
+
+    const shutdown = initFleetNotificationListener({
+      eventEmitter: emitter,
+      notificationService,
+      preferences,
+      resolveEmail,
+      debounceMs: DEBOUNCE_MS,
+    });
+
+    emitter.emit(botUpdated("t1", "v5.0.0"));
+    emitter.emit(botUpdated("t2", "v5.0.0"));
+
+    // Don't advance timers — flush via shutdown instead
+    await shutdown();
+
+    // Both tenants should have been flushed
+    expect(notificationService.notifyFleetUpdateComplete).toHaveBeenCalledTimes(2);
+
+    const calls = vi.mocked(notificationService.notifyFleetUpdateComplete).mock.calls;
+    const tenantIds = calls.map((c) => c[0]);
+    expect(tenantIds).toContain("t1");
+    expect(tenantIds).toContain("t2");
+  });
+
+  it("no further events after shutdown", async () => {
+    const notificationService = makeNotificationService();
+    const preferences = makePrefsRepo();
+    const resolveEmail = vi.fn().mockResolvedValue("owner@example.com");
+
+    const shutdown = initFleetNotificationListener({
+      eventEmitter: emitter,
+      notificationService,
+      preferences,
+      resolveEmail,
+      debounceMs: DEBOUNCE_MS,
+    });
+
+    await shutdown();
+
+    // Events after shutdown should not trigger anything
+    emitter.emit(botUpdated("t1"));
+    await vi.advanceTimersByTimeAsync(DEBOUNCE_MS + 50);
+
+    expect(notificationService.notifyFleetUpdateComplete).not.toHaveBeenCalled();
+  });
+});

package/src/fleet/init-fleet-updater.test.ts

@@ -0,0 +1,234 @@
+/**
+ * Tests the profile-filtering + onManualTenantsSkipped logic used inside initFleetUpdater.
+ *
+ * We can't import initFleetUpdater directly because it constructs ImagePoller,
+ * ContainerUpdater, etc. which pull in heavy dependencies that cause hangs.
+ * Instead, we replicate the getUpdatableProfiles closure from initFleetUpdater
+ * and test it in isolation via a RolloutOrchestrator with mock deps.
+ */
+import { describe, expect, it, vi } from "vitest";
+import type { IBotProfileRepository } from "./bot-profile-repository.js";
+import { RolloutOrchestrator } from "./rollout-orchestrator.js";
+import type { IRolloutStrategy } from "./rollout-strategy.js";
+import type { ITenantUpdateConfigRepository } from "./tenant-update-config-repository.js";
+import type { BotProfile } from "./types.js";
+import type { ContainerUpdater } from "./updater.js";
+import type { VolumeSnapshotManager } from "./volume-snapshot-manager.js";
+
+vi.mock("../config/logger.js", () => ({
+  logger: {
+    info: vi.fn(),
+    error: vi.fn(),
+    warn: vi.fn(),
+    debug: vi.fn(),
+  },
+}));
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeProfileWithFields(fields: { id: string; tenantId: string; updatePolicy: "auto" | "manual" }): BotProfile {
+  return {
+    ...fields,
+    image: "ghcr.io/org/img:latest",
+  } as unknown as BotProfile;
+}
+
+function makeProfileRepo(profiles: BotProfile[]): IBotProfileRepository {
+  return {
+    list: vi.fn().mockResolvedValue(profiles),
+    get: vi
+      .fn()
+      .mockImplementation((id: string) =>
+        Promise.resolve(profiles.find((p) => (p as unknown as { id: string }).id === id) ?? null),
+      ),
+    save: vi.fn().mockResolvedValue(undefined),
+    delete: vi.fn().mockResolvedValue(false),
+  } as unknown as IBotProfileRepository;
+}
+
+function makeConfigRepo(
+  configs: Record<string, { mode: "auto" | "manual"; preferredHourUtc: number }> = {},
+): ITenantUpdateConfigRepository {
+  return {
+    get: vi.fn().mockImplementation((tenantId: string) => {
+      const cfg = configs[tenantId];
+      return Promise.resolve(cfg ? { tenantId, ...cfg, updatedAt: Date.now() } : null);
+    }),
+    upsert: vi.fn().mockResolvedValue(undefined),
+    listAutoEnabled: vi.fn().mockResolvedValue([]),
+  };
+}
+
+function makeMockUpdater(): ContainerUpdater {
+  return {
+    updateBot: vi.fn().mockResolvedValue({
+      botId: "",
+      success: true,
+      previousImage: "",
+      newImage: "",
+      previousDigest: null,
+      newDigest: null,
+      rolledBack: false,
+    }),
+  } as unknown as ContainerUpdater;
+}
+
+function makeMockSnapshotManager(): VolumeSnapshotManager {
+  return {
+    snapshot: vi.fn(),
+    restore: vi.fn(),
+    delete: vi.fn(),
+  } as unknown as VolumeSnapshotManager;
+}
+
+function makeMockStrategy(): IRolloutStrategy {
+  return {
+    nextBatch: vi.fn().mockImplementation((remaining: BotProfile[]) => remaining),
+    pauseDuration: vi.fn().mockReturnValue(0),
+    onBotFailure: vi.fn().mockReturnValue("skip" as const),
+    maxRetries: vi.fn().mockReturnValue(0),
+    healthCheckTimeout: vi.fn().mockReturnValue(0),
+  };
+}
+
+/**
+ * Builds the same getUpdatableProfiles closure that initFleetUpdater creates,
+ * so we can test the filtering + callback logic without importing the heavy module.
+ */
+function buildGetUpdatableProfiles(
+  profileRepo: IBotProfileRepository,
+  configRepo: ITenantUpdateConfigRepository | undefined,
+  onManualTenantsSkipped: ((tenantIds: string[], imageTag: string) => void) | undefined,
+  imageTag = "v1.2.3",
+): () => Promise<BotProfile[]> {
+  return async () => {
+    const profiles = await profileRepo.list();
+
+    const manualPolicyIds: string[] = [];
+    const nonManualPolicy = profiles.filter((p) => {
+      if (p.updatePolicy === "manual") {
+        manualPolicyIds.push(p.tenantId);
+        return false;
+      }
+      return true;
+    });
+
+    if (!configRepo) {
+      if (manualPolicyIds.length > 0 && onManualTenantsSkipped) {
+        onManualTenantsSkipped([...new Set(manualPolicyIds)], imageTag);
+      }
+      return nonManualPolicy;
+    }
+
+    const configManualIds: string[] = [];
+    const results = await Promise.all(
+      nonManualPolicy.map(async (p) => {
+        const tenantCfg = await configRepo.get(p.tenantId);
+        if (tenantCfg && tenantCfg.mode === "manual") {
+          configManualIds.push(p.tenantId);
+          return null;
+        }
+        return p;
+      }),
+    );
+
+    const allManualIds = [...manualPolicyIds, ...configManualIds];
+    if (allManualIds.length > 0 && onManualTenantsSkipped) {
+      onManualTenantsSkipped([...new Set(allManualIds)], imageTag);
+    }
+
+    return results.filter((p): p is BotProfile => p !== null);
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("initFleetUpdater — onManualTenantsSkipped", () => {
+  it("callback fires with tenant IDs of manual-mode tenants (policy-based)", async () => {
+    const profiles = [
+      makeProfileWithFields({ id: "b1", tenantId: "t-manual", updatePolicy: "manual" }),
+      makeProfileWithFields({ id: "b2", tenantId: "t-auto", updatePolicy: "auto" }),
+    ];
+    const profileRepo = makeProfileRepo(profiles);
+    const onManualTenantsSkipped = vi.fn();
+
+    const orchestrator = new RolloutOrchestrator({
+      updater: makeMockUpdater(),
+      snapshotManager: makeMockSnapshotManager(),
+      strategy: makeMockStrategy(),
+      getUpdatableProfiles: buildGetUpdatableProfiles(profileRepo, undefined, onManualTenantsSkipped),
+    });
+
+    await orchestrator.rollout();
+
+    expect(onManualTenantsSkipped).toHaveBeenCalledWith(["t-manual"], "v1.2.3");
+  });
+
+  it("callback deduplicates tenant IDs", async () => {
+    const profiles = [
+      makeProfileWithFields({ id: "b1", tenantId: "t-dup", updatePolicy: "manual" }),
+      makeProfileWithFields({ id: "b2", tenantId: "t-dup", updatePolicy: "manual" }),
+      makeProfileWithFields({ id: "b3", tenantId: "t-auto", updatePolicy: "auto" }),
+    ];
+    const profileRepo = makeProfileRepo(profiles);
+    const onManualTenantsSkipped = vi.fn();
+
+    const orchestrator = new RolloutOrchestrator({
+      updater: makeMockUpdater(),
+      snapshotManager: makeMockSnapshotManager(),
+      strategy: makeMockStrategy(),
+      getUpdatableProfiles: buildGetUpdatableProfiles(profileRepo, undefined, onManualTenantsSkipped),
+    });
+
+    await orchestrator.rollout();
+
+    expect(onManualTenantsSkipped).toHaveBeenCalledWith(["t-dup"], "v1.2.3");
+  });
+
+  it("callback not called when no manual tenants exist", async () => {
+    const profiles = [
+      makeProfileWithFields({ id: "b1", tenantId: "t1", updatePolicy: "auto" }),
+      makeProfileWithFields({ id: "b2", tenantId: "t2", updatePolicy: "auto" }),
+    ];
+    const profileRepo = makeProfileRepo(profiles);
+    const onManualTenantsSkipped = vi.fn();
+
+    const orchestrator = new RolloutOrchestrator({
+      updater: makeMockUpdater(),
+      snapshotManager: makeMockSnapshotManager(),
+      strategy: makeMockStrategy(),
+      getUpdatableProfiles: buildGetUpdatableProfiles(profileRepo, undefined, onManualTenantsSkipped),
+    });
+
+    await orchestrator.rollout();
+
+    expect(onManualTenantsSkipped).not.toHaveBeenCalled();
+  });
+
+  it("callback fires with config-repo manual tenants when configRepo is provided", async () => {
+    const profiles = [
+      makeProfileWithFields({ id: "b1", tenantId: "t-cfg-manual", updatePolicy: "auto" }),
+      makeProfileWithFields({ id: "b2", tenantId: "t-cfg-auto", updatePolicy: "auto" }),
+    ];
+    const profileRepo = makeProfileRepo(profiles);
+    const configRepo = makeConfigRepo({
+      "t-cfg-manual": { mode: "manual", preferredHourUtc: 3 },
+    });
+    const onManualTenantsSkipped = vi.fn();
+
+    const orchestrator = new RolloutOrchestrator({
+      updater: makeMockUpdater(),
+      snapshotManager: makeMockSnapshotManager(),
+      strategy: makeMockStrategy(),
+      getUpdatableProfiles: buildGetUpdatableProfiles(profileRepo, configRepo, onManualTenantsSkipped),
+    });
+
+    await orchestrator.rollout();
+
+    expect(onManualTenantsSkipped).toHaveBeenCalledWith(["t-cfg-manual"], "v1.2.3");
+  });
+});

package/src/fleet/init-fleet-updater.ts

@@ -40,7 +40,7 @@ export interface FleetUpdaterConfig {
   /** Optional fleet event emitter. When provided, bot.updated / bot.update_failed events are emitted. */
   eventEmitter?: FleetEventEmitter;
   /** Called with manual-mode tenant IDs when a new image is available but they are excluded from rollout. */
-  onManualTenantsSkipped?: (tenantIds: string[]) => void;
+  onManualTenantsSkipped?: (tenantIds: string[], imageTag: string) => void;
 }
 
 export interface FleetUpdaterHandle {
@@ -91,6 +91,11 @@ export function initFleetUpdater(
   const snapshotManager = new VolumeSnapshotManager(docker, snapshotDir);
   const strategy = createRolloutStrategy(strategyType, strategyOptions);
 
+  // Captured by the onUpdateAvailable handler and read by getUpdatableProfiles.
+  // Set before each orchestrator.rollout() call so the callback receives the
+  // image tag that triggered this rollout rather than a stale "latest" placeholder.
+  let currentImageTag = "latest";
+
   const orchestrator = new RolloutOrchestrator({
     updater,
     snapshotManager,
@@ -110,7 +115,7 @@ export function initFleetUpdater(
 
       if (!configRepo) {
         if (manualPolicyIds.length > 0 && onManualTenantsSkipped) {
-          onManualTenantsSkipped([...new Set(manualPolicyIds)]);
+          onManualTenantsSkipped([...new Set(manualPolicyIds)], currentImageTag);
         }
         return nonManualPolicy;
       }
@@ -131,7 +136,7 @@ export function initFleetUpdater(
 
       const allManualIds = [...manualPolicyIds, ...configManualIds];
       if (allManualIds.length > 0 && onManualTenantsSkipped) {
-        onManualTenantsSkipped([...new Set(allManualIds)]);
+        onManualTenantsSkipped([...new Set(allManualIds)], currentImageTag);
       }
 
       return results.filter((p) => p !== null);
@@ -169,11 +174,24 @@ export function initFleetUpdater(
   // Wire the detection → orchestration pipeline.
   // Any digest change triggers a fleet-wide rollout because the managed image
   // is shared across all tenants — one new digest means all bots need updating.
-  poller.onUpdateAvailable = async (_botId: string, _newDigest: string) => {
+  poller.onUpdateAvailable = async (botId: string, _newDigest: string) => {
     if (orchestrator.isRolling) {
       logger.debug("Skipping update trigger — rollout already in progress");
       return;
     }
+
+    // Resolve the image tag from the bot that triggered the update so that
+    // onManualTenantsSkipped receives the real version instead of "latest".
+    try {
+      const triggeringProfile = await profileStore.get(botId);
+      if (triggeringProfile) {
+        const img = triggeringProfile.image;
+        currentImageTag = img.includes(":") ? (img.split(":").pop() ?? "latest") : "latest";
+      }
+    } catch {
+      // Best-effort — currentImageTag stays at previous value
+    }
+
     logger.info("New image digest detected — starting fleet-wide rollout");
     await orchestrator.rollout().catch((err) => {
       logger.error("Rollout failed", { err });