@k2b/cloud 0.6.0-rc.0

package/src/services/account-lifecycle/ipa-backfill.ts

@@ -0,0 +1,152 @@
+import type { JobContext, PumpConfig, Sync } from "@k2b/sync";
+import { sql } from "bun";
+import { freeipa } from "../../server/services/freeipa";
+import { getConfiguredExpiryDays } from "../account-model";
+import { getFreeIpaConfig } from "../freeipa-config";
+import { logger } from "../logging";
+
+const log = logger("auth:ipa:backfill");
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+export type IpaBackfillInput = { runId: string; minimumExpiry: string; createdBefore: string } | null;
+type IpaBackfillAccount = { userId: string; uid: string; minimumExpiry: string };
+type IpaBackfillItem = { key: string; uid: string };
+
+/** Capture one target before accepting work; retries never move it forward. */
+export const prepareIpaBackfill = async (runId: string): Promise<IpaBackfillInput> => {
+  const config = await getFreeIpaConfig();
+  if (!config.enabled) return null;
+  if (!config.configured) {
+    throw new Error(`FreeIPA is enabled but not fully configured. Missing: ${config.missingSettings.join(", ")}.`);
+  }
+  const days = await getConfiguredExpiryDays("ipa", "user");
+  if (days <= 0) return null;
+  const minimumExpiry = new Date(Date.now() + Math.max(days, 7) * DAY_MS);
+  minimumExpiry.setUTCHours(23, 59, 59, 0);
+  // Preserve Postgres timestamp precision when bounding the finite scan.
+  const [snapshot] = await sql<{ created_before: string }[]>`SELECT CURRENT_TIMESTAMP::text AS created_before`;
+  if (!snapshot) throw new Error("Could not capture the IPA backfill cutoff");
+  return { runId, minimumExpiry: minimumExpiry.toISOString(), createdBefore: snapshot.created_before };
+};
+
+export const ipaBackfillConfig: Omit<PumpConfig<IpaBackfillInput, string, IpaBackfillItem>, "dispatch"> = {
+  id: "auth:ipa:backfill",
+  owner: "core",
+  // Keep one bounded producer run at a time; account jobs own directory writes.
+  maxActiveRuns: 1,
+  dispatchConcurrency: 1,
+  leaseMs: 300_000,
+  retry: { maxAttempts: 2, backoffMs: [2000, 4000] },
+  pull: async ({ input, cursor, limit, signal }) => {
+    signal.throwIfAborted();
+    if (!input) return { items: [], nextCursor: null };
+    const rows = await sql<{ id: string; uid: string }[]>`
+      SELECT id, uid
+      FROM auth.users
+      WHERE provider = 'ipa'
+        AND created_at <= ${input.createdBefore}::timestamptz
+        AND (${cursor}::uuid IS NULL OR id > ${cursor}::uuid)
+        AND (account_expires IS NULL OR account_expires < ${input.minimumExpiry}::timestamptz)
+      ORDER BY id
+      LIMIT ${limit}
+    `;
+    signal.throwIfAborted();
+    return {
+      items: rows.map((row) => ({ key: row.id, uid: row.uid })),
+      nextCursor: rows.length === limit ? (rows.at(-1)?.id ?? null) : null,
+    };
+  },
+};
+
+export const processIpaBackfillAccount = async ({
+  input,
+  signal,
+}: Pick<JobContext<IpaBackfillAccount>, "input" | "signal">): Promise<void> => {
+  signal.throwIfAborted();
+  const config = await getFreeIpaConfig();
+  // Disabling the provider fails accepted work through the retry policy;
+  // it must not checkpoint unfinished directory effects as successful.
+  if (!config.enabled || !config.configured) throw new Error("FreeIPA backfill requires an enabled, configured provider");
+  const minimumExpiry = new Date(input.minimumExpiry);
+  await sql.begin(async (tx) => {
+    const [account] = await tx<{ account_expires: Date | null }[]>`
+        SELECT account_expires
+        FROM auth.users
+        WHERE id = ${input.userId}::uuid AND uid = ${input.uid} AND provider = 'ipa'
+        FOR UPDATE
+      `;
+    signal.throwIfAborted();
+    // The durable item identifies the original account, never a replacement
+    // with the same username. A later local expiry needs no backfill.
+    if (!account || (account.account_expires && account.account_expires >= minimumExpiry)) return;
+    const ipaSession = await freeipa.session.getServiceSession({ ...config, signal });
+    const remote = await freeipa.client.call({
+      url: config.url,
+      ipaSession,
+      method: "user_show",
+      args: [input.uid],
+      options: { all: true },
+      signal,
+    });
+    if (remote.error) throw new Error(`IPA backfill read failed for ${input.uid}: ${remote.error.message}`);
+    const record = remote.result?.result;
+    if (
+      !record ||
+      typeof record !== "object" ||
+      Array.isArray(record) ||
+      !("uid" in record) ||
+      freeipa.util.str(record.uid) !== input.uid
+    ) {
+      throw new Error(`IPA backfill received an invalid account for ${input.uid}`);
+    }
+    const expiryValue = "krbprincipalexpiration" in record ? record.krbprincipalexpiration : undefined;
+    const remoteExpiry = freeipa.util.parseGeneralizedTime(expiryValue);
+    if (freeipa.util.str(expiryValue) && !remoteExpiry) {
+      throw new Error(`IPA backfill received an invalid expiry for ${input.uid}`);
+    }
+    signal.throwIfAborted();
+    const target = remoteExpiry && remoteExpiry >= minimumExpiry ? remoteExpiry : minimumExpiry;
+    if (target === minimumExpiry) {
+      const response = await freeipa.client.call({
+        url: config.url,
+        ipaSession,
+        method: "user_mod",
+        args: [input.uid],
+        options: { krbprincipalexpiration: freeipa.util.toGeneralizedTime(target) },
+        signal,
+      });
+      if (response.error) throw new Error(`IPA backfill failed for ${input.uid}: ${response.error.message}`);
+    }
+    signal.throwIfAborted();
+    await tx`
+        UPDATE auth.users SET account_expires = ${target} WHERE id = ${input.userId}::uuid
+      `;
+    await tx`
+        INSERT INTO auth.user_ipa_data (user_id, synced_at)
+        VALUES (${input.userId}::uuid, now())
+        ON CONFLICT (user_id) DO UPDATE SET synced_at = EXCLUDED.synced_at
+      `;
+    signal.throwIfAborted();
+  });
+  log.info("IPA expiry backfill account complete", { userId: input.userId, uid: input.uid, minimumExpiry: input.minimumExpiry });
+};
+
+export const declareIpaBackfill = (sync: Sync) => {
+  const accounts = sync.job<IpaBackfillAccount>({
+    id: "auth:ipa:backfill:account",
+    owner: "core",
+    delivery: { maxInFlight: 1, ackWaitMs: 300_000, maxAttempts: 2, backoffMs: [2000, 4000] },
+  });
+  const pump = sync.pump({
+    ...ipaBackfillConfig,
+    dispatch: async ({ input, item, signal }) => {
+      signal.throwIfAborted();
+      if (!input) return;
+      await accounts.submit({
+        key: `${input.runId}:${item.key}`,
+        input: { userId: item.key, uid: item.uid, minimumExpiry: input.minimumExpiry },
+      });
+    },
+  });
+  return { pump, accounts };
+};

package/src/services/account-lifecycle/notification-sender.ts

@@ -0,0 +1,13 @@
+import type { TypedNotificationSendResult } from "../notifications/platform";
+
+export type AccountLifecycleNotificationSender = {
+  sendExpiryReminder: (input: {
+    reminderId: string;
+    userId: string;
+    firstName: string;
+    displayName: string;
+    expiresAt: string;
+    accountKind: "ipa" | "local-user" | "local-guest";
+    locale?: string;
+  }) => Promise<Pick<TypedNotificationSendResult, "id" | "status">>;
+};

package/src/services/account-lifecycle/scheduler.ts

@@ -0,0 +1,506 @@
+import type { DeliveryConfig, JobContext, Worker } from "@k2b/sync";
+import { lazySync } from "../../_internal/process-sync";
+import { logger, logging, trace } from "../logging";
+import { providers } from "../providers";
+import { get as getSetting } from "../settings";
+import { accountLifecycle } from "./index";
+import { declareIpaBackfill, prepareIpaBackfill, processIpaBackfillAccount } from "./ipa-backfill";
+import type { AccountLifecycleNotificationSender } from "./notification-sender";
+
+const log = logger("auth:lifecycle:scheduler");
+const ipaSyncLog = logger("auth:ipa:sync");
+const reminderLog = logger("auth:reminder:daily");
+const guestCleanupLog = logger("auth:guest:cleanup");
+const localUserCleanupLog = logger("auth:local-user:cleanup");
+const auditCleanupLog = logger("auth:lifecycle:audit:cleanup");
+const localUserBackfillLog = logger("auth:local-user:backfill");
+const guestBackfillLog = logger("auth:guest:backfill");
+const logCleanupLog = logger("logging");
+const DEFAULT_IPA_SYNC_CRON = "*/5 * * * *";
+const IPA_SYNC_LEASE_MS = 120_000;
+// Scheduled scans retry in minutes: a short FreeIPA or database blip must not
+// dead-letter a slot; anything longer surfaces once as a dead letter and the
+// next slot retries anyway. Sync releases a coalesced key when it dead-letters
+// the run (job `onDeadLetter`), so the stable `scheduled` key is not pinned
+// for the dead letter's retention and the next slot can submit again.
+const SCAN_RETRY = { maxAttempts: 3, backoffMs: [60_000, 300_000] } satisfies Partial<DeliveryConfig>;
+const ipaSyncMutex = lazySync((sync) => sync.mutex({ id: "auth:ipa:sync", ttlMs: IPA_SYNC_LEASE_MS, retry: { maxAttempts: 1 } }));
+let notificationSender: AccountLifecycleNotificationSender | null = null;
+
+type JobSummary = {
+  scanned: number;
+  changed: number;
+  skipped: number;
+  failed: number;
+};
+
+const abortedSummary = (): JobSummary => ({ scanned: 0, changed: 0, skipped: 0, failed: 0 });
+
+const toDemotionLog = (summary: JobSummary) => ({
+  expiredCandidates: summary.scanned,
+  demotedToGuest: summary.changed,
+  skipped: summary.skipped,
+  failed: summary.failed,
+});
+
+const toReminderLog = (summary: JobSummary) => ({
+  candidates: summary.scanned,
+  sent: summary.changed,
+  skipped: summary.skipped,
+  failed: summary.failed,
+});
+
+const toCleanupLog = (summary: JobSummary) => ({
+  candidates: summary.scanned,
+  deleted: summary.changed,
+  skipped: summary.skipped,
+  failed: summary.failed,
+});
+
+const toBackfillLog = (summary: JobSummary) => ({
+  candidates: summary.scanned,
+  updated: summary.changed,
+  skipped: summary.skipped,
+  failed: summary.failed,
+});
+
+/** Presentation of the run span Sync already opened for this job (`trace.syncSpanKey`). */
+const jobSpan = (ctx: JobContext<null>, id: string, name: string, source = id) => ({
+  name,
+  source,
+  appId: "core",
+  category: "job" as const,
+  kind: "consumer" as const,
+  spanKey: trace.syncSpanKey("job", id, ctx.jobId),
+});
+
+const getCronSetting = async (key: string, fallback: string): Promise<string> => {
+  const value = String((await getSetting<string>(key)) || "").trim();
+  return value.length > 0 ? value : fallback;
+};
+
+const getTimezoneSetting = async (): Promise<string> => {
+  const value = String((await getSetting<string>("app.timezone")) || "").trim();
+  return value.length > 0 ? value : "Europe/Berlin";
+};
+
+// ── Jobs ───────────────────────────────────────────────────────────────
+
+const ipaSyncJob = lazySync((sync) =>
+  sync.job<null>({
+    id: "auth:ipa:sync",
+    delivery: { ackWaitMs: IPA_SYNC_LEASE_MS, ...SCAN_RETRY },
+  }),
+);
+const processIpaSyncJob = async (ctx: JobContext<null>): Promise<void> => {
+  await trace.withSpan(
+    jobSpan(ctx, "auth:ipa:sync", "FreeIPA account sync"),
+    async () => {
+      if (ctx.signal.aborted) return abortedSummary();
+      const lock = await ipaSyncMutex().acquire({ resource: "global" });
+      if (!lock) {
+        ipaSyncLog.info("Sync skipped", { reason: "already_running" });
+        return abortedSummary();
+      }
+      const heartbeat = async () => {
+        await ctx.heartbeat();
+        if (!(await ipaSyncMutex().extend(lock, { ttlMs: IPA_SYNC_LEASE_MS }))) {
+          throw new Error("Lost FreeIPA synchronization lock");
+        }
+      };
+      try {
+        try {
+          await providers.ipa.sync.run({ signal: ctx.signal, heartbeat });
+        } catch (error) {
+          ipaSyncLog.error("Sync step failed", { step: "sync", error: error instanceof Error ? error.message : String(error) });
+          throw error;
+        }
+        await heartbeat();
+        try {
+          const summary = await accountLifecycle.demoteExpiredIpaUsers({ signal: ctx.signal, heartbeat });
+          ipaSyncLog.info("Expired IPA demotion complete", toDemotionLog(summary));
+          return summary;
+        } catch (error) {
+          ipaSyncLog.error("Expired IPA demotion step failed", {
+            step: "demote-expired",
+            error: error instanceof Error ? error.message : String(error),
+          });
+          throw error;
+        }
+      } finally {
+        await ipaSyncMutex()
+          .release(lock)
+          .catch((error) => {
+            ipaSyncLog.error("Failed to release synchronization lock", {
+              error: error instanceof Error ? error.message : String(error),
+            });
+          });
+      }
+    },
+    { summarize: (summary) => summary },
+  );
+};
+
+const reminderJob = lazySync((sync) =>
+  sync.job<null>({
+    id: "auth:reminder:daily",
+    delivery: { ackWaitMs: 180_000, ...SCAN_RETRY },
+  }),
+);
+const processReminderJob = async (ctx: JobContext<null>): Promise<void> => {
+  await trace.withSpan(
+    jobSpan(ctx, "auth:reminder:daily", "Account expiry reminders"),
+    async () => {
+      if (ctx.signal.aborted) return abortedSummary();
+      if (!notificationSender) throw new Error("Account lifecycle notification sender is not configured");
+      const summary = await accountLifecycle.sendExpiryReminders(notificationSender);
+      reminderLog.info("Reminder run complete", toReminderLog(summary));
+      return summary;
+    },
+    { summarize: (summary) => summary },
+  );
+};
+
+const guestCleanupJob = lazySync((sync) =>
+  sync.job<null>({
+    id: "auth:guest:cleanup",
+    delivery: { ackWaitMs: 120_000, ...SCAN_RETRY },
+  }),
+);
+const processGuestCleanupJob = async (ctx: JobContext<null>): Promise<void> => {
+  await trace.withSpan(
+    jobSpan(ctx, "auth:guest:cleanup", "Expired guest cleanup"),
+    async () => {
+      if (ctx.signal.aborted) return abortedSummary();
+      const summary = await accountLifecycle.cleanupExpiredGuests();
+      guestCleanupLog.info("Expired guest cleanup complete", toCleanupLog(summary));
+      return summary;
+    },
+    { summarize: (summary) => summary },
+  );
+};
+
+const localUserCleanupJob = lazySync((sync) =>
+  sync.job<null>({
+    id: "auth:local-user:cleanup",
+    delivery: { ackWaitMs: 120_000, ...SCAN_RETRY },
+  }),
+);
+const processLocalUserCleanupJob = async (ctx: JobContext<null>): Promise<void> => {
+  await trace.withSpan(
+    jobSpan(ctx, "auth:local-user:cleanup", "Expired local user cleanup"),
+    async () => {
+      if (ctx.signal.aborted) return abortedSummary();
+      const summary = await accountLifecycle.cleanupExpiredLocalUsers();
+      localUserCleanupLog.info("Expired local user cleanup complete", toCleanupLog(summary));
+      return summary;
+    },
+    { summarize: (summary) => summary },
+  );
+};
+
+const auditCleanupJob = lazySync((sync) =>
+  sync.job<null>({
+    id: "auth:lifecycle:audit:cleanup",
+    delivery: { ackWaitMs: 120_000, ...SCAN_RETRY },
+  }),
+);
+const processAuditCleanupJob = async (ctx: JobContext<null>): Promise<void> => {
+  await trace.withSpan(
+    jobSpan(ctx, "auth:lifecycle:audit:cleanup", "Lifecycle audit cleanup"),
+    async () => {
+      if (ctx.signal.aborted) return abortedSummary();
+      const summary = await accountLifecycle.cleanupLifecycleAudit();
+      auditCleanupLog.info("Lifecycle audit cleanup complete", toCleanupLog(summary));
+      return summary;
+    },
+    { summarize: (summary) => summary },
+  );
+};
+
+const logCleanupJob = lazySync((sync) =>
+  sync.job<null>({
+    id: "app:logs:cleanup",
+    delivery: { ackWaitMs: 120_000, ...SCAN_RETRY },
+  }),
+);
+const processLogCleanupJob = async (ctx: JobContext<null>): Promise<void> => {
+  await trace.withSpan(
+    jobSpan(ctx, "app:logs:cleanup", "Log cleanup", "logging"),
+    async () => {
+      if (ctx.signal.aborted) return { deleted: 0, retentionDays: 0 };
+      const configured = Number((await getSetting<number | string | null>("logs.retention_days")) ?? 30);
+      const retentionDays = Number.isFinite(configured) ? configured : 30;
+      const summary = await logging.cleanup(retentionDays);
+      logCleanupLog.info("Log cleanup complete", { deleted: summary.deleted, retentionDays });
+      return { deleted: summary.deleted, retentionDays };
+    },
+    { summarize: (summary) => summary },
+  );
+};
+
+const ipaBackfill = lazySync(declareIpaBackfill);
+
+const guestBackfillJob = lazySync((sync) =>
+  sync.job<null>({
+    id: "auth:guest:backfill",
+    delivery: { ackWaitMs: 300_000, maxAttempts: 2, backoffMs: [2000, 4000] },
+  }),
+);
+const processGuestBackfillJob = async (ctx: JobContext<null>): Promise<void> => {
+  await trace.withSpan(
+    jobSpan(ctx, "auth:guest:backfill", "Guest expiry backfill"),
+    async () => {
+      if (ctx.signal.aborted) return abortedSummary();
+      const summary = await accountLifecycle.runGuestBackfill();
+      guestBackfillLog.info("Guest expiry backfill complete", toBackfillLog(summary));
+      return summary;
+    },
+    { summarize: (summary) => summary },
+  );
+};
+
+const localUserBackfillJob = lazySync((sync) =>
+  sync.job<null>({
+    id: "auth:local-user:backfill",
+    delivery: { ackWaitMs: 300_000, maxAttempts: 2, backoffMs: [2000, 4000] },
+  }),
+);
+const processLocalUserBackfillJob = async (ctx: JobContext<null>): Promise<void> => {
+  await trace.withSpan(
+    jobSpan(ctx, "auth:local-user:backfill", "Local user expiry backfill"),
+    async () => {
+      if (ctx.signal.aborted) return abortedSummary();
+      const summary = await accountLifecycle.runLocalUserBackfill();
+      localUserBackfillLog.info("Local user expiry backfill complete", toBackfillLog(summary));
+      return summary;
+    },
+    { summarize: (summary) => summary },
+  );
+};
+
+// ── Scheduler ──────────────────────────────────────────────────────────
+
+// A tick that fails to submit its job retries once shortly after; with a single
+// attempt the daily slot would be dropped until the next day.
+const lifecycleScheduler = lazySync((sync) => sync.scheduler({ id: "auth-lifecycle", delivery: { maxAttempts: 2, backoffMs: [5_000] } }));
+
+let started = false;
+let workers: Worker[] = [];
+let registered = false;
+let registerPromise: Promise<void> | null = null;
+
+/**
+ * Register (or update) a cron-triggered schedule that fans out to the given
+ * job. `scheduler.create` is idempotent by id — same cron/tz keeps `nextRunAt`
+ * intact; a change resets it. Each slot submits with the job's stable
+ * `scheduled` key and `coalesce`: at most one scan is queued or running, a slot
+ * that fires during a run joins it, and the key is released when the run
+ * settles. Scans read their whole work set, so a joined slot loses nothing.
+ */
+const createSchedule = async (config: {
+  id: string;
+  cron: string;
+  tz: string;
+  family: string;
+  label: string;
+  source?: string;
+  submit: () => Promise<unknown>;
+}): Promise<void> => {
+  const source = config.source ?? config.id;
+  await lifecycleScheduler().create({
+    id: config.id,
+    cron: config.cron,
+    timezone: config.tz,
+    misfire: "latest",
+    meta: {
+      appId: "core",
+      family: config.family,
+      label: config.label,
+      source,
+    },
+    process: async () => {
+      await config.submit();
+    },
+  });
+};
+
+const createScheduleWithFallback = async (config: {
+  id: string;
+  cron: string;
+  fallbackCron: string;
+  tz: string;
+  submit: () => Promise<unknown>;
+  settingsKey: string;
+  family: string;
+  label: string;
+  source?: string;
+}): Promise<void> => {
+  try {
+    await createSchedule({
+      id: config.id,
+      cron: config.cron,
+      tz: config.tz,
+      family: config.family,
+      label: config.label,
+      source: config.source,
+      submit: config.submit,
+    });
+  } catch (error) {
+    if (config.cron === config.fallbackCron) throw error;
+    log.warn("Invalid configured cron, falling back to default", {
+      key: config.settingsKey,
+      configuredCron: config.cron,
+      fallbackCron: config.fallbackCron,
+      timezone: config.tz,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    await createSchedule({
+      id: config.id,
+      cron: config.fallbackCron,
+      tz: config.tz,
+      family: config.family,
+      label: config.label,
+      source: config.source,
+      submit: config.submit,
+    });
+  }
+};
+
+const doRegister = async (): Promise<void> => {
+  const [scheduleTz, ipaSyncCron, reminderCron, cleanupCron] = await Promise.all([
+    getTimezoneSetting(),
+    getCronSetting("freeipa.sync_cron", DEFAULT_IPA_SYNC_CRON),
+    getCronSetting("user.account.reminder_cron", "0 9 * * *"),
+    getCronSetting("app.cleanup_schedule", "0 4 * * *"),
+  ]);
+
+  await createScheduleWithFallback({
+    id: "auth:ipa:sync",
+    cron: ipaSyncCron,
+    fallbackCron: DEFAULT_IPA_SYNC_CRON,
+    tz: scheduleTz,
+    settingsKey: "freeipa.sync_cron",
+    family: "auth:ipa",
+    label: "FreeIPA account sync",
+    submit: () => ipaSyncJob().submit({ key: "scheduled", input: null, coalesce: true }),
+  });
+
+  await createSchedule({
+    id: "auth:reminder:daily",
+    cron: reminderCron,
+    tz: scheduleTz,
+    family: "auth:reminders",
+    label: "Daily account reminders",
+    submit: () => reminderJob().submit({ key: "scheduled", input: null, coalesce: true }),
+  });
+
+  await createSchedule({
+    id: "auth:guest:cleanup",
+    cron: cleanupCron,
+    tz: scheduleTz,
+    family: "auth:cleanup",
+    label: "Guest account cleanup",
+    submit: () => guestCleanupJob().submit({ key: "scheduled", input: null, coalesce: true }),
+  });
+
+  await createSchedule({
+    id: "auth:local-user:cleanup",
+    cron: cleanupCron,
+    tz: scheduleTz,
+    family: "auth:cleanup",
+    label: "Local user cleanup",
+    submit: () => localUserCleanupJob().submit({ key: "scheduled", input: null, coalesce: true }),
+  });
+
+  await createSchedule({
+    id: "auth:lifecycle:audit:cleanup",
+    cron: cleanupCron,
+    tz: scheduleTz,
+    family: "auth:cleanup",
+    label: "Account lifecycle audit cleanup",
+    submit: () => auditCleanupJob().submit({ key: "scheduled", input: null, coalesce: true }),
+  });
+
+  await createSchedule({
+    id: "app:logs:cleanup",
+    cron: cleanupCron,
+    tz: scheduleTz,
+    family: "app:cleanup",
+    label: "Application log cleanup",
+    submit: () => logCleanupJob().submit({ key: "scheduled", input: null, coalesce: true }),
+  });
+
+  registered = true;
+};
+
+const ensureRegistered = async (): Promise<void> => {
+  if (registered) return;
+  if (!registerPromise) {
+    registerPromise = doRegister().finally(() => {
+      registerPromise = null;
+    });
+  }
+  await registerPromise;
+};
+
+export const lifecycleJobs = {
+  start: async (config: { notificationSender: AccountLifecycleNotificationSender }): Promise<void> => {
+    notificationSender = config.notificationSender;
+    if (started) return;
+    await ensureRegistered();
+    try {
+      workers.push(await ipaSyncJob().process({}, processIpaSyncJob));
+      workers.push(await reminderJob().process({}, processReminderJob));
+      workers.push(await guestCleanupJob().process({}, processGuestCleanupJob));
+      workers.push(await localUserCleanupJob().process({}, processLocalUserCleanupJob));
+      workers.push(await auditCleanupJob().process({}, processAuditCleanupJob));
+      workers.push(await logCleanupJob().process({}, processLogCleanupJob));
+      workers.push(await ipaBackfill().accounts.process({ concurrency: 1 }, processIpaBackfillAccount));
+      workers.push(await ipaBackfill().pump.process());
+      workers.push(await guestBackfillJob().process({}, processGuestBackfillJob));
+      workers.push(await localUserBackfillJob().process({}, processLocalUserBackfillJob));
+      workers.push(await lifecycleScheduler().process());
+      started = true;
+    } catch (error) {
+      for (const worker of workers) worker.stop();
+      await Promise.all(workers.map((worker) => worker.drain()));
+      workers = [];
+      throw error;
+    }
+  },
+
+  stop: async (): Promise<void> => {
+    if (!started) {
+      notificationSender = null;
+      return;
+    }
+    for (const worker of workers) worker.stop();
+    await Promise.all(workers.map((worker) => worker.drain()));
+    workers = [];
+    started = false;
+    registered = false;
+    registerPromise = null;
+    notificationSender = null;
+  },
+
+  // Manual-only backfill triggers. Scheduled jobs run from their broker schedules
+  // (`listSchedules`; Admin Observability Jobs can run a schedule now).
+  submitIpaBackfill: async (): Promise<string> => {
+    const key = crypto.randomUUID();
+    await ipaBackfill().pump.start({ key, input: await prepareIpaBackfill(key) });
+    return key;
+  },
+  submitLocalUserBackfill: async (): Promise<string> =>
+    (await localUserBackfillJob().submit({ key: `manual:${Date.now()}`, input: null })).jobId,
+  submitGuestBackfill: async (): Promise<string> => (await guestBackfillJob().submit({ key: `manual:${Date.now()}`, input: null })).jobId,
+
+  /** Process-local worker state; schedule state (next run, failures) comes from `listSchedules`. */
+  metrics: () => ({
+    started,
+    registered,
+    active: workers.reduce((total, worker) => total + worker.active, 0),
+    capacity: workers.reduce((total, worker) => total + worker.capacity, 0),
+  }),
+  listSchedules: async () => lifecycleScheduler().list(),
+};

package/src/services/account-model.ts

@@ -0,0 +1,22 @@
+export { buildRoles } from "./accounts/authz";
+export {
+  calculateIpaProfileFromGroupNames,
+  canPersistStoredAdmin,
+  deriveIpaAdminFromGroupNames,
+  getConfiguredExpiryDays,
+  getDefaultAccountExpiry,
+  type IpaAccountTransitionPolicy,
+  type IpaMatchMode,
+  isAccountExpired,
+  isGuestProfile,
+  isIpaProvider,
+  isLocalProvider,
+  normalizeManualAccountExpiry,
+  parseIpaAccountTransitionPolicy,
+  parseIpaMatchMode,
+  parseManualAccountExpiry,
+  resolveAccountExpires,
+  resolveEffectiveAdminState,
+  resolveStoredAdminState,
+  resolveTargetAccountExpiry,
+} from "./accounts/model";