@k2b/cloud 0.6.0-rc.0

package/src/services/notifications/browser.ts

@@ -0,0 +1,252 @@
+import { createHash } from "node:crypto";
+import { sql } from "bun";
+import webpush from "web-push";
+import { z } from "zod";
+import { isSafeNotificationTargetHref } from "../../contracts/notification-types";
+import { type BrowserPushSubscription, BrowserPushSubscriptionSchema } from "../../contracts/user-notifications";
+import { decryptSecret, encryptSecret } from "../secrets";
+import { coreSettings } from "../settings/api";
+import { type NotificationDestination, registerNotificationChannel } from "./channels";
+import { sendPinnedWebPush } from "./web-push-transport";
+
+type EndpointRow = {
+  id: string;
+  endpoint_hash: string;
+  label: string;
+  secret_encrypted: string;
+};
+
+type BrowserDeliveryPayload = {
+  endpointId: string;
+  subscription: BrowserPushSubscription;
+  eventId: string;
+  title: string;
+  targetHref?: string;
+};
+
+const BrowserDeliveryPayloadSchema = z.object({
+  endpointId: z.uuid(),
+  subscription: BrowserPushSubscriptionSchema,
+  eventId: z.uuid(),
+  title: z.string().min(1).max(200),
+  targetHref: z.string().max(4_000).refine(isSafeNotificationTargetHref).optional(),
+});
+
+const BrowserDestinationContextSchema = z.object({
+  endpointId: z.uuid(),
+  subscription: BrowserPushSubscriptionSchema,
+});
+
+const endpointHash = (endpoint: string): string => createHash("sha256").update(endpoint).digest("hex");
+
+let vapidConfigPromise: Promise<{ publicKey: string; privateKey: string }> | null = null;
+let webPushConfigurationPromise: Promise<void> | null = null;
+let unregisterDriver: (() => void) | null = null;
+
+const ensureVapidConfig = (): Promise<{ publicKey: string; privateKey: string }> => {
+  if (vapidConfigPromise) return vapidConfigPromise;
+  vapidConfigPromise = sql
+    .begin(async (tx) => {
+      await tx`SELECT pg_advisory_xact_lock(hashtextextended('notifications:web-push-vapid', 0))`;
+      let [publicKey, privateKey] = await Promise.all([
+        coreSettings.get<string>("notifications.web_push_public_key"),
+        coreSettings.get<string>("notifications.web_push_private_key"),
+      ]);
+      if (!publicKey || !privateKey) {
+        const generated = webpush.generateVAPIDKeys();
+        publicKey = generated.publicKey;
+        privateKey = generated.privateKey;
+        await coreSettings.set("notifications.web_push_public_key", publicKey);
+        await coreSettings.set("notifications.web_push_private_key", privateKey);
+      }
+      return { publicKey, privateKey };
+    })
+    .catch((error) => {
+      vapidConfigPromise = null;
+      throw error;
+    });
+  return vapidConfigPromise;
+};
+
+const vapidSubject = async (): Promise<string> => {
+  const [configuredUrl, contactEmail] = await Promise.all([
+    coreSettings.get<string>("app.url"),
+    coreSettings.get<string>("app.contact_email"),
+  ]);
+  const rawUrl = configuredUrl?.trim();
+  if (rawUrl) {
+    try {
+      const parsed = new URL(/^https?:\/\//.test(rawUrl) ? rawUrl : `https://${rawUrl}`);
+      const isLocalhost = parsed.hostname === "localhost" || parsed.hostname === "127.0.0.1" || parsed.hostname === "[::1]";
+      if (parsed.protocol === "https:" && !isLocalhost) return parsed.origin;
+    } catch {
+      // Fall through to the deployment contact below.
+    }
+  }
+  return contactEmail?.trim() ? `mailto:${contactEmail.trim()}` : "mailto:notifications@cloud.local";
+};
+
+const ensureWebPushConfigured = (): Promise<void> => {
+  if (webPushConfigurationPromise) return webPushConfigurationPromise;
+  webPushConfigurationPromise = Promise.all([ensureVapidConfig(), vapidSubject()])
+    .then(([config, subject]) => {
+      webpush.setVapidDetails(subject, config.publicKey, config.privateKey);
+    })
+    .catch((error) => {
+      webPushConfigurationPromise = null;
+      throw error;
+    });
+  return webPushConfigurationPromise;
+};
+
+const disableEndpoint = async (endpointId: string): Promise<void> => {
+  await sql`
+    UPDATE notifications.endpoints
+    SET disabled_at = now(), updated_at = now()
+    WHERE id = ${endpointId}::uuid AND channel = 'browser'
+  `;
+};
+
+const browserDriver = {
+  id: "browser" as const,
+  resolveDestinations: async (recipient: { userId: string | null }) => {
+    if (!recipient.userId) return [];
+    const rows = await sql<EndpointRow[]>`
+      SELECT id, endpoint_hash, label, secret_encrypted
+      FROM notifications.endpoints
+      WHERE user_id = ${recipient.userId}::uuid
+        AND channel = 'browser'
+        AND verified_at IS NOT NULL
+        AND disabled_at IS NULL
+      ORDER BY last_seen_at DESC, id
+    `;
+    const destinations: NotificationDestination[] = [];
+    for (const row of rows) {
+      try {
+        const subscription = BrowserPushSubscriptionSchema.parse(await decryptSecret(row.secret_encrypted));
+        if (subscription.expirationTime && subscription.expirationTime <= Date.now()) {
+          await disableEndpoint(row.id);
+          continue;
+        }
+        destinations.push({
+          key: row.endpoint_hash,
+          label: row.label,
+          endpointId: row.id,
+          context: { endpointId: row.id, subscription },
+        });
+      } catch {
+        await disableEndpoint(row.id);
+      }
+    }
+    return destinations;
+  },
+  createPayload: ({ presentation, destination, event }) => {
+    const context = BrowserDestinationContextSchema.parse(destination.context);
+    return BrowserDeliveryPayloadSchema.parse({
+      endpointId: context.endpointId,
+      subscription: context.subscription,
+      eventId: event.id,
+      title: presentation.title,
+      targetHref: presentation.targetHref,
+    });
+  },
+  deliver: async (value: unknown) => {
+    const payload: BrowserDeliveryPayload = BrowserDeliveryPayloadSchema.parse(value);
+    await ensureWebPushConfigured();
+    try {
+      await sendPinnedWebPush(
+        payload.subscription,
+        JSON.stringify({
+          type: "cloud-notification",
+          eventId: payload.eventId,
+          title: payload.title,
+          targetHref: payload.targetHref,
+        }),
+        {
+          TTL: 24 * 60 * 60,
+          urgency: "normal",
+          topic: createHash("sha256").update(payload.eventId).digest("base64url").slice(0, 32),
+        },
+      );
+    } catch (error) {
+      const statusCode = error && typeof error === "object" && "statusCode" in error ? Number(error.statusCode) : null;
+      if (statusCode === 404 || statusCode === 410) {
+        await disableEndpoint(payload.endpointId);
+        throw Object.assign(new Error("Browser notification endpoint is no longer registered"), {
+          code: "endpoint_gone",
+          retryable: false,
+        });
+      }
+      if (statusCode !== null && statusCode >= 400 && statusCode < 500 && statusCode !== 408 && statusCode !== 429) {
+        throw Object.assign(new Error("Browser push provider rejected the delivery"), {
+          code: "provider_rejected",
+          retryable: false,
+        });
+      }
+      throw error;
+    }
+  },
+} satisfies import("./channels").NotificationChannelDriver;
+
+// Every app process may create deliveries; Core remains the only process that
+// runs the durable delivery queue.
+unregisterDriver = registerNotificationChannel(browserDriver);
+
+export const browserNotifications = {
+  start: async (): Promise<void> => {
+    unregisterDriver ??= registerNotificationChannel(browserDriver);
+    await ensureWebPushConfigured();
+  },
+  stop: (): void => {
+    unregisterDriver?.();
+    unregisterDriver = null;
+  },
+  configuration: async (): Promise<{ publicKey: string }> => {
+    const config = await ensureVapidConfig();
+    return { publicKey: config.publicKey };
+  },
+  registerEndpoint: async (config: {
+    userId: string;
+    subscription: BrowserPushSubscription;
+    label: string;
+  }): Promise<{ id: string; label: string }> => {
+    const subscription = BrowserPushSubscriptionSchema.parse(config.subscription);
+    const hash = endpointHash(subscription.endpoint);
+    const encrypted = await encryptSecret(subscription);
+    const label = config.label.trim().slice(0, 120) || "This browser";
+    const rows = await sql.begin(async (tx) => {
+      await tx`SELECT pg_advisory_xact_lock(hashtextextended(${`notifications:browser:${hash}`}, 0))`;
+      await tx`
+        UPDATE notifications.endpoints
+        SET disabled_at = now(), updated_at = now()
+        WHERE channel = 'browser' AND endpoint_hash = ${hash}
+          AND user_id <> ${config.userId}::uuid AND disabled_at IS NULL
+      `;
+      return tx<{ id: string; label: string }[]>`
+        INSERT INTO notifications.endpoints (
+          user_id, channel, endpoint_hash, label, secret_encrypted,
+          verified_at, last_seen_at, disabled_at, updated_at
+        ) VALUES (
+          ${config.userId}::uuid, 'browser', ${hash}, ${label}, ${encrypted},
+          now(), now(), NULL, now()
+        )
+        ON CONFLICT (user_id, channel, endpoint_hash) DO UPDATE
+        SET label = EXCLUDED.label, secret_encrypted = EXCLUDED.secret_encrypted,
+            verified_at = now(), last_seen_at = now(), disabled_at = NULL, updated_at = now()
+        RETURNING id, label
+      `;
+    });
+    return rows[0]!;
+  },
+  disableEndpoint: async (config: { userId: string; subscription: BrowserPushSubscription }): Promise<boolean> => {
+    const subscription = BrowserPushSubscriptionSchema.parse(config.subscription);
+    const rows = await sql<{ id: string }[]>`
+      UPDATE notifications.endpoints
+      SET disabled_at = now(), updated_at = now()
+      WHERE user_id = ${config.userId}::uuid AND channel = 'browser'
+        AND endpoint_hash = ${endpointHash(subscription.endpoint)} AND disabled_at IS NULL
+      RETURNING id
+    `;
+    return rows.length > 0;
+  },
+} as const;

package/src/services/notifications/catalog.ts

@@ -0,0 +1,113 @@
+import { sql } from "bun";
+import type { AnyBoundNotificationDefinition } from "../../contracts/notification-types";
+import { logger } from "../logging";
+import { toPgTextArray } from "../postgres";
+
+type NotificationCatalog = Readonly<Record<string, AnyBoundNotificationDefinition>>;
+
+const RETRY_MS = 5_000;
+const log = logger("notifications:catalog");
+
+const isSchemaUnavailable = (error: unknown): boolean => {
+  if (!error || typeof error !== "object" || !("code" in error)) return false;
+  return error.code === "42P01" || error.code === "42703" || error.code === "3F000";
+};
+
+const upsertDefinition = async (definition: AnyBoundNotificationDefinition, db: typeof sql = sql): Promise<void> => {
+  const recommended = toPgTextArray([...(definition.delivery?.recommended ?? [])]);
+  const required = toPgTextArray([...(definition.delivery?.required ?? [])]);
+  await db`
+    INSERT INTO notifications.definitions (
+      id, app_id, kind, label, description, presentation, recipient_kind,
+      recommended_channels, required_channels, active, last_seen_at, updated_at
+    ) VALUES (
+      ${definition.id}, ${definition.appId}, ${definition.key}, ${definition.label},
+      ${definition.description}, (${definition.presentation ? JSON.stringify(definition.presentation) : null}::text)::jsonb,
+      ${definition.recipient}, ${recommended}::text[],
+      ${required}::text[], true, now(), now()
+    )
+    ON CONFLICT (id) DO UPDATE SET
+      app_id = EXCLUDED.app_id,
+      kind = EXCLUDED.kind,
+      label = EXCLUDED.label,
+      description = EXCLUDED.description,
+      presentation = EXCLUDED.presentation,
+      recipient_kind = EXCLUDED.recipient_kind,
+      recommended_channels = EXCLUDED.recommended_channels,
+      required_channels = EXCLUDED.required_channels,
+      active = true,
+      last_seen_at = now(),
+      updated_at = now()
+  `;
+};
+
+/** Persist serializable app metadata; schemas and renderers remain code-owned. */
+export const registerNotificationDefinitions = async (appId: string, definitions: NotificationCatalog): Promise<void> => {
+  const current = Object.values(definitions);
+  if (current.some((definition) => definition.appId !== appId)) {
+    throw new Error(`Notification catalog for app "${appId}" contains a definition owned by another app`);
+  }
+  const ids = toPgTextArray(current.map((definition) => definition.id));
+  await sql.begin(async (tx) => {
+    for (const definition of current) await upsertDefinition(definition, tx);
+    await tx`
+      UPDATE notifications.definitions
+      SET active = false, updated_at = now()
+      WHERE app_id = ${appId} AND NOT (id = ANY(${ids}::text[]))
+    `;
+  });
+};
+
+/** Keep app startup rollout-safe when Core has not expanded the schema yet. */
+export const startNotificationDefinitionRegistration = async (
+  appId: string,
+  definitions: NotificationCatalog,
+  options: {
+    register?: (appId: string, definitions: NotificationCatalog) => Promise<void>;
+    retryMs?: number;
+  } = {},
+): Promise<() => void> => {
+  const register = options.register ?? registerNotificationDefinitions;
+  const retryMs = options.retryMs ?? RETRY_MS;
+  let stopped = false;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+
+  const schedule = () => {
+    if (stopped || timer) return;
+    timer = setTimeout(() => {
+      timer = undefined;
+      void attempt();
+    }, retryMs);
+    if (typeof timer === "object" && "unref" in timer) timer.unref();
+  };
+
+  const attempt = async (): Promise<void> => {
+    if (stopped) return;
+    try {
+      await register(appId, definitions);
+      log.info("Notification definitions registered after schema became available", { appId });
+    } catch (error) {
+      log.warn("Notification definition registration failed; retrying", {
+        appId,
+        error: error instanceof Error ? error.message : "Notification catalog registration failed",
+      });
+      schedule();
+    }
+  };
+
+  try {
+    await register(appId, definitions);
+  } catch (error) {
+    if (!isSchemaUnavailable(error)) throw error;
+    log.warn("Notification schema is not available yet; registration will retry", { appId });
+    schedule();
+  }
+
+  return () => {
+    stopped = true;
+    if (timer) clearTimeout(timer);
+    timer = undefined;
+  };
+};
+
+export const ensureNotificationDefinition = upsertDefinition;

package/src/services/notifications/channels.ts

@@ -0,0 +1,104 @@
+import { createHash } from "node:crypto";
+import type { EmailNotificationPresentation, NotificationChannelId, NotificationPresentation } from "../../contracts/notification-types";
+import { sendEmail } from "./email";
+
+export type ResolvedNotificationRecipient = {
+  userId: string | null;
+  email: string | null;
+};
+
+export type NotificationDestination = {
+  key: string;
+  label: string;
+  endpointId?: string;
+  context: unknown;
+};
+
+export type NotificationChannelDriver = {
+  id: NotificationChannelId | (string & {});
+  resolveDestinations: (recipient: ResolvedNotificationRecipient) => Promise<NotificationDestination[]>;
+  createPayload: (input: {
+    presentation: NotificationPresentation;
+    email?: EmailNotificationPresentation;
+    destination: NotificationDestination;
+    event: { id: string; definitionId: string };
+  }) => unknown;
+  deliver: (payload: unknown) => Promise<void>;
+};
+
+const drivers = new Map<string, NotificationChannelDriver>();
+
+export const registerNotificationChannel = (driver: NotificationChannelDriver): (() => void) => {
+  if (!/^[a-z][a-z0-9_-]{0,79}$/.test(driver.id)) {
+    throw new Error("Notification channel id must be a lowercase identifier of at most 80 characters");
+  }
+  const existing = drivers.get(driver.id);
+  if (existing && existing !== driver) throw new Error(`Notification channel "${driver.id}" is already registered`);
+  drivers.set(driver.id, driver);
+  return () => {
+    if (drivers.get(driver.id) === driver) drivers.delete(driver.id);
+  };
+};
+
+export const getNotificationChannel = (id: string): NotificationChannelDriver | undefined => drivers.get(id);
+
+export const listNotificationChannels = (): string[] => [...drivers.keys()].sort();
+
+const normalizeEmail = (value: string): string => value.trim().toLowerCase();
+const emailKey = (value: string): string => createHash("sha256").update(normalizeEmail(value)).digest("hex");
+
+const emailLabel = (value: string): string => {
+  const [local = "", domain = ""] = normalizeEmail(value).split("@", 2);
+  if (!domain) return "Email";
+  return `${local.slice(0, 1) || "*"}***@${domain}`;
+};
+
+type EmailPayload = {
+  to: string;
+  subject: string;
+  content?: string;
+  rawHtml?: string;
+  messageId?: string;
+};
+
+const parseEmailPayload = (value: unknown): EmailPayload => {
+  if (!value || typeof value !== "object") throw new Error("Invalid email notification payload");
+  const payload = value as Partial<EmailPayload>;
+  if (typeof payload.to !== "string" || typeof payload.subject !== "string") {
+    throw new Error("Invalid email notification payload");
+  }
+  if (payload.content !== undefined && typeof payload.content !== "string") throw new Error("Invalid email notification payload");
+  if (payload.rawHtml !== undefined && typeof payload.rawHtml !== "string") throw new Error("Invalid email notification payload");
+  if (payload.messageId !== undefined && typeof payload.messageId !== "string") throw new Error("Invalid email notification payload");
+  return { to: payload.to, subject: payload.subject, content: payload.content, rawHtml: payload.rawHtml, messageId: payload.messageId };
+};
+
+const emailDriver: NotificationChannelDriver = {
+  id: "email",
+  resolveDestinations: async (recipient) => {
+    if (!recipient.email) return [];
+    const email = normalizeEmail(recipient.email);
+    return [{ key: emailKey(email), label: emailLabel(email), context: { email } }];
+  },
+  createPayload: ({ presentation, email, destination, event }) => {
+    const context = destination.context as { email?: unknown };
+    if (typeof context.email !== "string") throw new Error("Email destination is missing an address");
+    return {
+      to: context.email,
+      subject: email?.subject ?? presentation.title,
+      content: email?.content ?? presentation.body,
+      rawHtml: email?.rawHtml,
+      messageId: `<cloud-notification-${event.id}@cloud.invalid>`,
+    } satisfies EmailPayload;
+  },
+  deliver: async (value) => {
+    const payload = parseEmailPayload(value);
+    await sendEmail(payload.to, payload.subject, {
+      content: payload.content,
+      rawHtml: payload.rawHtml,
+      messageId: payload.messageId,
+    });
+  },
+};
+
+registerNotificationChannel(emailDriver);

package/src/services/notifications/dispatcher.ts

@@ -0,0 +1,139 @@
+import { sql } from "bun";
+import { decryptSecret } from "../secrets";
+import { getNotificationChannel } from "./channels";
+
+const MAX_DELIVERY_ATTEMPTS = 5;
+const BASE_RETRY_MS = 2_000;
+const MAX_RETRY_MS = 5 * 60_000;
+
+type DeliveryRow = {
+  id: string;
+  event_id: string;
+  channel: string;
+  payload_encrypted: string | null;
+  required: boolean;
+  route_priority: number | null;
+  attempt_count: number;
+};
+
+export type DeliveryAttemptResult =
+  | { status: "skipped" | "delivered" | "failed" }
+  | { status: "retry"; retryAfterMs: number; error: string };
+
+const retryDelay = (attempt: number): number => Math.min(BASE_RETRY_MS * 2 ** Math.max(0, attempt - 1), MAX_RETRY_MS);
+
+const activateNextFallback = async (eventId: string): Promise<string[]> => {
+  const delivered = await sql<{ exists: boolean }[]>`
+    SELECT EXISTS (
+      SELECT 1 FROM notifications.deliveries
+      WHERE event_id = ${eventId}::uuid AND required = false AND status = 'delivered'
+    ) AS exists
+  `;
+  if (delivered[0]?.exists) {
+    await sql`
+      UPDATE notifications.deliveries
+      SET status = 'suppressed', error_code = 'fallback_not_needed',
+          error_message = 'A preferred channel was delivered.', payload_encrypted = NULL, updated_at = now()
+      WHERE event_id = ${eventId}::uuid AND required = false AND status = 'deferred'
+    `;
+    return [];
+  }
+
+  const active = await sql<{ exists: boolean }[]>`
+    SELECT EXISTS (
+      SELECT 1 FROM notifications.deliveries
+      WHERE event_id = ${eventId}::uuid AND required = false AND status IN ('pending', 'sending')
+    ) AS exists
+  `;
+  if (active[0]?.exists) return [];
+
+  const rows = await sql<{ id: string }[]>`
+    WITH next_route AS (
+      SELECT MIN(route_priority) AS priority
+      FROM notifications.deliveries
+      WHERE event_id = ${eventId}::uuid AND required = false AND status = 'deferred'
+    )
+    UPDATE notifications.deliveries d
+    SET status = 'pending', next_attempt_at = now(), updated_at = now()
+    FROM next_route
+    WHERE d.event_id = ${eventId}::uuid
+      AND d.required = false
+      AND d.status = 'deferred'
+      AND d.route_priority = next_route.priority
+    RETURNING d.id
+  `;
+  return rows.map((row) => row.id);
+};
+
+export const processNotificationDelivery = async (deliveryId: string): Promise<DeliveryAttemptResult & { activatedIds?: string[] }> => {
+  const rows = await sql<DeliveryRow[]>`
+    UPDATE notifications.deliveries
+    SET status = 'sending', attempt_count = attempt_count + 1,
+        last_attempt_at = now(), updated_at = now()
+    WHERE id = ${deliveryId}::uuid
+      AND status = 'pending'
+      AND (next_attempt_at IS NULL OR next_attempt_at <= now())
+    RETURNING id, event_id, channel, payload_encrypted, required, route_priority, attempt_count
+  `;
+  const delivery = rows[0];
+  if (!delivery) return { status: "skipped" };
+
+  try {
+    const driver = getNotificationChannel(delivery.channel);
+    if (!driver)
+      throw Object.assign(new Error(`Notification channel "${delivery.channel}" is unavailable`), { code: "channel_unavailable" });
+    if (!delivery.payload_encrypted) {
+      throw Object.assign(new Error("Notification delivery payload is unavailable"), { code: "payload_missing", retryable: false });
+    }
+    const payload = await decryptSecret(delivery.payload_encrypted);
+    await driver.deliver(payload);
+    await sql`
+      UPDATE notifications.deliveries
+      SET status = 'delivered', delivered_at = now(), next_attempt_at = NULL,
+          error_code = NULL, error_message = NULL, payload_encrypted = NULL, updated_at = now()
+      WHERE id = ${delivery.id}::uuid
+    `;
+    const activatedIds = delivery.required ? [] : await activateNextFallback(delivery.event_id);
+    return { status: "delivered", activatedIds };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "Notification delivery failed";
+    const code = error && typeof error === "object" && "code" in error ? String(error.code) : "provider_error";
+    const retryable = !(error && typeof error === "object" && "retryable" in error && error.retryable === false);
+    if (retryable && delivery.attempt_count < MAX_DELIVERY_ATTEMPTS) {
+      const retryAfterMs = retryDelay(delivery.attempt_count);
+      await sql`
+        UPDATE notifications.deliveries
+        SET status = 'pending', next_attempt_at = now() + (${retryAfterMs}::int * INTERVAL '1 millisecond'),
+            error_code = ${code}, error_message = ${message}, updated_at = now()
+        WHERE id = ${delivery.id}::uuid
+      `;
+      return { status: "retry", retryAfterMs, error: message };
+    }
+
+    await sql`
+      UPDATE notifications.deliveries
+      SET status = 'failed', next_attempt_at = NULL, error_code = ${code},
+          error_message = ${message}, payload_encrypted = NULL, updated_at = now()
+      WHERE id = ${delivery.id}::uuid
+    `;
+    const activatedIds = delivery.required ? [] : await activateNextFallback(delivery.event_id);
+    return { status: "failed", activatedIds };
+  }
+};
+
+export const recoverNotificationDeliveries = async (): Promise<string[]> => {
+  await sql`
+    UPDATE notifications.deliveries
+    SET status = 'pending', next_attempt_at = now(), error_code = 'lease_recovered',
+        error_message = 'Recovered an interrupted delivery attempt.', updated_at = now()
+    WHERE status = 'sending' AND last_attempt_at < now() - INTERVAL '5 minutes'
+  `;
+  const rows = await sql<{ id: string }[]>`
+    SELECT id
+    FROM notifications.deliveries
+    WHERE status = 'pending' AND (next_attempt_at IS NULL OR next_attempt_at <= now())
+    ORDER BY next_attempt_at NULLS FIRST, created_at
+    LIMIT 500
+  `;
+  return rows.map((row) => row.id);
+};