@helyx/bot 0.4.8 → 0.4.11

package/dist/shard-contracts.js

@@ -1,7 +1,10 @@
 import { z } from "zod";
 export const SHARD_CONTRACT_VERSION = 1;
-export const SHARD_CONTROL_MAX_BYTES = 64 * 1_024;
-export const SHARD_CONTROL_TIMEOUT_MS = 10_000;
+// Guild dashboard state includes the bounded module catalogue as well as the
+// guild's Discord resources. Keep this aligned with the cloud-control message
+// ceiling so hosted and self-hosted transports accept the same valid state.
+export const SHARD_CONTROL_MAX_BYTES = 256 * 1_024;
+export const SHARD_CONTROL_TIMEOUT_MS = 30_000;
 export const SHARD_HEALTH_STALE_MS = 30_000;
 export const SHARD_LEASE_HEARTBEAT_TIMEOUT_MS = 20_000;
 const snowflake = z.string().regex(/^\d{17,20}$/u);
@@ -43,6 +46,7 @@ const permissionState = z
 })
     .strict();
 const configurationValue = z.record(z.string().min(1).max(100), z.union([
+    z.null(),
     z.string().max(4_000),
     z.boolean(),
     z.number().finite(),
@@ -355,6 +359,7 @@ export const shardControlOperationSchema = z.discriminatedUnion("kind", [
         actionId: resourceType,
         expectedRevision: z.number().int().min(0),
         value: managedResourceObject,
+        actorAuthenticatedAt: dateTime.optional(),
         ...mutationContext.shape,
     })
         .strict(),
@@ -567,11 +572,246 @@ export const shardScheduledTaskResponseSchema = z
     result: scheduledTaskResult,
 })
     .strict();
+const numericWireValueSchema = z.discriminatedUnion("type", [
+    z
+        .object({ type: z.literal("string"), value: z.string().max(4_000) })
+        .strict(),
+    z.object({ type: z.literal("boolean"), value: z.boolean() }).strict(),
+    z.object({ type: z.literal("number"), value: z.number().finite() }).strict(),
+    z.object({ type: z.literal("date"), value: dateTime }).strict(),
+]);
+const numericWireRecordSchema = z
+    .record(z.string().regex(/^[a-z][a-z0-9_]*$/u), numericWireValueSchema)
+    .refine((value) => Object.keys(value).length <= 50);
+const numericScopeSchema = z.object({
+    moduleId,
+    collection: z.string().regex(/^[a-z][a-z0-9_]*$/u),
+});
+const numericMemberScopeSchema = {
+    ...numericScopeSchema.shape,
+    guildId: snowflake,
+    subjectId: safeIdentifier,
+};
+const numericOperationSchema = z
+    .discriminatedUnion("kind", [
+    z
+        .object({
+        kind: z.literal("enqueue"),
+        ...numericMemberScopeSchema,
+        occurredAt: dateTime,
+        cooldownSeconds: z.number().int().min(1).max(86_400),
+        identity: numericWireRecordSchema,
+        insert: numericWireRecordSchema,
+        increments: z
+            .record(z.string().regex(/^[a-z][a-z0-9_]*$/u), z.number().int().positive())
+            .refine((value) => {
+            const length = Object.keys(value).length;
+            return length >= 1 && length <= 20;
+        }),
+        maxima: z
+            .record(z.string().regex(/^[a-z][a-z0-9_]*$/u), dateTime)
+            .refine((value) => Object.keys(value).length <= 20)
+            .optional(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("pending.member"),
+        ...numericMemberScopeSchema,
+    })
+        .strict(),
+    z
+        .object({ kind: z.literal("flush.member"), ...numericMemberScopeSchema })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("flush.guild"),
+        ...numericScopeSchema.shape,
+        guildId: snowflake,
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("flush.subject"),
+        ...numericScopeSchema.shape,
+        subjectId: safeIdentifier,
+    })
+        .strict(),
+    z
+        .object({ kind: z.literal("flush.module"), ...numericScopeSchema.shape })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("barrier.acquire"),
+        ...numericScopeSchema.shape,
+        barrierType: z.enum({
+            member: "member",
+            guild: "guild",
+            subject: "subject",
+        }),
+        guildId: snowflake.optional(),
+        subjectId: safeIdentifier.optional(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("barrier.release"),
+        ...numericScopeSchema.shape,
+        barrierType: z.enum({
+            member: "member",
+            guild: "guild",
+            subject: "subject",
+        }),
+        guildId: snowflake.optional(),
+        subjectId: safeIdentifier.optional(),
+        barrierId: z.uuid(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("barrier.renew"),
+        ...numericScopeSchema.shape,
+        barrierType: z.enum({
+            member: "member",
+            guild: "guild",
+            subject: "subject",
+        }),
+        guildId: snowflake.optional(),
+        subjectId: safeIdentifier.optional(),
+        barrierId: z.uuid(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("suppress.subject"),
+        ...numericScopeSchema.shape,
+        subjectId: safeIdentifier,
+        cutoff: dateTime,
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("suppress.guild"),
+        ...numericScopeSchema.shape,
+        guildId: snowflake,
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("release.guild"),
+        ...numericScopeSchema.shape,
+        guildId: snowflake,
+        suppressionId: z.uuid(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("release.subject"),
+        ...numericScopeSchema.shape,
+        subjectId: safeIdentifier,
+        suppressionId: z.uuid(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("cooldown.member.clear"),
+        ...numericMemberScopeSchema,
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("cooldown.subject.clear"),
+        ...numericScopeSchema.shape,
+        subjectId: safeIdentifier,
+    })
+        .strict(),
+])
+    .superRefine((operation, context) => {
+    if (operation.kind !== "barrier.acquire" &&
+        operation.kind !== "barrier.release" &&
+        operation.kind !== "barrier.renew")
+        return;
+    const valid = operation.barrierType === "member"
+        ? operation.guildId !== undefined && operation.subjectId !== undefined
+        : operation.barrierType === "guild"
+            ? operation.guildId !== undefined && operation.subjectId === undefined
+            : operation.guildId === undefined &&
+                operation.subjectId !== undefined;
+    if (!valid)
+        context.addIssue({
+            code: "custom",
+            message: "Numeric barrier scope does not match its barrier type",
+        });
+});
+export const shardNumericRequestSchema = z
+    .object({
+    version: z.literal(SHARD_CONTRACT_VERSION),
+    type: z.literal("helyx.numeric.request"),
+    requestId: z.uuid(),
+    deadlineAt: dateTime,
+    identity: shardIdentitySchema,
+    operation: numericOperationSchema,
+})
+    .strict();
+export const shardNumericResponseSchema = z.discriminatedUnion("ok", [
+    z
+        .object({
+        version: z.literal(SHARD_CONTRACT_VERSION),
+        type: z.literal("helyx.numeric.response"),
+        requestId: z.uuid(),
+        ok: z.literal(true),
+        result: z.unknown(),
+    })
+        .strict(),
+    z
+        .object({
+        version: z.literal(SHARD_CONTRACT_VERSION),
+        type: z.literal("helyx.numeric.response"),
+        requestId: z.uuid(),
+        ok: z.literal(false),
+        safeCode: z.string().regex(/^[a-z0-9][a-z0-9._-]{0,99}$/u),
+    })
+        .strict(),
+]);
+export const shardNumericCommittedSchema = z
+    .object({
+    version: z.literal(SHARD_CONTRACT_VERSION),
+    type: z.literal("helyx.numeric.committed"),
+    requestId: z.uuid(),
+    deadlineAt: dateTime,
+    identity: shardIdentitySchema,
+    scope: numericScopeSchema.strict(),
+    payload: z.unknown(),
+})
+    .strict();
+export const shardNumericCommittedResponseSchema = z.discriminatedUnion("ok", [
+    z
+        .object({
+        version: z.literal(SHARD_CONTRACT_VERSION),
+        type: z.literal("helyx.numeric.committed.response"),
+        requestId: z.uuid(),
+        identity: shardIdentitySchema,
+        ok: z.literal(true),
+    })
+        .strict(),
+    z
+        .object({
+        version: z.literal(SHARD_CONTRACT_VERSION),
+        type: z.literal("helyx.numeric.committed.response"),
+        requestId: z.uuid(),
+        identity: shardIdentitySchema,
+        ok: z.literal(false),
+        safeCode: z.string().regex(/^[a-z0-9][a-z0-9._-]{0,99}$/u),
+    })
+        .strict(),
+]);
 export const supervisorMessageSchema = z.discriminatedUnion("type", [
     shardControlRequestSchema,
     shardCloudResponseSchema,
     shardOperationsHealthResponseSchema,
     shardScheduledTaskRequestSchema,
+    shardNumericResponseSchema,
+    shardNumericCommittedSchema,
     z
         .object({
         version: z.literal(SHARD_CONTRACT_VERSION),
@@ -598,6 +838,8 @@ export const workerMessageSchema = z.discriminatedUnion("type", [
     shardCloudRequestSchema,
     shardOperationsHealthRequestSchema,
     shardScheduledTaskResponseSchema,
+    shardNumericRequestSchema,
+    shardNumericCommittedResponseSchema,
     z
         .object({
         version: z.literal(SHARD_CONTRACT_VERSION),
@@ -758,6 +1000,9 @@ export async function executeShardControlOperation(handler, operation) {
                 value: operation.value,
                 actorUserId: operation.actorUserId,
                 correlationId: operation.correlationId,
+                ...(operation.actorAuthenticatedAt
+                    ? { actorAuthenticatedAt: operation.actorAuthenticatedAt }
+                    : {}),
             });
         case "support.notification.send": {
             return handler.sendSupportNotification({

package/dist/shard-supervisor.d.ts

@@ -1,12 +1,16 @@
 import type { BotEnvironment } from "@helyx/config";
 import { type DatabasePool } from "@helyx/database";
-import type { ModuleDefinition, ScheduledTaskExecutionResult } from "@helyx/sdk";
+import type { ModuleDefinition, NumericWriteBehindCommittedBatch, ScheduledTaskExecutionResult } from "@helyx/sdk";
 import { type ShardCloudRequest, type ShardIdentity, type ShardProcessHealth } from "./shard-contracts.js";
 import { type ShardCoordinator, type ShardGenerationDescriptor, type ShardLeaderLease, type ShardLease } from "./shard-coordinator.js";
 import type { BotHandle } from "./index.js";
 export declare const STEADY_STATE_LEASE_LOSS_BOUND_MS: number;
+export declare function createStartupWorkerHeartbeatMaintenance(heartbeat: () => void, intervalMs?: number): {
+    stop(): void;
+};
 export declare function deferScheduledRoutingFailures(route: () => Promise<ScheduledTaskExecutionResult>): Promise<ScheduledTaskExecutionResult>;
 export declare function startShardSupervisor(environment: BotEnvironment, processEnvironment: NodeJS.ProcessEnv, workerEntry: string): Promise<BotHandle>;
+export declare function splitNumericCommittedBatch(batch: NumericWriteBehindCommittedBatch, index: number): NumericWriteBehindCommittedBatch;
 export declare function buildShardWorkerEnvironment(source: NodeJS.ProcessEnv, identity: ShardIdentity, restartCount: number): NodeJS.ProcessEnv;
 export declare function shardRuntimeCompositionHash(modules: readonly ModuleDefinition[], internalGuildId?: string): string;
 export declare function boundedRespawnDelay(attempt: number): number;

package/dist/shard-supervisor.js

@@ -1,6 +1,7 @@
 import { safeError, ServiceRegistry } from "@helyx/core";
 import { checkDatabaseConnection, createDatabasePool, InteractionSessionRepository, RateLimitRepository, runConfiguredModuleMigrations, runCoreMigrations, RuntimeStateRepository, } from "@helyx/database";
 import { createPlatformServices, PostgresPrivacyProviderExecutor, } from "@helyx/platform";
+import { HELYX_SERVICE_NAMES } from "@helyx/sdk";
 import { Client, ShardingManager } from "discord.js";
 import { createHash, randomUUID } from "node:crypto";
 import { createRequire } from "node:module";
@@ -15,13 +16,16 @@ import { buildApplicationCommandDefinitions } from "./interactions.js";
 import { resolveModulePackageSpecifiers } from "./module-configuration.js";
 import { deriveSecureRandomSelectionKey, loadConfiguredModulePackages, platformServiceOptions, secureRandomSelectionWorkerKeyName, } from "./modules.js";
 import { createPermissionsModule } from "./permissions-module.js";
-import { SHARD_CONTROL_MAX_BYTES, SHARD_HEALTH_STALE_MS, assertShardMessageSize, controlOperationGuildId, executeShardControlOperation, gatewayBotInformationSchema, guildIdToShardId, shardCloudRequestSchema, shardHealthSnapshotSchema, workerMessageSchema, } from "./shard-contracts.js";
+import { SHARD_CONTROL_MAX_BYTES, SHARD_CONTROL_TIMEOUT_MS, SHARD_HEALTH_STALE_MS, assertShardMessageSize, controlOperationGuildId, executeShardControlOperation, gatewayBotInformationSchema, guildIdToShardId, shardCloudRequestSchema, shardHealthSnapshotSchema, workerMessageSchema, } from "./shard-contracts.js";
 import { RedisShardCoordinator, } from "./shard-coordinator.js";
 import { ShardControlRouter } from "./shard-router.js";
 import { createDeveloperOperationsModule } from "./developer-operations-module.js";
 import { SupervisorCloudDiscordDirectory } from "./supervisor-cloud-directory.js";
 import { createSupportModule } from "./support-module.js";
 import { createScheduledTaskRunner, schedulerOperationalHealth, } from "./scheduled-task-runner.js";
+import { numericWriteBehindRegistrations } from "./numeric-write-behind.js";
+import { executeRedisNumericOperation, RedisNumericWriteBehindService, } from "./numeric-write-behind-redis.js";
+import { encodeNumericIpcValue } from "./numeric-write-behind-proxy.js";
 const LEASE_RENEW_INTERVAL_MS = 10_000;
 const STARTUP_LEASE_RENEW_INTERVAL_MS = 5_000;
 const LEASE_VALIDATION_TIMEOUT_MS = 1_000;
@@ -31,6 +35,13 @@ const WORKER_SHUTDOWN_MS = 5_000;
 const MAX_RESTARTS_PER_WINDOW = 5;
 const RESTART_WINDOW_MS = 10 * 60 * 1_000;
 export const STEADY_STATE_LEASE_LOSS_BOUND_MS = LEASE_RENEW_INTERVAL_MS + LEASE_VALIDATION_TIMEOUT_MS + WORKER_SHUTDOWN_MS;
+export function createStartupWorkerHeartbeatMaintenance(heartbeat, intervalMs = STARTUP_LEASE_RENEW_INTERVAL_MS) {
+    const timer = setInterval(heartbeat, intervalMs);
+    timer.unref();
+    return {
+        stop: () => clearInterval(timer),
+    };
+}
 export async function deferScheduledRoutingFailures(route) {
     try {
         return await route();
@@ -43,6 +54,14 @@ export async function deferScheduledRoutingFailures(route) {
         };
     }
 }
+function sameNumericIdentity(left, right) {
+    return (left.generation === right.generation &&
+        left.deploymentOrder === right.deploymentOrder &&
+        left.supervisorId === right.supervisorId &&
+        left.shardId === right.shardId &&
+        left.shardCount === right.shardCount &&
+        left.fencingToken === right.fencingToken);
+}
 export async function startShardSupervisor(environment, processEnvironment, workerEntry) {
     if (environment.HELYX_SHARD_MODE === "off")
         throw new Error("Shard supervisor requires sharding to be enabled");
@@ -65,6 +84,8 @@ export async function startShardSupervisor(environment, processEnvironment, work
     let privacyDatabase;
     let controlServer;
     let coordinator;
+    let numericWriteBehind;
+    let numericRegistrations = [];
     let leaderLease;
     let cloudConnector;
     let cloudStarted = false;
@@ -73,6 +94,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
     let retentionTimer;
     let leaseTimer;
     let startupLeaseTimer;
+    let startupWorkerHeartbeats;
     let startupLeaseRefresh;
     let startupLeaseError;
     let shuttingDown = false;
@@ -80,6 +102,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
     const healthByShard = new Map();
     const pendingControl = new Map();
     const pendingScheduled = new Map();
+    const pendingNumericCommitted = new Map();
     const restartHistory = new Map();
     const restartCounts = new Map();
     const workers = new Map();
@@ -152,6 +175,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
             ...(cloudManagementModule ? [cloudManagementModule] : []),
             ...featureModules,
         ];
+        numericRegistrations = numericWriteBehindRegistrations(publicModules);
         scheduledModules = publicModules;
         const operationsModule = environment.DISCORD_INTERNAL_GUILD_ID
             ? createDeveloperOperationsModule({
@@ -186,6 +210,8 @@ export async function startShardSupervisor(environment, processEnvironment, work
             privacyDatabase = await createPrivacyDatabase(environment, database);
             assertStartupLeaseHealthy();
             const services = new Map(createPlatformServices(database, publicModules, platformServiceOptions(environment, publicModules)));
+            if (numericWriteBehind)
+                services.set(HELYX_SERVICE_NAMES.numericWriteBehind, numericWriteBehind);
             privacyService = new BotControlService(database, publicModules, unavailableResources, new ServiceRegistry(services), privacyDatabase
                 ? new PostgresPrivacyProviderExecutor({
                     readPool: database,
@@ -225,6 +251,21 @@ export async function startShardSupervisor(environment, processEnvironment, work
             });
             supervisorId = coordinator.supervisorId;
             await coordinator.start((targetShardId, request) => dispatchLocal(targetShardId, request), (request) => executeLeaderControl(request), (request) => executeLeaderCloud(request), (targetShardId, request) => dispatchLocalScheduled(targetShardId, request));
+            const numericPlatformServices = createPlatformServices(database, publicModules, platformServiceOptions(environment, publicModules));
+            numericWriteBehind = new RedisNumericWriteBehindService({
+                redisUrl: environment.HELYX_SHARD_REDIS_URL,
+                namespace: environment.HELYX_SHARD_REDIS_NAMESPACE,
+                writer: numericPlatformServices.get(HELYX_SERVICE_NAMES.records),
+                registrations: numericRegistrations,
+                supervisorId,
+                shardLeaseKeyPrefix: `helyx:shard:v1:${environment.HELYX_SHARD_REDIS_NAMESPACE}:lease:shard:`,
+                onError: (error) => logger.error({
+                    event: "numeric_write_behind.redis_failed",
+                    error: safeError(error),
+                }, "Persistent numeric write-behind operation failed"),
+            });
+            await numericWriteBehind.start();
+            installNumericCommitRouting(numericWriteBehind);
             deploymentOrder = await coordinator.assertGeneration(generationDescriptor, deploymentOrder);
             leaderLease = await coordinator.acquireLeader(generationDescriptor, deploymentOrder);
             startStartupLeaseMaintenance();
@@ -297,6 +338,11 @@ export async function startShardSupervisor(environment, processEnvironment, work
             execArgv: process.execArgv,
         });
         await reserveInitialSessionStarts(claimedIds.length, gateway);
+        if (coordinator) {
+            startupWorkerHeartbeats = createStartupWorkerHeartbeatMaintenance(() => {
+                void Promise.all([...localLeases].map(([shardId, lease]) => sendLeaseHeartbeat(shardId, lease)));
+            });
+        }
         await spawnInIdentifyBuckets(claimedIds, gateway.session_start_limit.max_concurrency);
         workersRoutable = true;
         if (leaderLease)
@@ -317,8 +363,11 @@ export async function startShardSupervisor(environment, processEnvironment, work
         await stopStartupLeaseMaintenance();
         if (coordinator)
             await withTimeout(renewStartupOwnership(coordinator, generationDescriptor, deploymentOrder, leaderLease), LEASE_VALIDATION_TIMEOUT_MS);
+        await Promise.all([...localLeases].map(([shardId, lease]) => sendLeaseHeartbeat(shardId, lease)));
         leaseTimer = setInterval(() => void maintainLeases(), LEASE_RENEW_INTERVAL_MS);
         leaseTimer.unref();
+        startupWorkerHeartbeats?.stop();
+        startupWorkerHeartbeats = undefined;
         process.once("SIGINT", () => void shutdown("SIGINT"));
         process.once("SIGTERM", () => void shutdown("SIGTERM"));
         logger.info({
@@ -414,17 +463,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
         };
         shard.env = buildShardWorkerEnvironment(processEnvironment, identity, restartCounts.get(shardId) ?? 0);
         healthByShard.set(shardId, startingHealth(shardId, restartCounts.get(shardId) ?? 0));
-        const launchHeartbeatTimer = coordinator
-            ? setInterval(() => void sendLeaseHeartbeat(shardId, lease), STARTUP_LEASE_RENEW_INTERVAL_MS)
-            : undefined;
-        launchHeartbeatTimer?.unref();
-        try {
-            await spawnWithShutdownGuard(() => shuttingDown, () => shard.spawn(30_000), () => shard.kill());
-        }
-        finally {
-            if (launchHeartbeatTimer)
-                clearInterval(launchHeartbeatTimer);
-        }
+        await spawnWithShutdownGuard(() => shuttingDown, () => shard.spawn(30_000), () => shard.kill());
         await sendLeaseHeartbeat(shardId, lease);
     }
     async function sendLeaseHeartbeat(shardId, lease) {
@@ -452,6 +491,135 @@ export async function startShardSupervisor(environment, processEnvironment, work
             ?.send(heartbeat)
             .catch(() => undefined);
     }
+    function installNumericCommitRouting(service) {
+        for (const registration of numericRegistrations) {
+            service.subscribeCommitted(registration, async (batch) => {
+                for (let index = 0; index < batch.mutations.length; index += 1) {
+                    const mutation = batch.mutations[index];
+                    const result = batch.results[index];
+                    const guildId = mutation?.identity.guild_id;
+                    if (typeof guildId !== "string" || !result)
+                        continue;
+                    const committedItem = splitNumericCommittedBatch(batch, index);
+                    const shardId = guildIdToShardId(guildId, shardCount);
+                    const lease = localLeases.get(shardId);
+                    const worker = workers.get(shardId);
+                    if (!lease || !worker?.process)
+                        throw new Error("Committed numeric batch owner is not local or available");
+                    const identity = {
+                        version: 1,
+                        mode: shardMode,
+                        scope: environment.HELYX_SHARD_SCOPE,
+                        generation: lease.generation,
+                        deploymentOrder: lease.deploymentOrder,
+                        supervisorId: lease.supervisorId,
+                        shardId,
+                        shardCount,
+                        fencingToken: lease.fencingToken,
+                    };
+                    const notification = {
+                        version: 1,
+                        type: "helyx.numeric.committed",
+                        requestId: randomUUID(),
+                        deadlineAt: new Date(Date.now() + SHARD_CONTROL_TIMEOUT_MS).toISOString(),
+                        identity,
+                        scope: {
+                            moduleId: batch.moduleId,
+                            collection: batch.collection,
+                        },
+                        payload: encodeNumericIpcValue(committedItem),
+                    };
+                    assertShardMessageSize(notification);
+                    const response = await dispatchNumericCommitted(shardId, identity, notification);
+                    if (!response.ok)
+                        throw new Error(`Committed numeric batch was not applied: ${response.safeCode}`);
+                }
+            });
+        }
+    }
+    function dispatchNumericCommitted(shardId, identity, notification) {
+        const worker = workers.get(shardId);
+        if (!worker?.process)
+            return Promise.reject(new Error("Committed numeric batch owner is unavailable"));
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                pendingNumericCommitted.delete(notification.requestId);
+                reject(new Error("Committed numeric batch acknowledgement timed out"));
+            }, SHARD_CONTROL_TIMEOUT_MS);
+            timer.unref();
+            pendingNumericCommitted.set(notification.requestId, {
+                shardId,
+                identity,
+                resolve,
+                timer,
+            });
+            void worker.send(notification).catch((error) => {
+                clearTimeout(timer);
+                pendingNumericCommitted.delete(notification.requestId);
+                reject(error instanceof Error
+                    ? error
+                    : new Error("Committed numeric delivery failed"));
+            });
+        });
+    }
+    async function handleNumericRequest(shardId, request) {
+        const lease = localLeases.get(shardId);
+        const numericService = numericWriteBehind;
+        const guildId = "guildId" in request.operation ? request.operation.guildId : undefined;
+        const valid = numericService !== undefined &&
+            lease !== undefined &&
+            request.identity.shardId === shardId &&
+            request.identity.shardCount === shardCount &&
+            request.identity.generation === lease.generation &&
+            request.identity.supervisorId === lease.supervisorId &&
+            request.identity.fencingToken === lease.fencingToken &&
+            request.identity.deploymentOrder === lease.deploymentOrder &&
+            request.identity.deploymentOrder !== undefined &&
+            new Date(request.deadlineAt).getTime() > Date.now() &&
+            (!guildId || guildIdToShardId(guildId, shardCount) === shardId);
+        let response;
+        if (!valid) {
+            response = {
+                version: 1,
+                type: "helyx.numeric.response",
+                requestId: request.requestId,
+                ok: false,
+                safeCode: "stale_owner",
+            };
+        }
+        else {
+            try {
+                if (!numericService)
+                    throw new Error("Numeric write-behind service is unavailable");
+                response = {
+                    version: 1,
+                    type: "helyx.numeric.response",
+                    requestId: request.requestId,
+                    ok: true,
+                    result: encodeNumericIpcValue(await executeRedisNumericOperation(numericService, request.operation, {
+                        identity: request.identity,
+                        shardLeaseKeyPrefix: `helyx:shard:v1:${environment.HELYX_SHARD_REDIS_NAMESPACE}:lease:shard:`,
+                    })),
+                };
+            }
+            catch (error) {
+                logger.error({
+                    event: "numeric_write_behind.ipc_failed",
+                    shardId,
+                    error: safeError(error),
+                }, "Numeric write-behind IPC operation failed");
+                response = {
+                    version: 1,
+                    type: "helyx.numeric.response",
+                    requestId: request.requestId,
+                    ok: false,
+                    safeCode: "operation_failed",
+                };
+            }
+        }
+        assertShardMessageSize(response);
+        await workers.get(shardId)?.send(response);
+    }
     async function handleWorkerMessage(shardId, raw) {
         if (serializedBytes(raw) > SHARD_CONTROL_MAX_BYTES)
             return;
@@ -489,6 +657,21 @@ export async function startShardSupervisor(environment, processEnvironment, work
             pending.resolve(message.data);
             return;
         }
+        if (message.data.type === "helyx.numeric.committed.response") {
+            const pending = pendingNumericCommitted.get(message.data.requestId);
+            if (!pending ||
+                pending.shardId !== shardId ||
+                !sameNumericIdentity(pending.identity, message.data.identity))
+                return;
+            pendingNumericCommitted.delete(message.data.requestId);
+            clearTimeout(pending.timer);
+            pending.resolve(message.data);
+            return;
+        }
+        if (message.data.type === "helyx.numeric.request") {
+            await handleNumericRequest(shardId, message.data);
+            return;
+        }
         if (message.data.type === "helyx.cloud.request") {
             const response = leaderLease
                 ? await executeLeaderCloud(message.data)
@@ -985,6 +1168,8 @@ export async function startShardSupervisor(environment, processEnvironment, work
             clearInterval(leaseTimer);
         if (startupLeaseTimer)
             clearInterval(startupLeaseTimer);
+        startupWorkerHeartbeats?.stop();
+        startupWorkerHeartbeats = undefined;
         if (retentionTimer)
             clearInterval(retentionTimer);
         workersRoutable = false;
@@ -1006,6 +1191,23 @@ export async function startShardSupervisor(environment, processEnvironment, work
             if (shard.process)
                 shard.kill();
         }));
+        if (numericWriteBehind) {
+            try {
+                const { drained } = await numericWriteBehind.stop();
+                if (!drained) {
+                    process.exitCode = 1;
+                    logger.error({ event: "numeric_write_behind.shutdown_not_drained", reason }, "Numeric write-behind exceeded its bounded shutdown drain");
+                }
+            }
+            catch (error) {
+                process.exitCode = 1;
+                logger.error({
+                    event: "numeric_write_behind.shutdown_failed",
+                    reason,
+                    error: safeError(error),
+                }, "Numeric write-behind shutdown failed");
+            }
+        }
         await cloudConnector?.stop().catch(() => undefined);
         if (coordinator) {
             const activeCoordinator = coordinator;
@@ -1019,6 +1221,18 @@ export async function startShardSupervisor(environment, processEnvironment, work
         logger.info({ event: "shard.supervisor_stopped", reason }, "Shard supervisor stopped");
     }
 }
+export function splitNumericCommittedBatch(batch, index) {
+    const mutation = batch.mutations[index];
+    const result = batch.results[index];
+    if (!Number.isSafeInteger(index) || index < 0 || !mutation || !result)
+        throw new Error("Numeric committed batch item is unavailable");
+    return {
+        ...batch,
+        itemIndexes: [batch.itemIndexes?.[index] ?? index],
+        mutations: [mutation],
+        results: [result],
+    };
+}
 async function runLeaderStartup(database, environment, modulePackages, modules, logger) {
     await runCoreMigrations(database);
     await runConfiguredModuleMigrations(database, modulePackages.map(({ definition, packageRoot }) => ({
@@ -1130,6 +1344,7 @@ export function buildShardWorkerEnvironment(source, identity, restartCount) {
         "DISCORD_INTERNAL_GUILD_ID",
         "HELYX_SUPPORT_STAFF_CHANNEL_ID",
         "HELYX_MODULE_PACKAGES",
+        "HELYX_MESSAGE_CONTENT_ENABLED",
         "HELYX_MODULE_CATALOGUE_PATH",
         "HELYX_CLOUD_CONNECT_ENABLED",
         "HELYX_OWNER_USER_IDS",