@helyx/bot 0.4.9 → 0.4.13

package/dist/shard-supervisor.js

@@ -1,6 +1,7 @@
 import { safeError, ServiceRegistry } from "@helyx/core";
-import { checkDatabaseConnection, createDatabasePool, InteractionSessionRepository, RateLimitRepository, runConfiguredModuleMigrations, runCoreMigrations, RuntimeStateRepository, } from "@helyx/database";
+import { checkDatabaseConnection, createDatabasePool, AuditRepository, InteractionSessionRepository, RateLimitRepository, runConfiguredModuleMigrations, runCoreMigrations, RuntimeStateRepository, } from "@helyx/database";
 import { createPlatformServices, PostgresPrivacyProviderExecutor, } from "@helyx/platform";
+import { HELYX_SERVICE_NAMES } from "@helyx/sdk";
 import { Client, ShardingManager } from "discord.js";
 import { createHash, randomUUID } from "node:crypto";
 import { createRequire } from "node:module";
@@ -15,22 +16,38 @@ import { buildApplicationCommandDefinitions } from "./interactions.js";
 import { resolveModulePackageSpecifiers } from "./module-configuration.js";
 import { deriveSecureRandomSelectionKey, loadConfiguredModulePackages, platformServiceOptions, secureRandomSelectionWorkerKeyName, } from "./modules.js";
 import { createPermissionsModule } from "./permissions-module.js";
-import { SHARD_CONTROL_MAX_BYTES, SHARD_HEALTH_STALE_MS, assertShardMessageSize, controlOperationGuildId, executeShardControlOperation, gatewayBotInformationSchema, guildIdToShardId, shardCloudRequestSchema, shardHealthSnapshotSchema, workerMessageSchema, } from "./shard-contracts.js";
+import { SHARD_CONTROL_MAX_BYTES, SHARD_CONTROL_TIMEOUT_MS, SHARD_HEALTH_STALE_MS, assertShardMessageSize, controlOperationGuildId, executeShardControlOperation, gatewayBotInformationSchema, guildIdToShardId, shardCloudRequestSchema, shardHealthSnapshotSchema, workerMessageSchema, } from "./shard-contracts.js";
 import { RedisShardCoordinator, } from "./shard-coordinator.js";
 import { ShardControlRouter } from "./shard-router.js";
 import { createDeveloperOperationsModule } from "./developer-operations-module.js";
 import { SupervisorCloudDiscordDirectory } from "./supervisor-cloud-directory.js";
 import { createSupportModule } from "./support-module.js";
 import { createScheduledTaskRunner, schedulerOperationalHealth, } from "./scheduled-task-runner.js";
+import { numericWriteBehindRegistrations } from "./numeric-write-behind.js";
+import { executeRedisNumericOperation, RedisNumericWriteBehindService, } from "./numeric-write-behind-redis.js";
+import { encodeNumericIpcValue } from "./numeric-write-behind-proxy.js";
 const LEASE_RENEW_INTERVAL_MS = 10_000;
 const STARTUP_LEASE_RENEW_INTERVAL_MS = 5_000;
 const LEASE_VALIDATION_TIMEOUT_MS = 1_000;
+const LEASE_RENEW_ATTEMPTS = 2;
 const RETENTION_CLEANUP_INTERVAL_MS = 60 * 60 * 1_000;
 const STARTUP_WAIT_MS = 60_000;
 const WORKER_SHUTDOWN_MS = 5_000;
 const MAX_RESTARTS_PER_WINDOW = 5;
 const RESTART_WINDOW_MS = 10 * 60 * 1_000;
-export const STEADY_STATE_LEASE_LOSS_BOUND_MS = LEASE_RENEW_INTERVAL_MS + LEASE_VALIDATION_TIMEOUT_MS + WORKER_SHUTDOWN_MS;
+export const STEADY_STATE_LEASE_LOSS_BOUND_MS = LEASE_RENEW_INTERVAL_MS +
+    LEASE_VALIDATION_TIMEOUT_MS * LEASE_RENEW_ATTEMPTS +
+    WORKER_SHUTDOWN_MS;
+const FATAL_SUPERVISOR_SHUTDOWN_REASONS = new Set([
+    "generation-lease-lost",
+    "leader-lease-lost",
+    "leader-promotion-failed",
+    "rebalance-failed",
+    "coordination-unavailable",
+]);
+export function supervisorShutdownExitCode(reason) {
+    return FATAL_SUPERVISOR_SHUTDOWN_REASONS.has(reason) ? 1 : undefined;
+}
 export function createStartupWorkerHeartbeatMaintenance(heartbeat, intervalMs = STARTUP_LEASE_RENEW_INTERVAL_MS) {
     const timer = setInterval(heartbeat, intervalMs);
     timer.unref();
@@ -50,6 +67,14 @@ export async function deferScheduledRoutingFailures(route) {
         };
     }
 }
+function sameNumericIdentity(left, right) {
+    return (left.generation === right.generation &&
+        left.deploymentOrder === right.deploymentOrder &&
+        left.supervisorId === right.supervisorId &&
+        left.shardId === right.shardId &&
+        left.shardCount === right.shardCount &&
+        left.fencingToken === right.fencingToken);
+}
 export async function startShardSupervisor(environment, processEnvironment, workerEntry) {
     if (environment.HELYX_SHARD_MODE === "off")
         throw new Error("Shard supervisor requires sharding to be enabled");
@@ -72,6 +97,8 @@ export async function startShardSupervisor(environment, processEnvironment, work
     let privacyDatabase;
     let controlServer;
     let coordinator;
+    let numericWriteBehind;
+    let numericRegistrations = [];
     let leaderLease;
     let cloudConnector;
     let cloudStarted = false;
@@ -88,6 +115,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
     const healthByShard = new Map();
     const pendingControl = new Map();
     const pendingScheduled = new Map();
+    const pendingNumericCommitted = new Map();
     const restartHistory = new Map();
     const restartCounts = new Map();
     const workers = new Map();
@@ -135,6 +163,11 @@ export async function startShardSupervisor(environment, processEnvironment, work
         });
         const commandClient = new Client({ intents: [] });
         const unavailableResources = {
+            listGuildPresence: (guildIds) => Promise.resolve(guildIds.map((guildId) => ({
+                guildId,
+                state: "unknown",
+                observedAt: new Date().toISOString(),
+            }))),
             getGuildResources: () => Promise.reject(new Error("Guild resources are worker-owned")),
             sendToChannel: () => Promise.reject(new Error("Guild resources are worker-owned")),
             deleteChannelMessage: () => Promise.reject(new Error("Guild resources are worker-owned")),
@@ -160,6 +193,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
             ...(cloudManagementModule ? [cloudManagementModule] : []),
             ...featureModules,
         ];
+        numericRegistrations = numericWriteBehindRegistrations(publicModules);
         scheduledModules = publicModules;
         const operationsModule = environment.DISCORD_INTERNAL_GUILD_ID
             ? createDeveloperOperationsModule({
@@ -194,6 +228,8 @@ export async function startShardSupervisor(environment, processEnvironment, work
             privacyDatabase = await createPrivacyDatabase(environment, database);
             assertStartupLeaseHealthy();
             const services = new Map(createPlatformServices(database, publicModules, platformServiceOptions(environment, publicModules)));
+            if (numericWriteBehind)
+                services.set(HELYX_SERVICE_NAMES.numericWriteBehind, numericWriteBehind);
             privacyService = new BotControlService(database, publicModules, unavailableResources, new ServiceRegistry(services), privacyDatabase
                 ? new PostgresPrivacyProviderExecutor({
                     readPool: database,
@@ -233,6 +269,21 @@ export async function startShardSupervisor(environment, processEnvironment, work
             });
             supervisorId = coordinator.supervisorId;
             await coordinator.start((targetShardId, request) => dispatchLocal(targetShardId, request), (request) => executeLeaderControl(request), (request) => executeLeaderCloud(request), (targetShardId, request) => dispatchLocalScheduled(targetShardId, request));
+            const numericPlatformServices = createPlatformServices(database, publicModules, platformServiceOptions(environment, publicModules));
+            numericWriteBehind = new RedisNumericWriteBehindService({
+                redisUrl: environment.HELYX_SHARD_REDIS_URL,
+                namespace: environment.HELYX_SHARD_REDIS_NAMESPACE,
+                writer: numericPlatformServices.get(HELYX_SERVICE_NAMES.records),
+                registrations: numericRegistrations,
+                supervisorId,
+                shardLeaseKeyPrefix: `helyx:shard:v1:${environment.HELYX_SHARD_REDIS_NAMESPACE}:lease:shard:`,
+                onError: (error) => logger.error({
+                    event: "numeric_write_behind.redis_failed",
+                    error: safeError(error),
+                }, "Persistent numeric write-behind operation failed"),
+            });
+            await numericWriteBehind.start();
+            installNumericCommitRouting(numericWriteBehind);
             deploymentOrder = await coordinator.assertGeneration(generationDescriptor, deploymentOrder);
             leaderLease = await coordinator.acquireLeader(generationDescriptor, deploymentOrder);
             startStartupLeaseMaintenance();
@@ -458,6 +509,135 @@ export async function startShardSupervisor(environment, processEnvironment, work
             ?.send(heartbeat)
             .catch(() => undefined);
     }
+    function installNumericCommitRouting(service) {
+        for (const registration of numericRegistrations) {
+            service.subscribeCommitted(registration, async (batch) => {
+                for (let index = 0; index < batch.mutations.length; index += 1) {
+                    const mutation = batch.mutations[index];
+                    const result = batch.results[index];
+                    const guildId = mutation?.identity.guild_id;
+                    if (typeof guildId !== "string" || !result)
+                        continue;
+                    const committedItem = splitNumericCommittedBatch(batch, index);
+                    const shardId = guildIdToShardId(guildId, shardCount);
+                    const lease = localLeases.get(shardId);
+                    const worker = workers.get(shardId);
+                    if (!lease || !worker?.process)
+                        throw new Error("Committed numeric batch owner is not local or available");
+                    const identity = {
+                        version: 1,
+                        mode: shardMode,
+                        scope: environment.HELYX_SHARD_SCOPE,
+                        generation: lease.generation,
+                        deploymentOrder: lease.deploymentOrder,
+                        supervisorId: lease.supervisorId,
+                        shardId,
+                        shardCount,
+                        fencingToken: lease.fencingToken,
+                    };
+                    const notification = {
+                        version: 1,
+                        type: "helyx.numeric.committed",
+                        requestId: randomUUID(),
+                        deadlineAt: new Date(Date.now() + SHARD_CONTROL_TIMEOUT_MS).toISOString(),
+                        identity,
+                        scope: {
+                            moduleId: batch.moduleId,
+                            collection: batch.collection,
+                        },
+                        payload: encodeNumericIpcValue(committedItem),
+                    };
+                    assertShardMessageSize(notification);
+                    const response = await dispatchNumericCommitted(shardId, identity, notification);
+                    if (!response.ok)
+                        throw new Error(`Committed numeric batch was not applied: ${response.safeCode}`);
+                }
+            });
+        }
+    }
+    function dispatchNumericCommitted(shardId, identity, notification) {
+        const worker = workers.get(shardId);
+        if (!worker?.process)
+            return Promise.reject(new Error("Committed numeric batch owner is unavailable"));
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                pendingNumericCommitted.delete(notification.requestId);
+                reject(new Error("Committed numeric batch acknowledgement timed out"));
+            }, SHARD_CONTROL_TIMEOUT_MS);
+            timer.unref();
+            pendingNumericCommitted.set(notification.requestId, {
+                shardId,
+                identity,
+                resolve,
+                timer,
+            });
+            void worker.send(notification).catch((error) => {
+                clearTimeout(timer);
+                pendingNumericCommitted.delete(notification.requestId);
+                reject(error instanceof Error
+                    ? error
+                    : new Error("Committed numeric delivery failed"));
+            });
+        });
+    }
+    async function handleNumericRequest(shardId, request) {
+        const lease = localLeases.get(shardId);
+        const numericService = numericWriteBehind;
+        const guildId = "guildId" in request.operation ? request.operation.guildId : undefined;
+        const valid = numericService !== undefined &&
+            lease !== undefined &&
+            request.identity.shardId === shardId &&
+            request.identity.shardCount === shardCount &&
+            request.identity.generation === lease.generation &&
+            request.identity.supervisorId === lease.supervisorId &&
+            request.identity.fencingToken === lease.fencingToken &&
+            request.identity.deploymentOrder === lease.deploymentOrder &&
+            request.identity.deploymentOrder !== undefined &&
+            new Date(request.deadlineAt).getTime() > Date.now() &&
+            (!guildId || guildIdToShardId(guildId, shardCount) === shardId);
+        let response;
+        if (!valid) {
+            response = {
+                version: 1,
+                type: "helyx.numeric.response",
+                requestId: request.requestId,
+                ok: false,
+                safeCode: "stale_owner",
+            };
+        }
+        else {
+            try {
+                if (!numericService)
+                    throw new Error("Numeric write-behind service is unavailable");
+                response = {
+                    version: 1,
+                    type: "helyx.numeric.response",
+                    requestId: request.requestId,
+                    ok: true,
+                    result: encodeNumericIpcValue(await executeRedisNumericOperation(numericService, request.operation, {
+                        identity: request.identity,
+                        shardLeaseKeyPrefix: `helyx:shard:v1:${environment.HELYX_SHARD_REDIS_NAMESPACE}:lease:shard:`,
+                    })),
+                };
+            }
+            catch (error) {
+                logger.error({
+                    event: "numeric_write_behind.ipc_failed",
+                    shardId,
+                    error: safeError(error),
+                }, "Numeric write-behind IPC operation failed");
+                response = {
+                    version: 1,
+                    type: "helyx.numeric.response",
+                    requestId: request.requestId,
+                    ok: false,
+                    safeCode: "operation_failed",
+                };
+            }
+        }
+        assertShardMessageSize(response);
+        await workers.get(shardId)?.send(response);
+    }
     async function handleWorkerMessage(shardId, raw) {
         if (serializedBytes(raw) > SHARD_CONTROL_MAX_BYTES)
             return;
@@ -495,6 +675,21 @@ export async function startShardSupervisor(environment, processEnvironment, work
             pending.resolve(message.data);
             return;
         }
+        if (message.data.type === "helyx.numeric.committed.response") {
+            const pending = pendingNumericCommitted.get(message.data.requestId);
+            if (!pending ||
+                pending.shardId !== shardId ||
+                !sameNumericIdentity(pending.identity, message.data.identity))
+                return;
+            pendingNumericCommitted.delete(message.data.requestId);
+            clearTimeout(pending.timer);
+            pending.resolve(message.data);
+            return;
+        }
+        if (message.data.type === "helyx.numeric.request") {
+            await handleNumericRequest(shardId, message.data);
+            return;
+        }
         if (message.data.type === "helyx.cloud.request") {
             const response = leaderLease
                 ? await executeLeaderCloud(message.data)
@@ -706,8 +901,19 @@ export async function startShardSupervisor(environment, processEnvironment, work
         try {
             if (!coordinator)
                 return;
-            const renewal = await renewSteadyStateOwnership(coordinator, generationDescriptor, deploymentOrder, localLeases, leaderLease).catch(() => undefined);
-            if (!renewal?.generationRenewed) {
+            let renewal;
+            try {
+                renewal = await retryOwnershipRenewal(() => renewSteadyStateOwnership(coordinator, generationDescriptor, deploymentOrder, localLeases, leaderLease));
+            }
+            catch (error) {
+                logger.error({
+                    event: "shard.coordination_renewal_failed",
+                    error: safeError(error),
+                }, "Shard coordination could not be renewed after a bounded retry");
+                await shutdown("coordination-unavailable");
+                return;
+            }
+            if (!renewal.generationRenewed) {
                 logger.error({ event: "shard.generation_lease_lost" }, "Shard generation lease was lost; stopping this supervisor safely");
                 await shutdown("generation-lease-lost");
                 return;
@@ -987,6 +1193,9 @@ export async function startShardSupervisor(environment, processEnvironment, work
         if (shuttingDown)
             return;
         shuttingDown = true;
+        const shutdownExitCode = supervisorShutdownExitCode(reason);
+        if (shutdownExitCode !== undefined)
+            process.exitCode = shutdownExitCode;
         if (leaseTimer)
             clearInterval(leaseTimer);
         if (startupLeaseTimer)
@@ -1014,6 +1223,23 @@ export async function startShardSupervisor(environment, processEnvironment, work
             if (shard.process)
                 shard.kill();
         }));
+        if (numericWriteBehind) {
+            try {
+                const { drained } = await numericWriteBehind.stop();
+                if (!drained) {
+                    process.exitCode = 1;
+                    logger.error({ event: "numeric_write_behind.shutdown_not_drained", reason }, "Numeric write-behind exceeded its bounded shutdown drain");
+                }
+            }
+            catch (error) {
+                process.exitCode = 1;
+                logger.error({
+                    event: "numeric_write_behind.shutdown_failed",
+                    reason,
+                    error: safeError(error),
+                }, "Numeric write-behind shutdown failed");
+            }
+        }
         await cloudConnector?.stop().catch(() => undefined);
         if (coordinator) {
             const activeCoordinator = coordinator;
@@ -1027,6 +1253,18 @@ export async function startShardSupervisor(environment, processEnvironment, work
         logger.info({ event: "shard.supervisor_stopped", reason }, "Shard supervisor stopped");
     }
 }
+export function splitNumericCommittedBatch(batch, index) {
+    const mutation = batch.mutations[index];
+    const result = batch.results[index];
+    if (!Number.isSafeInteger(index) || index < 0 || !mutation || !result)
+        throw new Error("Numeric committed batch item is unavailable");
+    return {
+        ...batch,
+        itemIndexes: [batch.itemIndexes?.[index] ?? index],
+        mutations: [mutation],
+        results: [result],
+    };
+}
 async function runLeaderStartup(database, environment, modulePackages, modules, logger) {
     await runCoreMigrations(database);
     await runConfiguredModuleMigrations(database, modulePackages.map(({ definition, packageRoot }) => ({
@@ -1138,6 +1376,7 @@ export function buildShardWorkerEnvironment(source, identity, restartCount) {
         "DISCORD_INTERNAL_GUILD_ID",
         "HELYX_SUPPORT_STAFF_CHANNEL_ID",
         "HELYX_MODULE_PACKAGES",
+        "HELYX_MESSAGE_CONTENT_ENABLED",
         "HELYX_MODULE_CATALOGUE_PATH",
         "HELYX_CLOUD_CONNECT_ENABLED",
         "HELYX_OWNER_USER_IDS",
@@ -1243,15 +1482,17 @@ function failedCloud(requestId, message) {
 }
 async function cleanupExpiredCoreRecords(database, logger) {
     try {
-        const [sessions, rateLimits] = await Promise.all([
+        const [sessions, rateLimits, verificationAudit] = await Promise.all([
             new InteractionSessionRepository(database).deleteExpired(),
             new RateLimitRepository(database).deleteExpired(),
+            new AuditRepository(database).deleteModuleEventsOlderThanMonths("helyx.verification", 12),
         ]);
-        if (sessions > 0 || rateLimits > 0)
+        if (sessions > 0 || rateLimits > 0 || verificationAudit > 0)
             logger.info({
                 event: "database.retention_cleaned",
                 interactionSessions: sessions,
                 rateLimitWindows: rateLimits,
+                verificationAuditEvents: verificationAudit,
             }, "Expired core records removed");
     }
     catch (error) {
@@ -1302,6 +1543,20 @@ async function waitForDeath(shard, timeoutMs) {
 export function boundedRespawnDelay(attempt) {
     return Math.min(30_000, 1_000 * 2 ** Math.max(0, attempt - 1));
 }
+export async function retryOwnershipRenewal(renew, attempts = LEASE_RENEW_ATTEMPTS) {
+    if (!Number.isSafeInteger(attempts) || attempts < 1)
+        throw new Error("Ownership renewal attempts must be a positive integer");
+    let lastError;
+    for (let attempt = 0; attempt < attempts; attempt += 1) {
+        try {
+            return await renew();
+        }
+        catch (error) {
+            lastError = error;
+        }
+    }
+    throw lastError;
+}
 export function selectFencedShardHealth(input) {
     const liveByShard = new Map(input.liveLeases.map((lease) => [lease.shardId, lease]));
     const selected = new Map();

package/dist/shard-worker.d.ts

@@ -35,8 +35,12 @@ export declare function parseShardWorkerEnvironment(processEnvironment: NodeJS.P
     HELYX_MODULE_PACKAGES: string[];
     HELYX_MODULE_CATALOGUE_PATH: string | undefined;
     HELYX_SUPPORT_STAFF_CHANNEL_ID: string | undefined;
+    HELYX_VERIFICATION_ENABLED: boolean;
+    HELYX_VERIFICATION_SIGNING_PUBLIC_KEYS: Record<string, string>;
     HELYX_BOT_CONTROL_PORT: number;
+    HELYX_GUILD_RECONCILIATION_INTERVAL_SECONDS: number;
     HELYX_PRIVACY_ENABLED: boolean;
+    HELYX_MESSAGE_CONTENT_ENABLED: boolean;
     HELYX_CLOUD_CONNECT_ENABLED: boolean;
     HELYX_CLOUD_ENROLMENT_TOKEN: string | undefined;
     HELYX_CLOUD_ENROLMENT_FILE: string;
@@ -51,6 +55,9 @@ export declare function parseShardWorkerEnvironment(processEnvironment: NodeJS.P
     NODE_ENV: "development" | "test" | "production";
     HELYX_BOT_CONTROL_SECRET?: string | undefined;
     HELYX_SUPPORT_SERVICE_SECRET?: string | undefined;
+    HELYX_VERIFICATION_INSTALLATION_ID?: string | undefined;
+    HELYX_VERIFICATION_URL?: string | undefined;
+    HELYX_VERIFICATION_INTERNAL_SECRET?: string | undefined;
     HELYX_PRIVACY_EXECUTOR_DATABASE_URL?: string | undefined;
     HELYX_PRIVACY_EXECUTOR_DATABASE_SSL?: "disable" | "require" | "verify-full" | undefined;
     HELYX_SHARD_COUNT?: number | undefined;

package/dist/shard-worker.js

@@ -1,6 +1,6 @@
 import { parseBotEnvironment } from "@helyx/config";
 import { safeError, ServiceRegistry } from "@helyx/core";
-import { checkDatabaseConnection, createDatabasePool } from "@helyx/database";
+import { checkDatabaseConnection, createDatabasePool, VerificationSetupRepository, } from "@helyx/database";
 import { createRuntime } from "@helyx/framework";
 import { createPlatformServices } from "@helyx/platform";
 import { HELYX_SERVICE_NAMES } from "@helyx/sdk";
@@ -9,8 +9,9 @@ import { monitorEventLoopDelay } from "node:perf_hooks";
 import pino from "pino";
 import { createCloudManagementModule } from "./cloud-management-module.js";
 import { BotControlService, DiscordGuildResourceDirectory, mapBotControlError, } from "./control-server.js";
-import { DiscordMemberRoleService, DiscordResourceInspection, installModuleEvents, } from "./events.js";
-import { installGuildTracking } from "./guilds.js";
+import { HttpVerificationService } from "./verification-service.js";
+import { DiscordMessageDeliveryService, DiscordMemberEnforcementService, DiscordMemberRoleService, DiscordVerificationMemberService, DiscordVerificationSetupService, DiscordResourceInspection, installModuleEvents, } from "./events.js";
+import { installGuildTracking, } from "./guilds.js";
 import { installInteractions, } from "./interactions.js";
 import { createManagementModule } from "./management-module.js";
 import { DiscordModuleLoggingService } from "./module-logging.js";
@@ -25,6 +26,8 @@ import { assertShardMessageSize, controlOperationGuildId, executeShardControlOpe
 import { gatewayIntentsForModules, readBotVersion, } from "./index.js";
 import { createSupportModule } from "./support-module.js";
 import { createLocalScheduledTaskRoute, } from "./scheduled-task-runner.js";
+import { createLocalNumericWriteBehind } from "./numeric-write-behind.js";
+import { WorkerNumericWriteBehindProxy } from "./numeric-write-behind-proxy.js";
 const HEALTH_INTERVAL_MS = 5_000;
 export const WORKER_ORPHAN_TERMINATION_BOUND_MS = SHARD_LEASE_HEARTBEAT_TIMEOUT_MS + 5_000;
 export function createLeaseHeartbeatWatchdog(onExpired, timeoutMs = SHARD_LEASE_HEARTBEAT_TIMEOUT_MS) {
@@ -71,16 +74,19 @@ export async function startShardWorker(processEnvironment = process.env, identit
     let shuttingDown = false;
     let runtimeStarted = false;
     let moduleEvents;
+    let guildTracking;
     let client;
     let cloudProxy;
     let shardHealthProvider;
     let leaseHeartbeatWatchdog;
     let forcedLeaseExit;
+    let numericWriteBehind;
     async function shutdown(reason = "programmatic") {
         if (shuttingDown)
             return;
         shuttingDown = true;
         status = "stopped";
+        guildTracking?.stop();
         if (healthTimer)
             clearInterval(healthTimer);
         leaseHeartbeatWatchdog?.stop();
@@ -92,6 +98,7 @@ export async function startShardWorker(processEnvironment = process.env, identit
         await moduleEvents?.stop().catch(() => undefined);
         if (runtimeStarted)
             await runtime?.stop().catch(() => undefined);
+        await numericWriteBehind?.stop();
         await client?.destroy();
         eventLoopDelay.disable();
         await database.end();
@@ -104,7 +111,7 @@ export async function startShardWorker(processEnvironment = process.env, identit
         const modulePackages = await loadConfiguredModulePackages(moduleSpecifiers);
         const featureModules = modulePackages.map(({ definition }) => definition);
         client = new Client({
-            intents: gatewayIntentsForModules(featureModules),
+            intents: gatewayIntentsForModules(featureModules, environment.HELYX_MESSAGE_CONTENT_ENABLED),
             partials: [Partials.Message, Partials.Channel, Partials.Reaction],
             allowedMentions: { parse: [] },
             shards: [identity.shardId],
@@ -156,14 +163,39 @@ export async function startShardWorker(processEnvironment = process.env, identit
             ...publicModules,
             ...(operationsModule ? [operationsModule] : []),
         ];
-        const platformServices = new Map(createPlatformServices(database, publicModules, platformServiceOptions(environment, publicModules)));
+        const platformServices = new Map();
+        const serviceAccess = new ServiceRegistry(platformServices);
+        for (const [name, service] of createPlatformServices(database, publicModules, {
+            ...platformServiceOptions(environment, publicModules),
+            activationServices: serviceAccess,
+        })) {
+            platformServices.set(name, service);
+        }
+        if (environment.HELYX_VERIFICATION_ENABLED &&
+            environment.HELYX_VERIFICATION_URL &&
+            environment.HELYX_VERIFICATION_INTERNAL_SECRET) {
+            platformServices.set(HELYX_SERVICE_NAMES.verification, new HttpVerificationService({
+                baseUrl: environment.HELYX_VERIFICATION_URL,
+                internalSecret: environment.HELYX_VERIFICATION_INTERNAL_SECRET,
+                installationId: environment.HELYX_VERIFICATION_INSTALLATION_ID,
+                publicSigningKeys: environment.HELYX_VERIFICATION_SIGNING_PUBLIC_KEYS,
+            }));
+            platformServices.set(HELYX_SERVICE_NAMES.verificationMembers, new DiscordVerificationMemberService(client));
+            platformServices.set(HELYX_SERVICE_NAMES.verificationSetup, new DiscordVerificationSetupService(client, new VerificationSetupRepository(database)));
+        }
+        numericWriteBehind =
+            identity.scope === "coordinated"
+                ? new WorkerNumericWriteBehindProxy(identity)
+                : createLocalNumericWriteBehind(platformServices, publicModules);
+        platformServices.set(HELYX_SERVICE_NAMES.numericWriteBehind, numericWriteBehind);
         platformServices.set(HELYX_SERVICE_NAMES.resourceAuthorization, new CoreResourceAuthorizationService(platformServices.get(HELYX_SERVICE_NAMES.permissions)));
+        platformServices.set(HELYX_SERVICE_NAMES.discordMessages, new DiscordMessageDeliveryService(client));
         platformServices.set(HELYX_SERVICE_NAMES.discordOwnedMessages, new DiscordOwnedMessageDeletionService(client));
+        platformServices.set(HELYX_SERVICE_NAMES.memberEnforcement, new DiscordMemberEnforcementService(client));
         platformServices.set(HELYX_SERVICE_NAMES.memberRoles, new DiscordMemberRoleService(client));
         platformServices.set(HELYX_SERVICE_NAMES.discordResources, new DiscordResourceInspection(client));
         if (featureModules.some(({ manifest }) => manifest.id === "helyx.logging"))
             platformServices.set(HELYX_SERVICE_NAMES.moduleLogging, new DiscordModuleLoggingService(client, database, publicModules, logger));
-        const serviceAccess = new ServiceRegistry(platformServices);
         const scheduledTaskRoute = createLocalScheduledTaskRoute({
             modules: publicModules,
             services: serviceAccess,
@@ -177,13 +209,15 @@ export async function startShardWorker(processEnvironment = process.env, identit
             ...(cloudManagementModule ? [cloudManagementModule.manifest.id] : []),
             ...(operationsModule ? [operationsModule.manifest.id] : []),
         ]);
-        const prepareGuilds = installGuildTracking(client, database, publicModules, logger, enabledCoreModuleIds, {
+        const shardOwnership = {
             generation: identity.generation,
             deploymentOrder: BigInt(identity.deploymentOrder ?? 1),
             supervisorId: identity.supervisorId,
             shardId: identity.shardId,
             fencingToken: BigInt(identity.fencingToken),
-        });
+        };
+        const prepareGuilds = installGuildTracking(client, database, publicModules, logger, enabledCoreModuleIds, shardOwnership, environment.HELYX_GUILD_RECONCILIATION_INTERVAL_SECONDS * 1_000, identity.shardCount);
+        guildTracking = prepareGuilds;
         const commandsReady = installInteractions(client, modules, shardWorkerCommandRegistration({
             ...(environment.DISCORD_INTERNAL_GUILD_ID
                 ? { internalGuildId: environment.DISCORD_INTERNAL_GUILD_ID }
@@ -201,7 +235,7 @@ export async function startShardWorker(processEnvironment = process.env, identit
                 guildId: environment.DISCORD_INTERNAL_GUILD_ID,
                 channelId: environment.HELYX_SUPPORT_STAFF_CHANNEL_ID,
             }
-            : undefined);
+            : undefined, shardOwnership, identity.shardCount);
         process.on("message", (raw) => {
             const parsed = supervisorMessageSchema.safeParse(raw);
             if (!parsed.success)
@@ -436,7 +470,27 @@ async function handleControl(identity, service, request, logger) {
             };
         }
     }
-    assertShardMessageSize(response);
+    try {
+        assertShardMessageSize(response);
+    }
+    catch (error) {
+        logger.error({
+            event: "shard.control_response_oversized",
+            requestId: request.requestId,
+            operation: request.operation.kind,
+            guildId,
+            error: safeError(error),
+        }, "Shard control response exceeded the transport limit");
+        response = {
+            version: 1,
+            type: "helyx.control.response",
+            requestId: request.requestId,
+            ok: false,
+            status: 502,
+            message: "The owning shard returned too much data",
+        };
+        assertShardMessageSize(response);
+    }
     process.send?.(response);
 }
 function health(identity, client, status, eventLoopDelayNanoseconds, queueDepth) {

package/dist/verification-service.d.ts

@@ -0,0 +1,50 @@
+import { type ActionReceipt, type ChallengeCreateRequest, type ChallengeCreateResponse, type DashboardAttemptPage, type DashboardAttemptQuery, type DashboardSafeAttemptDetail, type SignedAssessmentEnvelope, type VerificationInstallationClaims } from "@helyx/verification-contracts";
+import type { VerificationResultContext, VerificationService } from "@helyx/sdk";
+export interface HttpVerificationServiceOptions {
+    baseUrl: string;
+    internalSecret: string;
+    publicSigningKeys: Readonly<Record<string, string>>;
+    installationId: string;
+    fetchImplementation?: typeof fetch;
+}
+export declare class HttpVerificationService implements VerificationService {
+    #private;
+    constructor(options: HttpVerificationServiceOptions);
+    getInstallationContext(): Promise<{
+        installation: VerificationInstallationClaims;
+    }>;
+    createChallenge(input: ChallengeCreateRequest): Promise<ChallengeCreateResponse>;
+    cancelChallenges(input: {
+        installationId: string;
+        guildId: string;
+        userId?: string;
+    }): Promise<{
+        cancelled: number;
+    }>;
+    listAttempts(input: DashboardAttemptQuery & {
+        installationId: string;
+        guildId: string;
+    }): Promise<DashboardAttemptPage>;
+    getAttempt(input: {
+        installationId: string;
+        guildId: string;
+        attemptId: string;
+    }): Promise<DashboardSafeAttemptDetail | null>;
+    decideAttempt(input: {
+        installationId: string;
+        guildId: string;
+        attemptId: string;
+        expectedVersion: number;
+        actorUserId: string;
+        decision: "approve" | "deny" | "retry" | "dismiss_relationship";
+        reason: string;
+    }): Promise<DashboardSafeAttemptDetail>;
+    registerResultHandler(handler: (context: VerificationResultContext) => Promise<ActionReceipt>): () => void;
+    acceptResult(envelopeInput: SignedAssessmentEnvelope): Promise<ActionReceipt>;
+    health(): Promise<{
+        status: "healthy" | "degraded" | "unavailable";
+        provider: "available" | "degraded" | "unavailable";
+        allowanceRemaining?: number;
+    }>;
+}
+//# sourceMappingURL=verification-service.d.ts.map