@helyx/bot 0.3.1 → 0.4.1

package/dist/shard-supervisor.js

@@ -0,0 +1,1230 @@
+import { safeError, ServiceRegistry } from "@helyx/core";
+import { checkDatabaseConnection, createDatabasePool, InteractionSessionRepository, RateLimitRepository, runConfiguredModuleMigrations, runCoreMigrations, RuntimeStateRepository, } from "@helyx/database";
+import { createPlatformServices, PostgresPrivacyProviderExecutor, } from "@helyx/platform";
+import { Client, ShardingManager } from "discord.js";
+import { createHash, randomUUID } from "node:crypto";
+import { createRequire } from "node:module";
+import { join } from "node:path";
+import pino, {} from "pino";
+import { synchroniseApplicationCommands, } from "./command-sync.js";
+import { CloudConnectorController, } from "./cloud-connector.js";
+import { createCloudManagementModule } from "./cloud-management-module.js";
+import { BotControlService, createBotControlServer, mapBotControlError, } from "./control-server.js";
+import { createManagementModule } from "./management-module.js";
+import { buildApplicationCommandDefinitions } from "./interactions.js";
+import { resolveModulePackageSpecifiers } from "./module-configuration.js";
+import { loadConfiguredModulePackages } from "./modules.js";
+import { createPermissionsModule } from "./permissions-module.js";
+import { SHARD_CONTROL_MAX_BYTES, SHARD_HEALTH_STALE_MS, assertShardMessageSize, controlOperationGuildId, executeShardControlOperation, gatewayBotInformationSchema, shardCloudRequestSchema, shardHealthSnapshotSchema, workerMessageSchema, } from "./shard-contracts.js";
+import { RedisShardCoordinator, } from "./shard-coordinator.js";
+import { ShardControlRouter } from "./shard-router.js";
+import { createDeveloperOperationsModule } from "./developer-operations-module.js";
+import { SupervisorCloudDiscordDirectory } from "./supervisor-cloud-directory.js";
+const LEASE_RENEW_INTERVAL_MS = 10_000;
+const STARTUP_LEASE_RENEW_INTERVAL_MS = 5_000;
+const LEASE_VALIDATION_TIMEOUT_MS = 1_000;
+const RETENTION_CLEANUP_INTERVAL_MS = 60 * 60 * 1_000;
+const STARTUP_WAIT_MS = 60_000;
+const WORKER_SHUTDOWN_MS = 5_000;
+const MAX_RESTARTS_PER_WINDOW = 5;
+const RESTART_WINDOW_MS = 10 * 60 * 1_000;
+export const STEADY_STATE_LEASE_LOSS_BOUND_MS = LEASE_RENEW_INTERVAL_MS + LEASE_VALIDATION_TIMEOUT_MS + WORKER_SHUTDOWN_MS;
+export async function startShardSupervisor(environment, processEnvironment, workerEntry) {
+    if (environment.HELYX_SHARD_MODE === "off")
+        throw new Error("Shard supervisor requires sharding to be enabled");
+    const shardMode = environment.HELYX_SHARD_MODE;
+    const logger = pino({
+        level: environment.LOG_LEVEL,
+        base: { component: "bot-supervisor" },
+    });
+    let supervisorId = randomUUID();
+    const database = createDatabasePool({
+        connectionString: environment.DATABASE_URL,
+        sslMode: environment.DATABASE_SSL,
+        applicationName: "helyx-bot-supervisor",
+        ...(environment.HELYX_SHARD_SUPERVISOR_DATABASE_POOL_MAX === undefined
+            ? {}
+            : {
+                maxConnections: environment.HELYX_SHARD_SUPERVISOR_DATABASE_POOL_MAX,
+            }),
+    });
+    let privacyDatabase;
+    let controlServer;
+    let coordinator;
+    let leaderLease;
+    let cloudConnector;
+    let cloudStarted = false;
+    let maintainingLeases = false;
+    let rebalancingShards = false;
+    let retentionTimer;
+    let leaseTimer;
+    let startupLeaseTimer;
+    let startupLeaseRefresh;
+    let startupLeaseError;
+    let shuttingDown = false;
+    const localLeases = new Map();
+    const healthByShard = new Map();
+    const pendingControl = new Map();
+    const restartHistory = new Map();
+    const restartCounts = new Map();
+    const workers = new Map();
+    let sessionBudget;
+    let databaseHealthy = false;
+    let generationDescriptor;
+    let deploymentOrder = 1;
+    let shardCount = 1;
+    let privacyService;
+    let manager;
+    let promoteToLeader = () => Promise.resolve(false);
+    try {
+        await checkDatabaseConnection(database);
+        databaseHealthy = true;
+        const moduleSpecifiers = await resolveModulePackageSpecifiers(environment.HELYX_MODULE_PACKAGES, undefined, undefined, environment.HELYX_MODULE_CATALOGUE_PATH);
+        const modulePackages = await loadConfiguredModulePackages(moduleSpecifiers);
+        const featureModules = modulePackages.map(({ definition }) => definition);
+        const gateway = await fetchGatewayBotInformation(environment.DISCORD_TOKEN);
+        shardCount =
+            environment.HELYX_SHARD_MODE === "fixed"
+                ? environment.HELYX_SHARD_COUNT
+                : gateway.shards;
+        if (environment.HELYX_SHARD_SCOPE === "local" &&
+            environment.HELYX_SHARD_WORKERS_PER_REPLICA < shardCount)
+            throw new Error("Local shard scope requires workers-per-replica to cover every shard");
+        const generation = environment.HELYX_SHARD_GENERATION ?? `local-${Date.now().toString(36)}`;
+        sessionBudget = {
+            total: gateway.session_start_limit.total,
+            remaining: gateway.session_start_limit.remaining,
+            resetAt: new Date(Date.now() + gateway.session_start_limit.reset_after).toISOString(),
+            maxConcurrency: gateway.session_start_limit.max_concurrency,
+            observedAt: new Date().toISOString(),
+        };
+        const supervisorDirectory = new SupervisorCloudDiscordDirectory(environment.DISCORD_TOKEN, database, environment.HELYX_OWNER_USER_IDS);
+        cloudConnector = new CloudConnectorController({
+            environment,
+            database,
+            discord: supervisorDirectory,
+            modules: modulePackages,
+            coreVersion: botVersion(),
+            logger,
+        });
+        const commandClient = new Client({ intents: [] });
+        const unavailableResources = {
+            getGuildResources: () => Promise.reject(new Error("Guild resources are worker-owned")),
+            sendToChannel: () => Promise.reject(new Error("Guild resources are worker-owned")),
+            deleteChannelMessage: () => Promise.reject(new Error("Guild resources are worker-owned")),
+        };
+        const permissionsModule = createPermissionsModule(featureModules);
+        const managementModule = createManagementModule(featureModules, unavailableResources);
+        const cloudManagementModule = environment.HELYX_CLOUD_CONNECT_ENABLED
+            ? createCloudManagementModule({
+                connector: cloudConnector,
+                client: commandClient,
+                discord: supervisorDirectory,
+                configuredOwnerUserIds: environment.HELYX_OWNER_USER_IDS,
+            })
+            : undefined;
+        const publicModules = [
+            managementModule,
+            permissionsModule,
+            ...(cloudManagementModule ? [cloudManagementModule] : []),
+            ...featureModules,
+        ];
+        const operationsModule = environment.DISCORD_INTERNAL_GUILD_ID
+            ? createDeveloperOperationsModule({
+                internalGuildId: environment.DISCORD_INTERNAL_GUILD_ID,
+                approvedRoleIds: environment.APPROVED_ROLES,
+                botVersion: botVersion(),
+                ...(processEnvironment.RAILWAY_GIT_COMMIT_SHA
+                    ? {
+                        deploymentReference: processEnvironment.RAILWAY_GIT_COMMIT_SHA.slice(0, 12),
+                    }
+                    : {}),
+                coreFunctions: [
+                    managementModule,
+                    permissionsModule,
+                    ...(cloudManagementModule ? [cloudManagementModule] : []),
+                ].map(({ manifest }) => manifest),
+                modules: featureModules.map(({ manifest }) => manifest),
+                processLabel: "shard supervisor registration",
+                health: {
+                    getHealth: async () => boundedOperationalHealth(await buildHealth()),
+                },
+            })
+            : undefined;
+        const modules = [
+            ...publicModules,
+            ...(operationsModule ? [operationsModule] : []),
+        ];
+        const initialiseLeader = async () => {
+            await runLeaderStartup(database, environment, modulePackages, modules, logger);
+            assertStartupLeaseHealthy();
+            privacyDatabase = await createPrivacyDatabase(environment, database);
+            assertStartupLeaseHealthy();
+            const services = new Map(createPlatformServices(database, publicModules));
+            privacyService = new BotControlService(database, publicModules, unavailableResources, new ServiceRegistry(services), privacyDatabase
+                ? new PostgresPrivacyProviderExecutor({
+                    readPool: database,
+                    executorPool: privacyDatabase,
+                    modules: featureModules,
+                })
+                : undefined);
+            cloudConnector?.setControlHandler(new ShardControlRouter({
+                generation,
+                shardCount,
+                localLeases,
+                dispatchLocal,
+                privacy: privacyService,
+                localLeaderLease: leaderLease,
+                ...(coordinator ? { coordinator } : {}),
+            }));
+            await startLeaderSingletons();
+            assertStartupLeaseHealthy();
+        };
+        generationDescriptor = {
+            version: 1,
+            applicationId: environment.DISCORD_CLIENT_ID,
+            environment: environment.NODE_ENV,
+            generation,
+            shardCount,
+            catalogueHash: shardRuntimeCompositionHash(modules, environment.DISCORD_INTERNAL_GUILD_ID),
+        };
+        deploymentOrder = await nextShardDeploymentOrder(database);
+        if (environment.HELYX_SHARD_SCOPE === "coordinated") {
+            coordinator = new RedisShardCoordinator(environment.HELYX_SHARD_REDIS_URL, environment.HELYX_SHARD_REDIS_NAMESPACE, {
+                onError: (error) => logger.error({ event: "shard.redis_error", error: safeError(error) }, "Shard coordination failed"),
+            });
+            supervisorId = coordinator.supervisorId;
+            await coordinator.start((targetShardId, request) => dispatchLocal(targetShardId, request), (request) => executeLeaderControl(request), (request) => executeLeaderCloud(request));
+            deploymentOrder = await coordinator.assertGeneration(generationDescriptor, deploymentOrder);
+            leaderLease = await coordinator.acquireLeader(generationDescriptor, deploymentOrder);
+            startStartupLeaseMaintenance();
+        }
+        else {
+            leaderLease = {
+                generation,
+                supervisorId,
+                fencingToken: 1,
+            };
+        }
+        promoteToLeader = async () => {
+            if (!coordinator || leaderLease || shuttingDown)
+                return Boolean(leaderLease);
+            const candidate = await acquireFencedLeader(coordinator, generationDescriptor, deploymentOrder).catch(() => undefined);
+            if (!candidate)
+                return false;
+            if (shuttingDown) {
+                await coordinator.releaseLeader(candidate).catch(() => false);
+                return false;
+            }
+            leaderLease = candidate;
+            startStartupLeaseMaintenance();
+            try {
+                await initialiseLeader();
+                await stopStartupLeaseMaintenance();
+                await withTimeout(renewStartupOwnership(coordinator, generationDescriptor, deploymentOrder, leaderLease), LEASE_VALIDATION_TIMEOUT_MS);
+                logger.info({
+                    event: "shard.leader_promoted",
+                    fencingToken: candidate.fencingToken,
+                }, "Shard supervisor acquired the fenced deployment leader lease");
+                return true;
+            }
+            catch (error) {
+                logger.error({ event: "shard.leader_promotion_failed", error: safeError(error) }, "Shard supervisor could not promote to deployment leader");
+                await shutdown("leader-promotion-failed");
+                return false;
+            }
+        };
+        if (leaderLease) {
+            await initialiseLeader();
+        }
+        else {
+            await waitForLeaderStartup(database, generation);
+            assertStartupLeaseHealthy();
+        }
+        const router = new ShardControlRouter({
+            generation,
+            shardCount,
+            localLeases,
+            dispatchLocal,
+            ...(privacyService ? { privacy: privacyService } : {}),
+            ...(leaderLease ? { localLeaderLease: leaderLease } : {}),
+            ...(coordinator ? { coordinator } : {}),
+        });
+        if (!leaderLease)
+            cloudConnector.setControlHandler(router);
+        const claimedIds = await claimInitialShards();
+        manager = new ShardingManager(workerEntry, {
+            totalShards: shardCount,
+            shardList: claimedIds.length > 0 ? claimedIds : [0],
+            mode: "process",
+            respawn: false,
+            token: environment.DISCORD_TOKEN,
+            execArgv: process.execArgv,
+        });
+        await reserveInitialSessionStarts(claimedIds.length, gateway);
+        await spawnInIdentifyBuckets(claimedIds, gateway.session_start_limit.max_concurrency);
+        assertStartupLeaseHealthy();
+        if (environment.HELYX_BOT_CONTROL_SECRET) {
+            controlServer = await createBotControlServer({
+                service: router,
+                secret: environment.HELYX_BOT_CONTROL_SECRET,
+                logger: { level: environment.LOG_LEVEL },
+                shardHealth: buildHealth,
+            });
+            await controlServer.listen({
+                host: "::",
+                port: environment.HELYX_BOT_CONTROL_PORT,
+            });
+        }
+        await stopStartupLeaseMaintenance();
+        if (coordinator)
+            await withTimeout(renewStartupOwnership(coordinator, generationDescriptor, deploymentOrder, leaderLease), LEASE_VALIDATION_TIMEOUT_MS);
+        leaseTimer = setInterval(() => void maintainLeases(), LEASE_RENEW_INTERVAL_MS);
+        leaseTimer.unref();
+        process.once("SIGINT", () => void shutdown("SIGINT"));
+        process.once("SIGTERM", () => void shutdown("SIGTERM"));
+        logger.info({
+            event: "shard.supervisor_ready",
+            mode: environment.HELYX_SHARD_MODE,
+            scope: environment.HELYX_SHARD_SCOPE,
+            generation,
+            supervisorId,
+            shardCount,
+            localShardCount: claimedIds.length,
+        }, "Shard supervisor started");
+        return { shutdown };
+    }
+    catch (error) {
+        await shutdown("startup-failed");
+        throw error;
+    }
+    async function claimInitialShards() {
+        const maximum = environment.HELYX_SHARD_SCOPE === "local"
+            ? shardCount
+            : environment.HELYX_SHARD_WORKERS_PER_REPLICA;
+        const claimed = [];
+        for (let shardId = 0; shardId < shardCount && claimed.length < maximum; shardId++) {
+            const lease = coordinator && generationDescriptor
+                ? await coordinator.claimShard(generationDescriptor, shardId, deploymentOrder)
+                : {
+                    generation: generationDescriptor.generation,
+                    deploymentOrder,
+                    supervisorId,
+                    shardId,
+                    fencingToken: 1,
+                };
+            if (!lease)
+                continue;
+            localLeases.set(shardId, lease);
+            claimed.push(shardId);
+        }
+        return claimed;
+    }
+    async function reserveInitialSessionStarts(count, gateway) {
+        if (count === 0)
+            return;
+        if (coordinator) {
+            const reserved = await coordinator.reserveSessionStarts(count, gateway.session_start_limit.remaining, gateway.session_start_limit.reset_after);
+            if (!reserved)
+                throw new Error("Discord session-start budget cannot support this replica");
+        }
+        else if (gateway.session_start_limit.remaining < count) {
+            throw new Error("Discord session-start budget cannot support planned shards");
+        }
+    }
+    async function spawnInIdentifyBuckets(shardIds, maxConcurrency) {
+        for (let offset = 0; offset < shardIds.length; offset += maxConcurrency) {
+            if (shuttingDown)
+                return;
+            const batch = shardIds.slice(offset, offset + maxConcurrency);
+            await Promise.all(batch.map((shardId) => spawnWorker(shardId)));
+            if (offset + maxConcurrency < shardIds.length)
+                await wait(5_500);
+        }
+    }
+    async function spawnWorker(shardId) {
+        if (shuttingDown)
+            return;
+        const lease = localLeases.get(shardId);
+        if (!lease || !manager)
+            throw new Error("Shard lease is unavailable");
+        if (coordinator) {
+            const allowed = await coordinator.acquireIdentifyPermit(shardId, sessionBudget?.maxConcurrency ?? 1, 15_000);
+            if (!allowed)
+                throw new Error("Discord identify bucket is unavailable");
+        }
+        if (shuttingDown)
+            return;
+        let shard = workers.get(shardId);
+        if (!shard) {
+            shard = manager.createShard(shardId);
+            workers.set(shardId, shard);
+            shard.on("message", (raw) => void handleWorkerMessage(shardId, raw));
+            shard.on("death", () => handleWorkerDeath(shardId));
+            shard.on("error", (error) => logger.error({ event: "shard.worker_error", shardId, error: safeError(error) }, "Shard worker emitted an error"));
+        }
+        const identity = {
+            version: 1,
+            mode: shardMode,
+            scope: environment.HELYX_SHARD_SCOPE,
+            generation: generationDescriptor.generation,
+            deploymentOrder: lease.deploymentOrder,
+            supervisorId: lease.supervisorId,
+            shardId,
+            shardCount,
+            fencingToken: lease.fencingToken,
+        };
+        shard.env = buildShardWorkerEnvironment(processEnvironment, identity, restartCounts.get(shardId) ?? 0);
+        healthByShard.set(shardId, startingHealth(shardId, restartCounts.get(shardId) ?? 0));
+        const launchHeartbeatTimer = coordinator
+            ? setInterval(() => void sendLeaseHeartbeat(shardId, lease), STARTUP_LEASE_RENEW_INTERVAL_MS)
+            : undefined;
+        launchHeartbeatTimer?.unref();
+        try {
+            await spawnWithShutdownGuard(() => shuttingDown, () => shard.spawn(30_000), () => shard.kill());
+        }
+        finally {
+            if (launchHeartbeatTimer)
+                clearInterval(launchHeartbeatTimer);
+        }
+        await sendLeaseHeartbeat(shardId, lease);
+    }
+    async function sendLeaseHeartbeat(shardId, lease) {
+        if (shuttingDown || !coordinator || !workers.get(shardId)?.process)
+            return;
+        const identity = {
+            version: 1,
+            mode: shardMode,
+            scope: environment.HELYX_SHARD_SCOPE,
+            generation: lease.generation,
+            deploymentOrder: lease.deploymentOrder,
+            supervisorId: lease.supervisorId,
+            shardId,
+            shardCount,
+            fencingToken: lease.fencingToken,
+        };
+        const heartbeat = {
+            version: 1,
+            type: "helyx.lease.heartbeat",
+            identity,
+        };
+        assertShardMessageSize(heartbeat);
+        await workers
+            .get(shardId)
+            ?.send(heartbeat)
+            .catch(() => undefined);
+    }
+    async function handleWorkerMessage(shardId, raw) {
+        if (serializedBytes(raw) > SHARD_CONTROL_MAX_BYTES)
+            return;
+        const message = workerMessageSchema.safeParse(raw);
+        if (!message.success)
+            return;
+        if (message.data.type === "helyx.health.update") {
+            const lease = localLeases.get(shardId);
+            if (lease &&
+                message.data.identity.shardId === shardId &&
+                message.data.identity.generation === lease.generation &&
+                message.data.identity.fencingToken === lease.fencingToken) {
+                healthByShard.set(shardId, message.data.health);
+                await coordinator
+                    ?.setHealth(message.data.identity, message.data.health)
+                    .catch(() => undefined);
+            }
+            return;
+        }
+        if (message.data.type === "helyx.control.response") {
+            const pending = pendingControl.get(message.data.requestId);
+            if (!pending || pending.shardId !== shardId)
+                return;
+            pendingControl.delete(message.data.requestId);
+            clearTimeout(pending.timer);
+            pending.resolve(message.data);
+            return;
+        }
+        if (message.data.type === "helyx.cloud.request") {
+            const response = leaderLease
+                ? await executeLeaderCloud(message.data)
+                : await routeCloudToLeader(message.data);
+            await workers.get(shardId)?.send(response);
+            return;
+        }
+        if (message.data.type === "helyx.operations.health.request") {
+            const response = {
+                version: 1,
+                type: "helyx.operations.health.response",
+                requestId: message.data.requestId,
+                health: boundedOperationalHealth(await buildHealth()),
+            };
+            assertShardMessageSize(response);
+            await workers.get(shardId)?.send(response);
+        }
+    }
+    function dispatchLocal(shardId, request) {
+        const shard = workers.get(shardId);
+        if (!shard?.process)
+            return Promise.resolve(unavailableControl(request.requestId));
+        return new Promise((resolve) => {
+            const timer = setTimeout(() => {
+                pendingControl.delete(request.requestId);
+                resolve(unavailableControl(request.requestId, 504));
+            }, Math.max(1, new Date(request.deadlineAt).getTime() - Date.now()));
+            timer.unref();
+            pendingControl.set(request.requestId, { shardId, resolve, timer });
+            void shard.send(request).catch(() => {
+                clearTimeout(timer);
+                pendingControl.delete(request.requestId);
+                resolve(unavailableControl(request.requestId));
+            });
+        });
+    }
+    async function executeLeaderControl(request) {
+        if (!leaderLease || !privacyService)
+            return unavailableControl(request.requestId);
+        if (request.generation !== leaderLease.generation ||
+            request.fencingToken !== leaderLease.fencingToken ||
+            controlOperationGuildId(request.operation))
+            return unavailableControl(request.requestId);
+        try {
+            return {
+                version: 1,
+                type: "helyx.control.response",
+                requestId: request.requestId,
+                ok: true,
+                result: await executeShardControlOperation(privacyService, request.operation),
+            };
+        }
+        catch (error) {
+            const mapped = mapBotControlError(error);
+            return {
+                version: 1,
+                type: "helyx.control.response",
+                requestId: request.requestId,
+                ok: false,
+                status: mapped?.status ?? 500,
+                message: mapped?.message ?? "The shard leader rejected the request",
+            };
+        }
+    }
+    async function executeLeaderCloud(request) {
+        const parsed = shardCloudRequestSchema.parse(request);
+        if (!leaderLease ||
+            !(await currentLeaderOwnership()) ||
+            !cloudConnector ||
+            !cloudStarted ||
+            cloudRequestExpired(parsed))
+            return failedCloud(parsed.requestId, "Cloud leader is unavailable");
+        try {
+            let result;
+            switch (parsed.operation.kind) {
+                case "cloud.status":
+                    result = cloudConnector.status();
+                    break;
+                case "cloud.consent.get":
+                    result = await cloudConnector.getSupportConsent();
+                    break;
+                case "cloud.consent.update": {
+                    if (cloudRequestExpired(parsed))
+                        return failedCloud(parsed.requestId, "Cloud request expired");
+                    result = await cloudConnector.updateSupportConsent({
+                        supportInspectionEnabled: parsed.operation.supportInspectionEnabled,
+                        supportDataClasses: parsed.operation.supportDataClasses,
+                        diagnosticsEnabled: parsed.operation.diagnosticsEnabled,
+                    });
+                    break;
+                }
+                case "cloud.enrol":
+                    if (cloudRequestExpired(parsed))
+                        return failedCloud(parsed.requestId, "Cloud request expired");
+                    result = await cloudConnector.enrol(parsed.operation.token, parsed.operation.source);
+                    break;
+                case "cloud.disconnect":
+                    if (cloudRequestExpired(parsed))
+                        return failedCloud(parsed.requestId, "Cloud request expired");
+                    await cloudConnector.disconnect();
+                    result = null;
+                    break;
+                case "cloud.rotate":
+                    if (cloudRequestExpired(parsed))
+                        return failedCloud(parsed.requestId, "Cloud request expired");
+                    result = await cloudConnector.rotate();
+                    break;
+            }
+            const response = {
+                version: 1,
+                type: "helyx.cloud.response",
+                requestId: parsed.requestId,
+                ok: true,
+                result,
+            };
+            assertShardMessageSize(response);
+            return response;
+        }
+        catch {
+            return failedCloud(parsed.requestId, "Cloud operation failed safely");
+        }
+    }
+    async function routeCloudToLeader(request) {
+        const lease = await coordinator?.getLeaderLease().catch(() => undefined);
+        if (!coordinator || !lease)
+            return failedCloud(request.requestId, "Cloud leader is unavailable");
+        return coordinator
+            .routeLeaderCloud(lease, request)
+            .catch(() => failedCloud(request.requestId, "Cloud leader is unavailable"));
+    }
+    function handleWorkerDeath(shardId) {
+        healthByShard.set(shardId, {
+            ...startingHealth(shardId, restartCounts.get(shardId) ?? 0),
+            status: "failed",
+        });
+        for (const [requestId, pending] of pendingControl) {
+            if (pending.shardId !== shardId)
+                continue;
+            pendingControl.delete(requestId);
+            clearTimeout(pending.timer);
+            pending.resolve(unavailableControl(requestId));
+        }
+        if (shuttingDown || !localLeases.has(shardId))
+            return;
+        const now = Date.now();
+        const recent = (restartHistory.get(shardId) ?? []).filter((time) => now - time < RESTART_WINDOW_MS);
+        if (recent.length >= MAX_RESTARTS_PER_WINDOW) {
+            logger.error({ event: "shard.circuit_open", shardId, restartCount: recent.length }, "Shard worker restart circuit opened");
+            return;
+        }
+        recent.push(now);
+        restartHistory.set(shardId, recent);
+        restartCounts.set(shardId, (restartCounts.get(shardId) ?? 0) + 1);
+        const delay = boundedRespawnDelay(recent.length);
+        setTimeout(() => void respawnWorker(shardId), delay).unref();
+    }
+    async function respawnWorker(shardId) {
+        if (shuttingDown || !localLeases.has(shardId))
+            return;
+        try {
+            const gateway = await fetchGatewayBotInformation(environment.DISCORD_TOKEN);
+            await reserveInitialSessionStarts(1, gateway);
+            await spawnWorker(shardId);
+        }
+        catch (error) {
+            logger.error({ event: "shard.respawn_failed", shardId, error: safeError(error) }, "Shard worker respawn failed");
+            handleWorkerDeath(shardId);
+        }
+    }
+    async function maintainLeases() {
+        if (shuttingDown || maintainingLeases)
+            return;
+        maintainingLeases = true;
+        try {
+            if (!coordinator)
+                return;
+            const renewal = await renewSteadyStateOwnership(coordinator, generationDescriptor, deploymentOrder, localLeases, leaderLease).catch(() => undefined);
+            if (!renewal?.generationRenewed) {
+                logger.error({ event: "shard.generation_lease_lost" }, "Shard generation lease was lost; stopping this supervisor safely");
+                await shutdown("generation-lease-lost");
+                return;
+            }
+            for (const shardId of renewal.lostShardIds) {
+                localLeases.delete(shardId);
+                workers.get(shardId)?.kill();
+                healthByShard.set(shardId, {
+                    ...startingHealth(shardId, restartCounts.get(shardId) ?? 0),
+                    status: "failed",
+                });
+            }
+            if (leaderLease) {
+                if (!renewal.leaderRenewed) {
+                    logger.error({ event: "shard.leader_lease_lost" }, "Shard leader lease was lost; stopping this supervisor safely");
+                    await shutdown("leader-lease-lost");
+                    return;
+                }
+            }
+            else {
+                if (!renewal.leaderAvailable) {
+                    const promoted = await promoteToLeader();
+                    if (!promoted && shuttingDown)
+                        return;
+                }
+            }
+            await Promise.all([...localLeases].map(([shardId, lease]) => sendLeaseHeartbeat(shardId, lease)));
+        }
+        finally {
+            maintainingLeases = false;
+        }
+        if (!shuttingDown)
+            await rebalanceShards().catch(async (error) => {
+                logger.error({ event: "shard.rebalance_failed", error: safeError(error) }, "Shard rebalance failed; stopping this supervisor safely");
+                await shutdown("rebalance-failed");
+            });
+    }
+    async function rebalanceShards() {
+        if (!coordinator || rebalancingShards)
+            return;
+        const capacity = environment.HELYX_SHARD_WORKERS_PER_REPLICA - localLeases.size;
+        if (capacity < 1)
+            return;
+        rebalancingShards = true;
+        try {
+            const occupied = new Set((await withTimeout(coordinator.getShardLeases(shardCount), LEASE_VALIDATION_TIMEOUT_MS).catch(() => [])).map(({ shardId }) => shardId));
+            const candidates = Array.from({ length: shardCount }, (_, shardId) => shardId)
+                .filter((shardId) => !localLeases.has(shardId) && !occupied.has(shardId))
+                .slice(0, capacity);
+            const results = await Promise.all(candidates.map(async (shardId) => {
+                const lease = await withTimeout(coordinator.claimShard(generationDescriptor, shardId, deploymentOrder), LEASE_VALIDATION_TIMEOUT_MS).catch(() => undefined);
+                return lease ? { shardId, lease } : undefined;
+            }));
+            const claimed = [];
+            for (const result of results) {
+                if (!result || shuttingDown)
+                    continue;
+                localLeases.set(result.shardId, result.lease);
+                claimed.push(result.shardId);
+            }
+            if (claimed.length < 1 || shuttingDown)
+                return;
+            const gateway = await fetchGatewayBotInformation(environment.DISCORD_TOKEN);
+            if (shuttingDown)
+                return;
+            await reserveInitialSessionStarts(claimed.length, gateway);
+            if (shuttingDown)
+                return;
+            await spawnInIdentifyBuckets(claimed, gateway.session_start_limit.max_concurrency);
+        }
+        finally {
+            rebalancingShards = false;
+        }
+    }
+    async function startLeaderSingletons() {
+        await cleanupExpiredCoreRecords(database, logger);
+        retentionTimer = setInterval(() => void cleanupExpiredCoreRecords(database, logger), RETENTION_CLEANUP_INTERVAL_MS);
+        retentionTimer.unref();
+        await cloudConnector?.start();
+        cloudStarted = true;
+    }
+    function startStartupLeaseMaintenance() {
+        if (!coordinator || startupLeaseTimer)
+            return;
+        const refresh = () => {
+            if (startupLeaseRefresh || startupLeaseError)
+                return;
+            startupLeaseRefresh = withTimeout(renewStartupOwnership(coordinator, generationDescriptor, deploymentOrder, leaderLease), LEASE_VALIDATION_TIMEOUT_MS)
+                .catch((error) => {
+                startupLeaseError =
+                    error instanceof Error
+                        ? error
+                        : new Error("Shard startup ownership was lost");
+            })
+                .finally(() => {
+                startupLeaseRefresh = undefined;
+            });
+        };
+        startupLeaseTimer = setInterval(refresh, STARTUP_LEASE_RENEW_INTERVAL_MS);
+        startupLeaseTimer.unref();
+    }
+    function assertStartupLeaseHealthy() {
+        if (startupLeaseError)
+            throw startupLeaseError;
+    }
+    async function stopStartupLeaseMaintenance() {
+        if (startupLeaseTimer)
+            clearInterval(startupLeaseTimer);
+        startupLeaseTimer = undefined;
+        await startupLeaseRefresh;
+        assertStartupLeaseHealthy();
+    }
+    async function currentLeaderOwnership() {
+        if (!leaderLease)
+            return false;
+        if (!coordinator)
+            return true;
+        try {
+            const current = await withTimeout(coordinator.getLeaderLease(), LEASE_VALIDATION_TIMEOUT_MS);
+            return (current?.generation === leaderLease.generation &&
+                current.supervisorId === leaderLease.supervisorId &&
+                current.fencingToken === leaderLease.fencingToken);
+        }
+        catch {
+            return false;
+        }
+    }
+    async function buildHealth() {
+        const now = Date.now();
+        let redisReady = true;
+        if (coordinator)
+            await withTimeout(coordinator.checkReady(), LEASE_VALIDATION_TIMEOUT_MS).catch(() => {
+                redisReady = false;
+            });
+        await checkDatabaseConnection(database)
+            .then(() => {
+            databaseHealthy = true;
+        })
+            .catch(() => {
+            databaseHealthy = false;
+        });
+        let coordinatedHealth = [];
+        let liveLeases = [];
+        if (coordinator) {
+            try {
+                [coordinatedHealth, liveLeases] = await withTimeout(Promise.all([
+                    coordinator.getHealth(shardCount),
+                    coordinator.getShardLeases(shardCount),
+                ]), LEASE_VALIDATION_TIMEOUT_MS);
+            }
+            catch {
+                redisReady = false;
+            }
+        }
+        const coordinationRedis = coordinator ? (redisReady ? "healthy" : "unavailable") : "not_required";
+        const healthMap = selectFencedShardHealth({
+            generation: generationDescriptor.generation,
+            shardCount,
+            coordinatedHealth,
+            liveLeases,
+            localHealth: healthByShard,
+            localLeases,
+            coordinated: Boolean(coordinator),
+        });
+        const shards = Array.from({ length: shardCount }, (_, shardId) => {
+            const value = healthMap.get(shardId);
+            if (!value)
+                return { ...startingHealth(shardId, 0), status: "failed" };
+            if (now - new Date(value.observedAt).getTime() > SHARD_HEALTH_STALE_MS)
+                return { ...value, status: "failed" };
+            return value;
+        });
+        const readyShardCount = shards.filter((shard) => shard.status === "ready").length;
+        const stale = [...healthMap.values()].some((value) => now - new Date(value.observedAt).getTime() > SHARD_HEALTH_STALE_MS);
+        return shardHealthSnapshotSchema.parse({
+            version: 1,
+            mode: shardMode,
+            scope: environment.HELYX_SHARD_SCOPE,
+            status: stale
+                ? "stale"
+                : readyShardCount === shardCount &&
+                    coordinationRedis !== "unavailable" &&
+                    databaseHealthy
+                    ? "healthy"
+                    : readyShardCount === 0
+                        ? "unavailable"
+                        : readyShardCount < shardCount
+                            ? "starting"
+                            : "degraded",
+            observedAt: new Date().toISOString(),
+            generation: generationDescriptor.generation,
+            supervisorId,
+            configuredShardCount: shardCount,
+            readyShardCount,
+            coordination: {
+                redis: coordinationRedis,
+                database: databaseHealthy ? "healthy" : "unavailable",
+            },
+            ...(sessionBudget ? { sessionStarts: sessionBudget } : {}),
+            shards,
+        });
+    }
+    async function shutdown(reason = "programmatic") {
+        if (shuttingDown)
+            return;
+        shuttingDown = true;
+        if (leaseTimer)
+            clearInterval(leaseTimer);
+        if (startupLeaseTimer)
+            clearInterval(startupLeaseTimer);
+        if (retentionTimer)
+            clearInterval(retentionTimer);
+        await controlServer?.close().catch(() => undefined);
+        await Promise.allSettled([...workers.values()].map(async (shard) => {
+            if (!shard.process)
+                return;
+            await shard
+                .send({ version: 1, type: "helyx.shutdown", reason })
+                .catch(() => undefined);
+            await waitForDeath(shard, WORKER_SHUTDOWN_MS);
+            if (shard.process)
+                shard.kill();
+        }));
+        await cloudConnector?.stop().catch(() => undefined);
+        if (coordinator) {
+            const activeCoordinator = coordinator;
+            await Promise.allSettled([...localLeases.values()].map((lease) => activeCoordinator.releaseShard(lease)));
+            if (leaderLease)
+                await activeCoordinator.releaseLeader(leaderLease).catch(() => false);
+            await activeCoordinator.stop();
+        }
+        await privacyDatabase?.end().catch(() => undefined);
+        await database.end().catch(() => undefined);
+        logger.info({ event: "shard.supervisor_stopped", reason }, "Shard supervisor stopped");
+    }
+}
+async function runLeaderStartup(database, environment, modulePackages, modules, logger) {
+    await runCoreMigrations(database);
+    await runConfiguredModuleMigrations(database, modulePackages.map(({ definition, packageRoot }) => ({
+        moduleId: definition.manifest.id,
+        moduleVersion: definition.manifest.version,
+        migrationsDirectory: join(packageRoot, "migrations"),
+    })));
+    const repository = new RuntimeStateRepository(database);
+    await synchroniseApplicationCommands({
+        applicationId: environment.DISCORD_CLIENT_ID,
+        token: environment.DISCORD_TOKEN,
+        modules,
+        ...(environment.DISCORD_INTERNAL_GUILD_ID
+            ? { internalGuildId: environment.DISCORD_INTERNAL_GUILD_ID }
+            : {}),
+        state: new RuntimeCommandHashStore(repository),
+    });
+    await setRuntimeState(repository, `shards:${environment.HELYX_SHARD_GENERATION ?? "local"}:startup`, {
+        ready: true,
+        completedAt: new Date().toISOString(),
+    });
+    logger.info({ event: "shard.singleton_startup_complete" }, "Shard singleton startup complete");
+}
+class RuntimeCommandHashStore {
+    repository;
+    constructor(repository) {
+        this.repository = repository;
+    }
+    get(key) {
+        return this.repository.get(key);
+    }
+    async set(key, value, expectedVersion = null) {
+        const updated = await this.repository.compareAndSet({
+            key,
+            value: value,
+            expectedVersion,
+        });
+        if (!updated)
+            throw new Error("Runtime state changed concurrently");
+        return updated;
+    }
+}
+async function setRuntimeState(repository, key, value) {
+    const current = await repository.get(key);
+    const updated = await repository.compareAndSet({
+        key,
+        value,
+        expectedVersion: current?.version ?? null,
+    });
+    if (!updated)
+        throw new Error("Runtime state changed concurrently");
+}
+async function waitForLeaderStartup(database, generation) {
+    const deadline = Date.now() + STARTUP_WAIT_MS;
+    while (Date.now() < deadline) {
+        try {
+            const state = await new RuntimeStateRepository(database).get(`shards:${generation}:startup`);
+            if (typeof state?.value === "object" &&
+                state.value !== null &&
+                !Array.isArray(state.value) &&
+                state.value.ready === true)
+                return;
+        }
+        catch {
+            // The deployment leader may still be applying the additive migration.
+        }
+        await wait(250);
+    }
+    throw new Error("Timed out waiting for shard deployment leader startup");
+}
+async function createPrivacyDatabase(environment, runtimeDatabase) {
+    if (!environment.HELYX_PRIVACY_ENABLED ||
+        !environment.HELYX_PRIVACY_EXECUTOR_DATABASE_URL ||
+        !environment.HELYX_PRIVACY_EXECUTOR_DATABASE_SSL)
+        return undefined;
+    const privacy = createDatabasePool({
+        connectionString: environment.HELYX_PRIVACY_EXECUTOR_DATABASE_URL,
+        sslMode: environment.HELYX_PRIVACY_EXECUTOR_DATABASE_SSL,
+        applicationName: "helyx-bot-shard-privacy-leader",
+        maxConnections: 2,
+    });
+    await checkDatabaseConnection(privacy);
+    const [runtimeIdentity, privacyIdentity] = await Promise.all([
+        runtimeDatabase.query("SELECT current_user::text AS current_user"),
+        privacy.query("SELECT current_user::text AS current_user"),
+    ]);
+    if (runtimeIdentity.rows[0]?.current_user ===
+        privacyIdentity.rows[0]?.current_user) {
+        await privacy.end();
+        throw new Error("Privacy executor must use a distinct least-privilege database role");
+    }
+    return privacy;
+}
+async function fetchGatewayBotInformation(token) {
+    const response = await fetch("https://discord.com/api/v10/gateway/bot", {
+        headers: { Authorization: `Bot ${token}` },
+        signal: AbortSignal.timeout(10_000),
+    });
+    if (!response.ok)
+        throw new Error("Discord gateway planning request failed");
+    return gatewayBotInformationSchema.parse(await response.json());
+}
+export function buildShardWorkerEnvironment(source, identity, restartCount) {
+    const allowed = new Set([
+        "DATABASE_URL",
+        "DATABASE_SSL",
+        "DISCORD_TOKEN",
+        "DISCORD_CLIENT_ID",
+        "DISCORD_INTERNAL_GUILD_ID",
+        "HELYX_MODULE_PACKAGES",
+        "HELYX_MODULE_CATALOGUE_PATH",
+        "HELYX_CLOUD_CONNECT_ENABLED",
+        "HELYX_OWNER_USER_IDS",
+        "APPROVED_ROLES",
+        "RAILWAY_GIT_COMMIT_SHA",
+        "HELYX_SHARD_MODE",
+        "HELYX_SHARD_COUNT",
+        "HELYX_SHARD_SCOPE",
+        "HELYX_SHARD_WORKERS_PER_REPLICA",
+        "HELYX_SHARD_REDIS_NAMESPACE",
+        "HELYX_SHARD_GENERATION",
+        "HELYX_SHARD_WORKER_DATABASE_POOL_MAX",
+        "HELYX_SHARD_SUPERVISOR_DATABASE_POOL_MAX",
+        "HELYX_SHARD_DATABASE_CONNECTION_BUDGET",
+        "LOG_LEVEL",
+        "NODE_ENV",
+        "PATH",
+        "SystemRoot",
+        "TEMP",
+        "TMP",
+    ]);
+    const environment = Object.fromEntries(Object.entries(source).filter(([key, value]) => allowed.has(key) && value !== undefined));
+    return {
+        ...environment,
+        SHARDING_MANAGER: "true",
+        SHARDING_MANAGER_MODE: "process",
+        SHARDS: String(identity.shardId),
+        SHARD_COUNT: String(identity.shardCount),
+        HELYX_SHARD_WORKER_CONTRACT: JSON.stringify(identity),
+        HELYX_SHARD_RESTART_COUNT: String(restartCount),
+    };
+}
+export function shardRuntimeCompositionHash(modules, internalGuildId) {
+    const commands = buildApplicationCommandDefinitions(modules, internalGuildId);
+    return createHash("sha256")
+        .update(JSON.stringify({
+        botVersion: botVersion(),
+        modules: modules
+            .map(({ manifest }) => ({
+            id: manifest.id,
+            version: manifest.version,
+        }))
+            .sort((left, right) => left.id.localeCompare(right.id)),
+        commands: {
+            global: commands.global.sort((left, right) => left.name.localeCompare(right.name)),
+            guild: commands.guild.sort((left, right) => left.name.localeCompare(right.name)),
+        },
+    }))
+        .digest("hex");
+}
+function startingHealth(shardId, restartCount) {
+    return {
+        shardId,
+        status: "starting",
+        guildCount: 0,
+        gatewayPingMs: -1,
+        uptimeSeconds: 0,
+        restartCount,
+        eventLoopDelayMs: 0,
+        memoryRssBytes: 0,
+        queueDepth: 0,
+        observedAt: new Date().toISOString(),
+    };
+}
+function unavailableControl(requestId, status = 503) {
+    return {
+        version: 1,
+        type: "helyx.control.response",
+        requestId,
+        ok: false,
+        status,
+        message: status === 504
+            ? "The owning shard did not respond in time"
+            : "The owning shard is temporarily unavailable",
+    };
+}
+function failedCloud(requestId, message) {
+    return {
+        version: 1,
+        type: "helyx.cloud.response",
+        requestId,
+        ok: false,
+        message,
+    };
+}
+async function cleanupExpiredCoreRecords(database, logger) {
+    try {
+        const [sessions, rateLimits] = await Promise.all([
+            new InteractionSessionRepository(database).deleteExpired(),
+            new RateLimitRepository(database).deleteExpired(),
+        ]);
+        if (sessions > 0 || rateLimits > 0)
+            logger.info({
+                event: "database.retention_cleaned",
+                interactionSessions: sessions,
+                rateLimitWindows: rateLimits,
+            }, "Expired core records removed");
+    }
+    catch (error) {
+        logger.warn({ event: "database.retention_cleanup_failed", error: safeError(error) }, "Expired core record cleanup failed");
+    }
+}
+function serializedBytes(value) {
+    try {
+        return Buffer.byteLength(JSON.stringify(value), "utf8");
+    }
+    catch {
+        return SHARD_CONTROL_MAX_BYTES + 1;
+    }
+}
+function boundedOperationalHealth(health) {
+    return shardHealthSnapshotSchema.parse({
+        ...health,
+        shards: health.shards.slice(0, 25),
+    });
+}
+function wait(milliseconds) {
+    return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+async function withTimeout(operation, timeoutMs) {
+    let timer;
+    try {
+        return await Promise.race([
+            operation,
+            new Promise((_resolve, reject) => {
+                timer = setTimeout(() => reject(new Error("Shard ownership validation timed out")), timeoutMs);
+                timer.unref();
+            }),
+        ]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+async function waitForDeath(shard, timeoutMs) {
+    if (!shard.process)
+        return;
+    await Promise.race([
+        new Promise((resolve) => shard.once("death", () => resolve())),
+        wait(timeoutMs),
+    ]);
+}
+export function boundedRespawnDelay(attempt) {
+    return Math.min(30_000, 1_000 * 2 ** Math.max(0, attempt - 1));
+}
+export function selectFencedShardHealth(input) {
+    const liveByShard = new Map(input.liveLeases.map((lease) => [lease.shardId, lease]));
+    const selected = new Map();
+    for (const record of input.coordinatedHealth) {
+        const lease = liveByShard.get(record.identity.shardId);
+        if (record.identity.generation !== input.generation ||
+            record.identity.shardCount !== input.shardCount ||
+            record.health.shardId !== record.identity.shardId ||
+            !lease ||
+            !identityMatchesLease(record.identity, lease))
+            continue;
+        selected.set(record.identity.shardId, record.health);
+    }
+    for (const [shardId, health] of input.localHealth) {
+        if (!input.coordinated) {
+            selected.set(shardId, health);
+            continue;
+        }
+        const localLease = input.localLeases.get(shardId);
+        const liveLease = liveByShard.get(shardId);
+        if (localLease && liveLease && sameShardLease(localLease, liveLease))
+            selected.set(shardId, health);
+    }
+    return selected;
+}
+export async function renewSteadyStateOwnership(coordinator, descriptor, deploymentOrder, shardLeases, leaderLease, timeoutMs = LEASE_VALIDATION_TIMEOUT_MS) {
+    return withTimeout((async () => {
+        const generationRenewed = await coordinator.refreshGeneration(descriptor, deploymentOrder);
+        if (!generationRenewed)
+            return {
+                generationRenewed: false,
+                lostShardIds: [...shardLeases.keys()],
+                leaderRenewed: false,
+                leaderAvailable: false,
+            };
+        const shardRenewals = await Promise.all([...shardLeases].map(async ([shardId, lease]) => ({
+            shardId,
+            renewed: await coordinator.renewShard(lease),
+        })));
+        const currentLeader = leaderLease
+            ? undefined
+            : await coordinator.getLeaderLease();
+        const leaderRenewed = leaderLease
+            ? await coordinator.renewLeader(leaderLease)
+            : false;
+        return {
+            generationRenewed: true,
+            lostShardIds: shardRenewals
+                .filter(({ renewed }) => !renewed)
+                .map(({ shardId }) => shardId),
+            leaderRenewed,
+            leaderAvailable: leaderLease ? leaderRenewed : Boolean(currentLeader),
+        };
+    })(), timeoutMs);
+}
+export async function acquireFencedLeader(coordinator, descriptor, deploymentOrder, timeoutMs = LEASE_VALIDATION_TIMEOUT_MS) {
+    return withTimeout(coordinator.acquireLeader(descriptor, deploymentOrder), timeoutMs);
+}
+export function cloudRequestExpired(request) {
+    return new Date(request.deadlineAt).getTime() <= Date.now();
+}
+export async function spawnWithShutdownGuard(isShuttingDown, spawn, kill) {
+    if (isShuttingDown())
+        return false;
+    await spawn();
+    if (!isShuttingDown())
+        return true;
+    kill();
+    return false;
+}
+function identityMatchesLease(identity, lease) {
+    return (identity.generation === lease.generation &&
+        (identity.deploymentOrder ?? 1) === lease.deploymentOrder &&
+        identity.supervisorId === lease.supervisorId &&
+        identity.shardId === lease.shardId &&
+        identity.fencingToken === lease.fencingToken);
+}
+function sameShardLease(left, right) {
+    return (left.generation === right.generation &&
+        left.deploymentOrder === right.deploymentOrder &&
+        left.supervisorId === right.supervisorId &&
+        left.shardId === right.shardId &&
+        left.fencingToken === right.fencingToken);
+}
+export async function nextShardDeploymentOrder(database) {
+    let value;
+    try {
+        value = await new RuntimeStateRepository(database).nextShardDeploymentOrder();
+    }
+    catch (error) {
+        if (error.code !== "42P01")
+            throw error;
+        return 1;
+    }
+    if (value > BigInt(Number.MAX_SAFE_INTEGER)) {
+        throw new Error("Shard deployment order exceeds the safe runtime range");
+    }
+    return Number(value);
+}
+export async function renewStartupOwnership(coordinator, descriptor, deploymentOrder, leaderLease) {
+    if (!(await coordinator.refreshGeneration(descriptor, deploymentOrder)))
+        throw new Error("Shard generation ownership was lost during startup");
+    if (leaderLease && !(await coordinator.renewLeader(leaderLease)))
+        throw new Error("Shard leader ownership was lost during startup");
+}
+function botVersion() {
+    const metadata = createRequire(import.meta.url)("../package.json");
+    if (typeof metadata !== "object" ||
+        metadata === null ||
+        !("version" in metadata) ||
+        typeof metadata.version !== "string") {
+        throw new Error("The bot package version is invalid");
+    }
+    return metadata.version;
+}
+//# sourceMappingURL=shard-supervisor.js.map