@helyx/bot 0.4.33 → 0.4.36

package/dist/shard-supervisor.js

@@ -23,6 +23,7 @@ import { createDeveloperOperationsModule } from "./developer-operations-module.j
 import { SupervisorCloudDiscordDirectory } from "./supervisor-cloud-directory.js";
 import { createSupportModule } from "./support-module.js";
 import { createOAuthModule } from "./oauth-module.js";
+import { availabilityCatalogue, availabilityCatalogueDefinitions, } from "./module-availability-control.js";
 import { createScheduledTaskRunner, schedulerOperationalHealth, } from "./scheduled-task-runner.js";
 import { numericWriteBehindRegistrations } from "./numeric-write-behind.js";
 import { executeRedisNumericOperation, RedisNumericWriteBehindService, } from "./numeric-write-behind-redis.js";
@@ -56,6 +57,13 @@ export function createStartupWorkerHeartbeatMaintenance(heartbeat, intervalMs =
         stop: () => clearInterval(timer),
     };
 }
+export function relinquishRestartCircuitOwnership(input) {
+    const lease = input.localLeases.get(input.shardId);
+    if (!lease)
+        return Promise.resolve(false);
+    input.localLeases.delete(input.shardId);
+    return input.release(lease);
+}
 export async function deferScheduledRoutingFailures(route) {
     try {
         return await route();
@@ -119,6 +127,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
     const pendingNumericCommitted = new Map();
     const restartHistory = new Map();
     const restartCounts = new Map();
+    const openRestartCircuits = new Set();
     const workers = new Map();
     let sessionBudget;
     let databaseHealthy = false;
@@ -229,6 +238,10 @@ export async function startShardSupervisor(environment, processEnvironment, work
             ...publicModules,
             ...(operationsModule ? [operationsModule] : []),
         ];
+        const availabilityModules = availabilityCatalogueDefinitions(modules);
+        const availabilityService = {
+            listAvailabilityModules: () => availabilityCatalogue(availabilityModules),
+        };
         const initialiseLeader = async () => {
             await runLeaderStartup(database, environment, modulePackages, modules, logger);
             assertStartupLeaseHealthy();
@@ -258,6 +271,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
                 localLeases,
                 dispatchLocal,
                 privacy: privacyService,
+                availability: availabilityService,
                 localLeaderLease: leaderLease,
                 ...(coordinator ? { coordinator } : {}),
                 ...(environment.DISCORD_INTERNAL_GUILD_ID
@@ -351,6 +365,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
             localLeases,
             dispatchLocal,
             ...(privacyService ? { privacy: privacyService } : {}),
+            availability: availabilityService,
             ...(leaderLease ? { localLeaderLease: leaderLease } : {}),
             ...(coordinator ? { coordinator } : {}),
             ...(environment.DISCORD_INTERNAL_GUILD_ID
@@ -526,8 +541,9 @@ export async function startShardSupervisor(environment, processEnvironment, work
     }
     function installNumericCommitRouting(service) {
         for (const registration of numericRegistrations) {
-            service.subscribeCommitted(registration, async (batch) => {
+            service.subscribeCommitted(registration, async (batch, signal) => {
                 for (let index = 0; index < batch.mutations.length; index += 1) {
+                    signal.throwIfAborted();
                     const mutation = batch.mutations[index];
                     const result = batch.results[index];
                     const guildId = mutation?.identity.guild_id;
@@ -563,32 +579,53 @@ export async function startShardSupervisor(environment, processEnvironment, work
                         payload: encodeNumericIpcValue(committedItem),
                     };
                     assertShardMessageSize(notification);
-                    const response = await dispatchNumericCommitted(shardId, identity, notification);
+                    const response = await dispatchNumericCommitted(shardId, identity, notification, signal);
                     if (!response.ok)
                         throw new Error(`Committed numeric batch was not applied: ${response.safeCode}`);
                 }
             });
         }
     }
-    function dispatchNumericCommitted(shardId, identity, notification) {
+    function dispatchNumericCommitted(shardId, identity, notification, signal) {
         const worker = workers.get(shardId);
         if (!worker?.process)
             return Promise.reject(new Error("Committed numeric batch owner is unavailable"));
         return new Promise((resolve, reject) => {
+            const onAbort = () => {
+                const pending = pendingNumericCommitted.get(notification.requestId);
+                if (!pending)
+                    return;
+                pendingNumericCommitted.delete(notification.requestId);
+                clearTimeout(pending.timer);
+                pending.removeAbortListener();
+                reject(signal.reason instanceof Error
+                    ? signal.reason
+                    : new Error("Numeric claim ownership was lost"));
+            };
             const timer = setTimeout(() => {
+                const pending = pendingNumericCommitted.get(notification.requestId);
                 pendingNumericCommitted.delete(notification.requestId);
+                pending?.removeAbortListener();
                 reject(new Error("Committed numeric batch acknowledgement timed out"));
             }, SHARD_CONTROL_TIMEOUT_MS);
             timer.unref();
+            signal.addEventListener("abort", onAbort, { once: true });
             pendingNumericCommitted.set(notification.requestId, {
                 shardId,
                 identity,
                 resolve,
                 timer,
+                removeAbortListener: () => signal.removeEventListener("abort", onAbort),
             });
+            if (signal.aborted) {
+                onAbort();
+                return;
+            }
             void worker.send(notification).catch((error) => {
                 clearTimeout(timer);
+                const pending = pendingNumericCommitted.get(notification.requestId);
                 pendingNumericCommitted.delete(notification.requestId);
+                pending?.removeAbortListener();
                 reject(error instanceof Error
                     ? error
                     : new Error("Committed numeric delivery failed"));
@@ -713,6 +750,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
                 return;
             pendingNumericCommitted.delete(message.data.requestId);
             clearTimeout(pending.timer);
+            pending.removeAbortListener();
             pending.resolve(message.data);
             return;
         }
@@ -902,6 +940,25 @@ export async function startShardSupervisor(environment, processEnvironment, work
         const now = Date.now();
         const recent = (restartHistory.get(shardId) ?? []).filter((time) => now - time < RESTART_WINDOW_MS);
         if (recent.length >= MAX_RESTARTS_PER_WINDOW) {
+            openRestartCircuits.add(shardId);
+            const circuitCoordinator = coordinator;
+            if (circuitCoordinator)
+                void relinquishRestartCircuitOwnership({
+                    shardId,
+                    localLeases,
+                    release: (lease) => circuitCoordinator.releaseShard(lease),
+                })
+                    .then((released) => {
+                    if (!released)
+                        logger.warn({ event: "shard.circuit_lease_release_stale", shardId }, "Shard restart circuit lease was already reassigned");
+                })
+                    .catch((error) => logger.warn({
+                    event: "shard.circuit_lease_release_failed",
+                    shardId,
+                    error: safeError(error),
+                }, "Shard restart circuit lease will expire without renewal"));
+            else
+                localLeases.delete(shardId);
             logger.error({ event: "shard.circuit_open", shardId, restartCount: recent.length }, "Shard worker restart circuit opened");
             return;
         }
@@ -991,7 +1048,9 @@ export async function startShardSupervisor(environment, processEnvironment, work
         try {
             const occupied = new Set((await withTimeout(coordinator.getShardLeases(shardCount), LEASE_VALIDATION_TIMEOUT_MS).catch(() => [])).map(({ shardId }) => shardId));
             const candidates = Array.from({ length: shardCount }, (_, shardId) => shardId)
-                .filter((shardId) => !localLeases.has(shardId) && !occupied.has(shardId))
+                .filter((shardId) => !localLeases.has(shardId) &&
+                !occupied.has(shardId) &&
+                !openRestartCircuits.has(shardId))
                 .slice(0, capacity);
             const results = await Promise.all(candidates.map(async (shardId) => {
                 const lease = await withTimeout(coordinator.claimShard(generationDescriptor, shardId, deploymentOrder), LEASE_VALIDATION_TIMEOUT_MS).catch(() => undefined);

package/dist/shard-worker.d.ts

@@ -50,12 +50,15 @@ export declare function parseShardWorkerEnvironment(processEnvironment: NodeJS.P
     HELYX_OAUTH_ENABLED: boolean;
     HELYX_CLOUD_ENDPOINT: string;
     HELYX_OWNER_USER_IDS: string[];
-    HELYX_SHARD_MODE: "off" | "auto" | "fixed";
+    HELYX_SHARD_MODE: "fixed" | "off" | "auto";
     HELYX_SHARD_SCOPE: "local" | "coordinated";
     HELYX_CLOUD_AUDIT_RETENTION_DAYS: number;
     HELYX_CLOUD_CREDENTIAL_AUDIT_RETENTION_DAYS: number;
     LOG_LEVEL: "error" | "fatal" | "warn" | "info" | "debug" | "trace" | "silent";
     NODE_ENV: "development" | "test" | "production";
+    HELYX_STATUS_ENABLED: boolean;
+    HELYX_STATUS_URL: string;
+    HELYX_STATUS_HEARTBEAT_INTERVAL_SECONDS: number;
     HELYX_BOT_CONTROL_SECRET?: string | undefined;
     HELYX_SUPPORT_SERVICE_SECRET?: string | undefined;
     HELYX_VERIFICATION_INSTALLATION_ID?: string | undefined;
@@ -69,6 +72,9 @@ export declare function parseShardWorkerEnvironment(processEnvironment: NodeJS.P
     HELYX_SHARD_REDIS_NAMESPACE?: string | undefined;
     HELYX_SHARD_GENERATION?: string | undefined;
     HELYX_SHARD_DATABASE_CONNECTION_BUDGET?: number | undefined;
+    HELYX_STATUS_HEARTBEAT_SECRET?: string | undefined;
+    CF_ACCESS_CLIENT_ID?: string | undefined;
+    CF_ACCESS_CLIENT_SECRET?: string | undefined;
 };
 export declare function shardWorkerCommandRegistration(input: {
     internalGuildId?: string;

package/dist/shard-worker.js

@@ -3,7 +3,7 @@ import { safeError, ServiceRegistry } from "@helyx/core";
 import { checkDatabaseConnection, createDatabasePool } from "@helyx/database";
 import { createRuntime } from "@helyx/framework";
 import { createPlatformServices } from "@helyx/platform";
-import { HELYX_SERVICE_NAMES } from "@helyx/sdk";
+import { HELYX_SERVICE_NAMES, } from "@helyx/sdk";
 import { Client, Events, Partials } from "discord.js";
 import { monitorEventLoopDelay } from "node:perf_hooks";
 import pino from "pino";
@@ -179,6 +179,9 @@ export async function startShardWorker(processEnvironment = process.env, identit
         })) {
             platformServices.set(name, service);
         }
+        await serviceAccess
+            .get(HELYX_SERVICE_NAMES.moduleAvailability)
+            .refresh();
         if (environment.HELYX_VERIFICATION_ENABLED &&
             environment.HELYX_VERIFICATION_URL &&
             environment.HELYX_VERIFICATION_INTERNAL_SECRET) {

package/dist/support-module.js

@@ -1,5 +1,6 @@
 import { defineModule, HELYX_SERVICE_NAMES, } from "@helyx/sdk";
 import { randomUUID } from "node:crypto";
+import { INTERNAL_MODULE_MANIFEST_FIELDS } from "./internal-module-manifest.js";
 const MODAL_ID = "helyx.support.bug";
 const MODULE_FIELD_ID = "helyx.support.bug.module";
 const SUMMARY_FIELD_ID = "helyx.support.bug.summary";
@@ -11,7 +12,7 @@ const HOUR_SECONDS = 60 * 60;
 export function createSupportModule(transport) {
     return defineModule({
         manifest: {
-            manifestVersion: "0.1",
+            ...INTERNAL_MODULE_MANIFEST_FIELDS,
             releaseStatus: "live",
             id: MODULE_ID,
             name: "Helyx Support",
@@ -28,6 +29,7 @@ export function createSupportModule(transport) {
                 {
                     name: "bug",
                     description: "Report a Helyx bug to the support team",
+                    acknowledgement: "immediate-modal",
                     registration: "global",
                     contexts: ["guild"],
                     integrationTypes: ["guild_install"],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helyx/bot",
-  "version": "0.4.33",
+  "version": "0.4.36",
   "description": "Production Discord runtime for the modular Helyx platform.",
   "keywords": [
     "helyx",
@@ -53,9 +53,10 @@
     "@helyx/core": ">=0.1.0 <0.2.0",
     "@helyx/database": ">=0.8.0 <0.9.0",
     "@helyx/framework": ">=0.1.0 <0.2.0",
-    "@helyx/module-manifest": ">=0.6.0 <0.7.0",
-    "@helyx/platform": ">=0.5.0 <0.6.0",
-    "@helyx/sdk": ">=0.7.0 <0.8.0",
+    "@helyx/module-manifest": ">=0.7.0 <0.8.0",
+    "@helyx/platform": ">=0.6.0 <0.7.0",
+    "@helyx/sdk": ">=0.8.0 <0.9.0",
+    "@helyx/status-client": ">=0.1.0 <0.2.0",
     "@helyx/verification-contracts": ">=0.3.0 <0.4.0",
     "discord.js": "14.27.0",
     "fastify": "5.10.0",