@helyx/bot 0.4.35 → 0.4.37

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.4.37
+
+### Patch Changes
+
+- Publish the launch-readiness bot from a clean build and update the embedded self-host catalogue to the corrected runtime package.
+
+## 0.4.36
+
+### Patch Changes
+
+- Harden startup cleanup, lease ownership, database transaction handling, worker shutdown, and stale-work fencing across the hosted and self-hosted runtime.
+
 ## 0.4.35
 
 ### Patch Changes

package/dist/cloud-management-module.js

@@ -1,6 +1,7 @@
 import { defineModule, } from "@helyx/sdk";
 import { Events } from "discord.js";
 import { ClientCloudDiscordDirectory, CloudConnectorUserError, } from "./cloud-connector.js";
+import { INTERNAL_MODULE_MANIFEST_FIELDS } from "./internal-module-manifest.js";
 const CONNECT_MODAL_ID = "helyx.cloud.connect";
 const CONNECT_CODE_FIELD_ID = "helyx.cloud.connect.code";
 const DISCONNECT_MODAL_ID = "helyx.cloud.disconnect";
@@ -72,7 +73,7 @@ export function createCloudManagementModule(input) {
     };
     return defineModule({
         manifest: {
-            manifestVersion: "0.1",
+            ...INTERNAL_MODULE_MANIFEST_FIELDS,
             releaseStatus: "live",
             id: "helyx.cloud-connect",
             name: "Helyx Cloud Connect",

package/dist/control-server.js

@@ -7,6 +7,7 @@ import Fastify, {} from "fastify";
 import { createHash, timingSafeEqual } from "node:crypto";
 import { z, ZodError } from "zod";
 import { addChannelMessageReaction as addDiscordChannelMessageReaction, deleteChannelMessage as deleteDiscordChannelMessage, editChannelMessage as editDiscordChannelMessage, normalizeDiscordPermission, sendToChannel as sendDiscordMessageToChannel, } from "./interactions.js";
+import { runNestedGuildBarriers } from "./numeric-barriers.js";
 import { SHARD_HEALTH_STALE_MS, } from "./shard-contracts.js";
 import { dashboardCategoryLabel } from "./dashboard-category.js";
 import { assertVersion, dashboardActionPermissionId, declaredManagedResourcePermission, fromDashboardPolicy, managedResourcePermissionId, permissionPoliciesEqual, permissionPolicyKey, safeOptionalVersion, safeVersion, toDashboardPolicy, } from "./control-permissions.js";
@@ -663,15 +664,11 @@ export class BotControlService {
             await setEnabled();
         }
         else {
-            const runBoundary = async (index) => {
-                const scope = scopes[index];
-                if (!scope) {
-                    await setEnabled();
-                    return;
-                }
-                await writeBehind.runGuildBarrier(scope, () => runBoundary(index + 1));
-            };
-            await runBoundary(0);
+            await runNestedGuildBarriers({
+                scopes,
+                writeBehind,
+                operation: setEnabled,
+            });
             for (const scope of scopes)
                 await writeBehind.flushGuild(scope);
             for (const scope of scopes)

package/dist/developer-operations-module.js

@@ -1,6 +1,7 @@
 import { defineModule, } from "@helyx/sdk";
 import { randomUUID } from "node:crypto";
 import { SHARD_CONTROL_TIMEOUT_MS, assertShardMessageSize, shardOperationsHealthResponseSchema, } from "./shard-contracts.js";
+import { INTERNAL_MODULE_MANIFEST_FIELDS } from "./internal-module-manifest.js";
 const DEVELOPER_MODULE_ID = "helyx.developer-operations";
 const DEVELOPER_MODULE_NAME = "Helyx Developer Operations";
 const DEVELOPER_MODULE_VERSION = "0.2.0";
@@ -9,7 +10,7 @@ export function createDeveloperOperationsModule(input) {
     const approvedRoles = new Set(input.approvedRoleIds);
     return defineModule({
         manifest: {
-            manifestVersion: "0.1",
+            ...INTERNAL_MODULE_MANIFEST_FIELDS,
             releaseStatus: "live",
             id: DEVELOPER_MODULE_ID,
             name: DEVELOPER_MODULE_NAME,

package/dist/events.js

@@ -300,8 +300,11 @@ export function installModuleEvents(client, modules, logger, services) {
     async function handlePrivilegedMessageCreate(message, contributions) {
         if (!message.guildId || !message.channelId)
             return;
+        const installations = availabilityAwareInstallations(services);
         for (const { moduleId, contribution } of contributions) {
             try {
+                if (!(await installations.isModuleEnabled(message.guildId, moduleId)))
+                    continue;
                 await contribution.execute({
                     serverId: message.guildId,
                     channelId: message.channelId,
@@ -827,13 +830,18 @@ class BoundedGuildEventQueue {
         this.#accepting = false;
         if (this.#active === 0 && this.#pending.length === 0)
             return;
-        await Promise.race([
-            new Promise((resolve) => this.#drainWaiters.add(resolve)),
+        const drain = new Promise((resolve) => this.#drainWaiters.add(resolve));
+        const drainedBeforeDeadline = await Promise.race([
+            drain.then(() => true),
             new Promise((resolve) => {
-                const timer = setTimeout(resolve, 5_000);
+                const timer = setTimeout(() => resolve(false), 5_000);
                 timer.unref();
             }),
         ]);
+        if (!drainedBeforeDeadline) {
+            this.logger.warn({ event: "module.event_stop_deadline_exceeded" }, "Module gateway event queue is still draining after its shutdown deadline");
+            await drain;
+        }
         this.#pending.length = 0;
     }
     #pump() {

package/dist/internal-module-manifest.d.ts

@@ -0,0 +1,13 @@
+export declare const INTERNAL_MODULE_MANIFEST_FIELDS: {
+    readonly manifestVersion: "0.3";
+    readonly moduleKind: "internal_dependency";
+    readonly customerSelectable: false;
+    readonly countsAsInstalledModule: false;
+    readonly healthExposure: "delegated";
+    readonly privacy: {
+        containsPersonalData: false;
+        providerVersion: number;
+        collections: never[];
+    };
+};
+//# sourceMappingURL=internal-module-manifest.d.ts.map

package/dist/internal-module-manifest.js

@@ -0,0 +1,14 @@
+const noOwnedPersonalData = {
+    containsPersonalData: false,
+    providerVersion: 1,
+    collections: [],
+};
+export const INTERNAL_MODULE_MANIFEST_FIELDS = {
+    manifestVersion: "0.3",
+    moduleKind: "internal_dependency",
+    customerSelectable: false,
+    countsAsInstalledModule: false,
+    healthExposure: "delegated",
+    privacy: noOwnedPersonalData,
+};
+//# sourceMappingURL=internal-module-manifest.js.map

package/dist/management-module.js

@@ -1,5 +1,6 @@
 import { isCustomerSelectableModule, } from "@helyx/module-manifest";
 import { defineModule, HELYX_SERVICE_NAMES, ModuleStateValidationError, } from "@helyx/sdk";
+import { INTERNAL_MODULE_MANIFEST_FIELDS } from "./internal-module-manifest.js";
 const MODULE_SELECT_ID = "helyx.management.module";
 const SETTING_SELECT_ID = "helyx.management.setting";
 const VALUE_SELECT_ID = "helyx.management.value";
@@ -31,7 +32,7 @@ export function createManagementModule(configurableModules, resources) {
     const byId = new Map(modules.map((module) => [module.manifest.id, module]));
     return defineModule({
         manifest: {
-            manifestVersion: "0.1",
+            ...INTERNAL_MODULE_MANIFEST_FIELDS,
             releaseStatus: "live",
             id: "helyx.management",
             name: "Helyx management",

package/dist/numeric-barriers.d.ts

@@ -0,0 +1,11 @@
+import type { NumericWriteBehindScope, NumericWriteBehindService } from "@helyx/sdk";
+type GuildScope = NumericWriteBehindScope & {
+    guildId: string;
+};
+export declare function runNestedGuildBarriers(input: {
+    scopes: readonly GuildScope[];
+    writeBehind: NumericWriteBehindService;
+    operation(): Promise<unknown>;
+}): Promise<void>;
+export {};
+//# sourceMappingURL=numeric-barriers.d.ts.map

package/dist/numeric-barriers.js

@@ -0,0 +1,14 @@
+export async function runNestedGuildBarriers(input) {
+    const run = async (index, inheritedSignal) => {
+        inheritedSignal?.throwIfAborted();
+        const scope = input.scopes[index];
+        if (!scope) {
+            await input.operation();
+            inheritedSignal?.throwIfAborted();
+            return;
+        }
+        await input.writeBehind.runGuildBarrier(scope, (signal) => run(index + 1, inheritedSignal ? AbortSignal.any([inheritedSignal, signal]) : signal));
+    };
+    await run(0);
+}
+//# sourceMappingURL=numeric-barriers.js.map

package/dist/numeric-write-behind-proxy.d.ts

@@ -19,13 +19,13 @@ export declare class WorkerNumericWriteBehindProxy implements NumericWriteBehind
         subjectId: string;
     }): Promise<void>;
     flushModule(input: NumericWriteBehindScope): Promise<void>;
-    runMemberBarrier<Result>(input: NumericWriteBehindMemberScope, operation: () => Promise<Result>): Promise<Result>;
+    runMemberBarrier<Result>(input: NumericWriteBehindMemberScope, operation: (signal: AbortSignal) => Promise<Result>): Promise<Result>;
     runGuildBarrier<Result>(input: NumericWriteBehindScope & {
         guildId: string;
-    }, operation: () => Promise<Result>): Promise<Result>;
+    }, operation: (signal: AbortSignal) => Promise<Result>): Promise<Result>;
     runSubjectBarrier<Result>(input: NumericWriteBehindScope & {
         subjectId: string;
-    }, operation: () => Promise<Result>): Promise<Result>;
+    }, operation: (signal: AbortSignal) => Promise<Result>): Promise<Result>;
     suppressSubject(input: NumericWriteBehindScope & {
         subjectId: string;
         cutoff: Date;
@@ -55,7 +55,7 @@ export declare class WorkerNumericWriteBehindProxy implements NumericWriteBehind
     clearSubjectCooldown(input: NumericWriteBehindScope & {
         subjectId: string;
     }): Promise<void>;
-    subscribeCommitted(input: NumericWriteBehindScope, handler: (batch: NumericWriteBehindCommittedBatch) => void | Promise<void>): () => void;
+    subscribeCommitted(input: NumericWriteBehindScope, handler: (batch: NumericWriteBehindCommittedBatch, signal: AbortSignal) => void | Promise<void>): () => void;
     stop(): void;
 }
 export declare function encodeNumericIpcValue(input: unknown): unknown;

package/dist/numeric-write-behind-proxy.js

@@ -1,6 +1,7 @@
 import {} from "@helyx/sdk";
 import { randomUUID } from "node:crypto";
 import { assertShardMessageSize, SHARD_CONTROL_TIMEOUT_MS, shardNumericCommittedSchema, shardNumericResponseSchema, } from "./shard-contracts.js";
+import { RenewableLease } from "./renewable-lease.js";
 export class WorkerNumericWriteBehindProxy {
     #identity;
     #pending = new Map();
@@ -157,37 +158,44 @@ export class WorkerNumericWriteBehindProxy {
             ...(input.guildId ? { guildId: input.guildId } : {}),
             ...(input.subjectId ? { subjectId: input.subjectId } : {}),
         }));
-        let renewalFailure;
-        const renew = setInterval(() => {
-            void this.#call({
+        const renewal = new RenewableLease({
+            intervalMs: 20_000,
+            renew: () => this.#call({
                 kind: "barrier.renew",
                 ...scope(input),
                 barrierType,
                 ...(input.guildId ? { guildId: input.guildId } : {}),
                 ...(input.subjectId ? { subjectId: input.subjectId } : {}),
                 barrierId: acquired.barrierId,
-            }).catch((error) => {
-                renewalFailure =
-                    error instanceof Error
-                        ? error
-                        : new Error("Numeric barrier renewal failed");
+            }).then(() => undefined),
+        });
+        renewal.start();
+        const failures = [];
+        let result;
+        let completed = false;
+        try {
+            result = await renewal.run(async (signal) => {
+                if (barrierType === "member")
+                    await this.flushMember(input);
+                else if (barrierType === "guild")
+                    await this.flushGuild(input);
+                else
+                    await this.flushSubject(input);
+                signal.throwIfAborted();
+                return operation(signal);
             });
-        }, 20_000);
-        renew.unref();
+            completed = true;
+        }
+        catch (error) {
+            failures.push(error);
+        }
         try {
-            if (barrierType === "member")
-                await this.flushMember(input);
-            else if (barrierType === "guild")
-                await this.flushGuild(input);
-            else
-                await this.flushSubject(input);
-            const result = await operation();
-            if (renewalFailure)
-                throw renewalFailure;
-            return result;
+            await renewal.stop();
         }
-        finally {
-            clearInterval(renew);
+        catch (error) {
+            failures.push(error);
+        }
+        try {
             await this.#call({
                 kind: "barrier.release",
                 ...scope(input),
@@ -197,6 +205,16 @@ export class WorkerNumericWriteBehindProxy {
                 barrierId: acquired.barrierId,
             });
         }
+        catch (error) {
+            failures.push(error);
+        }
+        if (failures.length === 1)
+            throw failures[0];
+        if (failures.length > 1)
+            throw new AggregateError(failures, "Numeric barrier cleanup failed");
+        if (!completed)
+            throw new Error("Numeric barrier operation did not complete");
+        return result;
     }
     async #handleCommitted(committed) {
         let response;
@@ -204,8 +222,11 @@ export class WorkerNumericWriteBehindProxy {
             if (new Date(committed.deadlineAt).getTime() <= Date.now())
                 throw new Error("Numeric committed notification expired");
             const batch = decodeIpcValue(committed.payload);
-            for (const subscriber of this.#subscribers.get(scopeKey(committed.scope)) ?? [])
-                await subscriber(batch);
+            const signal = AbortSignal.timeout(Math.max(1, new Date(committed.deadlineAt).getTime() - Date.now()));
+            for (const subscriber of this.#subscribers.get(scopeKey(committed.scope)) ?? []) {
+                signal.throwIfAborted();
+                await subscriber(batch, signal);
+            }
             response = {
                 version: 1,
                 type: "helyx.numeric.committed.response",

package/dist/numeric-write-behind-redis.d.ts

@@ -51,13 +51,13 @@ export declare class RedisNumericWriteBehindService implements NumericWriteBehin
         subjectId: string;
     }): Promise<void>;
     flushModule(input: NumericWriteBehindScope): Promise<void>;
-    runMemberBarrier<Result>(input: NumericWriteBehindMemberScope, operation: () => Promise<Result>): Promise<Result>;
+    runMemberBarrier<Result>(input: NumericWriteBehindMemberScope, operation: (signal: AbortSignal) => Promise<Result>): Promise<Result>;
     runGuildBarrier<Result>(input: NumericWriteBehindScope & {
         guildId: string;
-    }, operation: () => Promise<Result>): Promise<Result>;
+    }, operation: (signal: AbortSignal) => Promise<Result>): Promise<Result>;
     runSubjectBarrier<Result>(input: NumericWriteBehindScope & {
         subjectId: string;
-    }, operation: () => Promise<Result>): Promise<Result>;
+    }, operation: (signal: AbortSignal) => Promise<Result>): Promise<Result>;
     suppressSubject(input: NumericWriteBehindScope & {
         subjectId: string;
         cutoff: Date;
@@ -87,7 +87,7 @@ export declare class RedisNumericWriteBehindService implements NumericWriteBehin
     clearSubjectCooldown(input: NumericWriteBehindScope & {
         subjectId: string;
     }): Promise<void>;
-    subscribeCommitted(input: NumericWriteBehindScope, handler: (batch: NumericWriteBehindCommittedBatch) => void | Promise<void>): () => void;
+    subscribeCommitted(input: NumericWriteBehindScope, handler: (batch: NumericWriteBehindCommittedBatch, signal: AbortSignal) => void | Promise<void>): () => void;
     health(): NumericWriteBehindHealth;
     acquireBarrier(input: {
         scope: NumericWriteBehindScope;

package/dist/numeric-write-behind-redis.js

@@ -2,6 +2,7 @@ import {} from "@helyx/sdk";
 import {} from "@helyx/platform";
 import { randomUUID } from "node:crypto";
 import { createClient } from "redis";
+import { RenewableLease } from "./renewable-lease.js";
 const REDIS_FLUSH_INTERVAL_MS = 30_000;
 const REDIS_BATCH_SIZE = 100;
 const REDIS_CAPACITY = 10_000;
@@ -189,40 +190,13 @@ export class RedisNumericWriteBehindService {
         return this.#flush(input, "", "", true);
     }
     async runMemberBarrier(input, operation) {
-        const barrierId = await this.#acquireBarrier(input, "member", input.guildId, input.subjectId);
-        const stopRenewal = this.#startBarrierRenewal(input, "member", input.guildId, input.subjectId, barrierId);
-        try {
-            await this.flushMember(input);
-            return await operation();
-        }
-        finally {
-            stopRenewal();
-            await this.#releaseBarrier(input, "member", input.guildId, input.subjectId, barrierId);
-        }
+        return this.#runBarrier(input, "member", input.guildId, input.subjectId, () => this.flushMember(input), operation);
     }
     async runGuildBarrier(input, operation) {
-        const barrierId = await this.#acquireBarrier(input, "guild", input.guildId, "");
-        const stopRenewal = this.#startBarrierRenewal(input, "guild", input.guildId, "", barrierId);
-        try {
-            await this.flushGuild(input);
-            return await operation();
-        }
-        finally {
-            stopRenewal();
-            await this.#releaseBarrier(input, "guild", input.guildId, "", barrierId);
-        }
+        return this.#runBarrier(input, "guild", input.guildId, "", () => this.flushGuild(input), operation);
     }
     async runSubjectBarrier(input, operation) {
-        const barrierId = await this.#acquireBarrier(input, "subject", "", input.subjectId);
-        const stopRenewal = this.#startBarrierRenewal(input, "subject", "", input.subjectId, barrierId);
-        try {
-            await this.flushSubject(input);
-            return await operation();
-        }
-        finally {
-            stopRenewal();
-            await this.#releaseBarrier(input, "subject", "", input.subjectId, barrierId);
-        }
+        return this.#runBarrier(input, "subject", "", input.subjectId, () => this.flushSubject(input), operation);
     }
     async suppressSubject(input) {
         this.#assertAvailable(input);
@@ -420,16 +394,17 @@ export class RedisNumericWriteBehindService {
     async #drainRegistration(registration, guildId, subjectId, force) {
         while (true) {
             const proposedOperationId = randomUUID();
+            const proposedClaimToken = randomUUID();
             const raw = await this.#redis.eval(CLAIM_SCRIPT, {
                 keys: keysFor(this.#prefix, registration),
                 arguments: [
                     String(Date.now()),
                     String(REDIS_CLAIM_LEASE_MS),
                     proposedOperationId,
+                    proposedClaimToken,
                     String(REDIS_BATCH_SIZE),
                     guildId,
                     subjectId,
-                    force ? "1" : "0",
                     this.#supervisorId ?? "",
                     this.#shardLeaseKeyPrefix ?? "",
                     String(REDIS_RETRY_RETENTION_MS),
@@ -451,33 +426,45 @@ export class RedisNumericWriteBehindService {
                 ...this.#health,
                 inFlightEntries: claim.entries.length,
             };
+            const renewal = new RenewableLease({
+                intervalMs: Math.max(250, Math.floor(REDIS_CLAIM_LEASE_MS / 3)),
+                renew: () => this.#renewClaim(registration, claim),
+                onError: (error) => this.#onError?.(error),
+            });
+            renewal.start();
             try {
-                const output = await this.#writer.mutateNumericBatch({
-                    moduleId: registration.moduleId,
-                    collection: registration.collection,
-                    operationId: claim.operationId,
-                    uniqueBy: registration.uniqueBy,
-                    mutations: claim.entries.map(decodeMutation),
-                    select: registration.select,
-                });
-                if (output.results.length !== claim.entries.length)
-                    throw new Error("Redis numeric writer result count was invalid");
-                const committed = {
-                    moduleId: registration.moduleId,
-                    collection: registration.collection,
-                    operationId: claim.operationId,
-                    replayed: output.replayed,
-                    mutations: claim.entries.map(decodeMutation),
-                    results: output.results,
-                };
-                for (const subscriber of this.#subscribers.get(scopeKey(registration)) ?? [])
-                    await subscriber(committed);
-                const acknowledged = await this.#redis.eval(ACK_SCRIPT, {
-                    keys: keysFor(this.#prefix, registration),
-                    arguments: [claim.operationId],
+                await renewal.run(async (signal) => {
+                    const output = await this.#writer.mutateNumericBatch({
+                        moduleId: registration.moduleId,
+                        collection: registration.collection,
+                        operationId: claim.operationId,
+                        uniqueBy: registration.uniqueBy,
+                        mutations: claim.entries.map(decodeMutation),
+                        select: registration.select,
+                    });
+                    signal.throwIfAborted();
+                    if (output.results.length !== claim.entries.length)
+                        throw new Error("Redis numeric writer result count was invalid");
+                    const committed = {
+                        moduleId: registration.moduleId,
+                        collection: registration.collection,
+                        operationId: claim.operationId,
+                        replayed: output.replayed,
+                        mutations: claim.entries.map(decodeMutation),
+                        results: output.results,
+                    };
+                    for (const subscriber of this.#subscribers.get(scopeKey(registration)) ?? []) {
+                        signal.throwIfAborted();
+                        await subscriber(committed, signal);
+                    }
+                    signal.throwIfAborted();
+                    const acknowledged = await this.#redis.eval(ACK_SCRIPT, {
+                        keys: keysFor(this.#prefix, registration),
+                        arguments: [claim.operationId, claim.claimToken],
+                    });
+                    if (Number(acknowledged) !== 1)
+                        throw new Error("Redis numeric claim ownership changed");
                 });
-                if (Number(acknowledged) !== 1)
-                    throw new Error("Redis numeric claim ownership changed");
             }
             catch (error) {
                 const attempt = claim.attempt + 1;
@@ -485,7 +472,12 @@ export class RedisNumericWriteBehindService {
                     Math.min(REDIS_RETRY_MAX_MS, REDIS_RETRY_BASE_MS * 2 ** Math.min(claim.attempt, 16));
                 await this.#redis.eval(RETRY_SCRIPT, {
                     keys: keysFor(this.#prefix, registration),
-                    arguments: [claim.operationId, String(attempt), String(retryAt)],
+                    arguments: [
+                        claim.operationId,
+                        claim.claimToken,
+                        String(attempt),
+                        String(retryAt),
+                    ],
                 });
                 this.#health = {
                     ...this.#health,
@@ -498,6 +490,9 @@ export class RedisNumericWriteBehindService {
                     throw error;
                 return;
             }
+            finally {
+                await renewal.stop();
+            }
             this.#health = {
                 ...this.#health,
                 inFlightEntries: 0,
@@ -505,6 +500,20 @@ export class RedisNumericWriteBehindService {
             };
         }
     }
+    async #renewClaim(registration, claim) {
+        const now = Date.now();
+        const renewed = await this.#redis.eval(RENEW_CLAIM_SCRIPT, {
+            keys: keysFor(this.#prefix, registration),
+            arguments: [
+                claim.operationId,
+                claim.claimToken,
+                String(now),
+                String(now + REDIS_CLAIM_LEASE_MS),
+            ],
+        });
+        if (Number(renewed) !== 1)
+            throw new Error("Redis numeric claim lease expired");
+    }
     async #acquireBarrier(scope, kind, guildId, subjectId) {
         this.#assertAvailable(scope);
         const barrierId = randomUUID();
@@ -550,12 +559,47 @@ export class RedisNumericWriteBehindService {
         if (Number(result) !== 1)
             throw new Error("Numeric write-behind barrier lease expired");
     }
-    #startBarrierRenewal(scope, kind, guildId, subjectId, barrierId) {
-        const timer = setInterval(() => {
-            void this.#renewBarrier(scope, kind, guildId, subjectId, barrierId).catch((error) => this.#onError?.(toError(error)));
-        }, Math.max(250, Math.floor(this.#barrierLeaseMs / 3)));
-        timer.unref();
-        return () => clearInterval(timer);
+    async #runBarrier(scope, kind, guildId, subjectId, flush, operation) {
+        const barrierId = await this.#acquireBarrier(scope, kind, guildId, subjectId);
+        const renewal = new RenewableLease({
+            intervalMs: Math.max(250, Math.floor(this.#barrierLeaseMs / 3)),
+            renew: () => this.#renewBarrier(scope, kind, guildId, subjectId, barrierId),
+            onError: (error) => this.#onError?.(error),
+        });
+        renewal.start();
+        const failures = [];
+        let result;
+        let completed = false;
+        try {
+            result = await renewal.run(async (signal) => {
+                await flush();
+                signal.throwIfAborted();
+                return operation(signal);
+            });
+            completed = true;
+        }
+        catch (error) {
+            failures.push(error);
+        }
+        try {
+            await renewal.stop();
+        }
+        catch (error) {
+            failures.push(error);
+        }
+        try {
+            await this.#releaseBarrier(scope, kind, guildId, subjectId, barrierId);
+        }
+        catch (error) {
+            failures.push(error);
+        }
+        if (failures.length === 1)
+            throw failures[0];
+        if (failures.length > 1)
+            throw new AggregateError(failures, "Numeric barrier cleanup failed");
+        if (!completed)
+            throw new Error("Numeric barrier operation did not complete");
+        return result;
     }
     async #waitForSuppression(scope, script, arguments_) {
         const deadline = Date.now() + this.#suppressionDrainTimeoutMs;
@@ -785,6 +829,9 @@ function parseClaim(raw) {
         !("operationId" in parsed) ||
         typeof parsed.operationId !== "string" ||
         !UUID.test(parsed.operationId) ||
+        !("claimToken" in parsed) ||
+        typeof parsed.claimToken !== "string" ||
+        !UUID.test(parsed.claimToken) ||
         !("entries" in parsed) ||
         !Array.isArray(parsed.entries) ||
         parsed.entries.length < 1 ||
@@ -940,10 +987,10 @@ ${BARRIER_HELPERS_LUA}
 local now = tonumber(ARGV[1])
 local lease_ms = tonumber(ARGV[2])
 local proposed_id = ARGV[3]
-local limit = tonumber(ARGV[4])
-local guild_filter = ARGV[5]
-local subject_filter = ARGV[6]
-local force = ARGV[7] == '1'
+local claim_token = ARGV[4]
+local limit = tonumber(ARGV[5])
+local guild_filter = ARGV[6]
+local subject_filter = ARGV[7]
 local owner = ARGV[8]
 local lease_prefix = ARGV[9]
 local retry_retention_ms = tonumber(ARGV[10])
@@ -958,7 +1005,7 @@ for index = 1, #staged, 2 do
     redis.call('HDEL', KEYS[3], member_key)
   end
 end
-local due = redis.call('ZRANGEBYSCORE', KEYS[5], '-inf', force and '+inf' or now, 'LIMIT', 0, 100)
+local due = redis.call('ZRANGEBYSCORE', KEYS[5], '-inf', now, 'LIMIT', 0, 100)
 for _, operation_id in ipairs(due) do
   local payload = redis.call('HGET', KEYS[4], operation_id)
   if payload then
@@ -1000,6 +1047,7 @@ for _, operation_id in ipairs(due) do
       end
       claim.leaseKey = lease_key
       claim.leaseValue = lease_value
+      claim.claimToken = claim_token
       local encoded = cjson.encode(claim)
       redis.call('HSET', KEYS[4], operation_id, encoded)
       redis.call('ZADD', KEYS[5], now + lease_ms, operation_id)
@@ -1049,6 +1097,7 @@ if #entries == 0 then
 end
 local claim = {
   operationId = proposed_id,
+  claimToken = claim_token,
   entries = entries,
   attempt = 0,
   expiresAt = now + retry_retention_ms,
@@ -1068,6 +1117,7 @@ const ACK_SCRIPT = `
 local payload = redis.call('HGET', KEYS[4], ARGV[1])
 if not payload then return 0 end
 local claim = cjson.decode(payload)
+if claim.claimToken ~= ARGV[2] then return 0 end
 if claim.leaseKey and claim.leaseKey ~= '' and
    redis.call('GET', claim.leaseKey) ~= claim.leaseValue then return 0 end
 for _, entry in ipairs(claim.entries) do
@@ -1082,13 +1132,25 @@ const RETRY_SCRIPT = `
 local payload = redis.call('HGET', KEYS[4], ARGV[1])
 if not payload then return 0 end
 local claim = cjson.decode(payload)
-claim.attempt = tonumber(ARGV[2])
+if claim.claimToken ~= ARGV[2] then return 0 end
+claim.attempt = tonumber(ARGV[3])
 redis.call('HSET', KEYS[4], ARGV[1], cjson.encode(claim))
-local retry_at = tonumber(ARGV[3])
+local retry_at = tonumber(ARGV[4])
 if claim.expiresAt and retry_at > tonumber(claim.expiresAt) then retry_at = tonumber(claim.expiresAt) end
 redis.call('ZADD', KEYS[5], retry_at, ARGV[1])
 return 1
 `;
+const RENEW_CLAIM_SCRIPT = `
+local payload = redis.call('HGET', KEYS[4], ARGV[1])
+if not payload then return 0 end
+local claim = cjson.decode(payload)
+if claim.claimToken ~= ARGV[2] then return 0 end
+if claim.leaseKey and claim.leaseKey ~= '' and
+   redis.call('GET', claim.leaseKey) ~= claim.leaseValue then return 0 end
+if tonumber(redis.call('ZSCORE', KEYS[5], ARGV[1]) or 0) <= tonumber(ARGV[3]) then return 0 end
+redis.call('ZADD', KEYS[5], ARGV[4], ARGV[1])
+return 1
+`;
 const HEALTH_SCRIPT = `
 local oldest = redis.call('ZRANGE', KEYS[2], 0, 0, 'WITHSCORES')
 return cjson.encode({

package/dist/oauth-module.js

@@ -1,4 +1,5 @@
 import { defineModule, HELYX_SERVICE_NAMES, } from "@helyx/sdk";
+import { INTERNAL_MODULE_MANIFEST_FIELDS } from "./internal-module-manifest.js";
 const MODULE_ID = "helyx.oauth";
 const MODAL_ID = "helyx.oauth.revoke";
 const SERVICE_FIELD_ID = "helyx.oauth.revoke.service";
@@ -15,7 +16,7 @@ const everyonePolicy = {
 export function createOAuthModule() {
     return defineModule({
         manifest: {
-            manifestVersion: "0.1",
+            ...INTERNAL_MODULE_MANIFEST_FIELDS,
             releaseStatus: "live",
             id: MODULE_ID,
             name: "Helyx OAuth",

package/dist/permissions-module.js

@@ -2,6 +2,7 @@ import { defineModule, HELYX_SERVICE_NAMES, } from "@helyx/sdk";
 import { isCustomerSelectableModule } from "@helyx/module-manifest";
 import { PermissionFlagsBits } from "discord.js";
 import { canMutatePermissionPolicy, permissionMutationDenialMessage, } from "@helyx/core";
+import { INTERNAL_MODULE_MANIFEST_FIELDS } from "./internal-module-manifest.js";
 const MODULE_SELECT_ID = "helyx.permissions.module";
 const TARGET_SELECT_ID = "helyx.permissions.target";
 const MODULE_PREVIOUS_ID = "helyx.permissions.module.previous";
@@ -26,7 +27,7 @@ export function createPermissionsModule(configurableModules) {
         .sort((left, right) => left.manifest.name.localeCompare(right.manifest.name, "en-GB"));
     return defineModule({
         manifest: {
-            manifestVersion: "0.1",
+            ...INTERNAL_MODULE_MANIFEST_FIELDS,
             releaseStatus: "live",
             id: "helyx.permissions",
             name: "Permissions",

package/dist/renewable-lease.d.ts

@@ -0,0 +1,16 @@
+interface RenewableLeaseOptions {
+    intervalMs: number;
+    renew(): Promise<void>;
+    onError?(error: Error): void;
+}
+export declare class RenewableLease {
+    #private;
+    private readonly options;
+    constructor(options: RenewableLeaseOptions);
+    start(): void;
+    assertOwned(): void;
+    run<Result>(operation: (signal: AbortSignal) => Promise<Result>): Promise<Result>;
+    stop(): Promise<void>;
+}
+export {};
+//# sourceMappingURL=renewable-lease.d.ts.map

package/dist/renewable-lease.js

@@ -0,0 +1,66 @@
+export class RenewableLease {
+    options;
+    #controller = new AbortController();
+    #lost;
+    #resolveLost;
+    #failure;
+    #renewing;
+    #timer;
+    constructor(options) {
+        this.options = options;
+        let resolveLost;
+        this.#lost = new Promise((resolve) => {
+            resolveLost = resolve;
+        });
+        this.#resolveLost = resolveLost;
+    }
+    start() {
+        if (this.#timer || this.#failure)
+            return;
+        this.#timer = setInterval(() => this.#tick(), this.options.intervalMs);
+        this.#timer.unref();
+    }
+    assertOwned() {
+        if (this.#failure)
+            throw this.#failure;
+    }
+    async run(operation) {
+        this.assertOwned();
+        const operationOutcome = Promise.resolve()
+            .then(() => operation(this.#controller.signal))
+            .then((value) => ({ kind: "completed", value }), (error) => ({ kind: "failed", error }));
+        const outcome = await Promise.race([
+            operationOutcome,
+            this.#lost.then((error) => ({ kind: "lease-lost", error })),
+        ]);
+        if (outcome.kind === "lease-lost") {
+            await operationOutcome;
+            throw outcome.error;
+        }
+        if (outcome.kind === "failed")
+            throw outcome.error;
+        this.assertOwned();
+        return outcome.value;
+    }
+    async stop() {
+        if (this.#timer)
+            clearInterval(this.#timer);
+        this.#timer = undefined;
+        await this.#renewing;
+    }
+    #tick() {
+        if (this.#renewing || this.#failure)
+            return;
+        const renewing = this.options.renew().catch((error) => {
+            const failure = error instanceof Error ? error : new Error("Lease renewal failed");
+            this.#failure = failure;
+            this.#controller.abort(failure);
+            this.#resolveLost(failure);
+            this.options.onError?.(failure);
+        });
+        this.#renewing = renewing.finally(() => {
+            this.#renewing = undefined;
+        });
+    }
+}
+//# sourceMappingURL=renewable-lease.js.map

package/dist/scheduled-task-runner.js

@@ -58,7 +58,14 @@ export class ScheduledTaskRunner {
         const active = this.#active;
         if (!active)
             return;
-        await Promise.race([active, wait(this.#stopTimeoutMs)]);
+        const drainedBeforeDeadline = await Promise.race([
+            active.then(() => true),
+            wait(this.#stopTimeoutMs).then(() => false),
+        ]);
+        if (!drainedBeforeDeadline) {
+            this.#logger.warn({ event: "scheduler.stop_deadline_exceeded" }, "Scheduled task runner is still draining after its shutdown deadline");
+            await active;
+        }
     }
     async runOnce() {
         if (this.#stopping)

package/dist/shard-supervisor.d.ts

@@ -9,6 +9,11 @@ export declare function supervisorShutdownExitCode(reason: string): number | und
 export declare function createStartupWorkerHeartbeatMaintenance(heartbeat: () => void, intervalMs?: number): {
     stop(): void;
 };
+export declare function relinquishRestartCircuitOwnership(input: {
+    shardId: number;
+    localLeases: Map<number, ShardLease>;
+    release(lease: ShardLease): Promise<boolean>;
+}): Promise<boolean>;
 export declare function deferScheduledRoutingFailures(route: () => Promise<ScheduledTaskExecutionResult>): Promise<ScheduledTaskExecutionResult>;
 export declare function startShardSupervisor(environment: BotEnvironment, processEnvironment: NodeJS.ProcessEnv, workerEntry: string): Promise<BotHandle>;
 export declare function splitNumericCommittedBatch(batch: NumericWriteBehindCommittedBatch, index: number): NumericWriteBehindCommittedBatch;

package/dist/shard-supervisor.js

@@ -57,6 +57,13 @@ export function createStartupWorkerHeartbeatMaintenance(heartbeat, intervalMs =
         stop: () => clearInterval(timer),
     };
 }
+export function relinquishRestartCircuitOwnership(input) {
+    const lease = input.localLeases.get(input.shardId);
+    if (!lease)
+        return Promise.resolve(false);
+    input.localLeases.delete(input.shardId);
+    return input.release(lease);
+}
 export async function deferScheduledRoutingFailures(route) {
     try {
         return await route();
@@ -120,6 +127,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
     const pendingNumericCommitted = new Map();
     const restartHistory = new Map();
     const restartCounts = new Map();
+    const openRestartCircuits = new Set();
     const workers = new Map();
     let sessionBudget;
     let databaseHealthy = false;
@@ -533,8 +541,9 @@ export async function startShardSupervisor(environment, processEnvironment, work
     }
     function installNumericCommitRouting(service) {
         for (const registration of numericRegistrations) {
-            service.subscribeCommitted(registration, async (batch) => {
+            service.subscribeCommitted(registration, async (batch, signal) => {
                 for (let index = 0; index < batch.mutations.length; index += 1) {
+                    signal.throwIfAborted();
                     const mutation = batch.mutations[index];
                     const result = batch.results[index];
                     const guildId = mutation?.identity.guild_id;
@@ -570,32 +579,53 @@ export async function startShardSupervisor(environment, processEnvironment, work
                         payload: encodeNumericIpcValue(committedItem),
                     };
                     assertShardMessageSize(notification);
-                    const response = await dispatchNumericCommitted(shardId, identity, notification);
+                    const response = await dispatchNumericCommitted(shardId, identity, notification, signal);
                     if (!response.ok)
                         throw new Error(`Committed numeric batch was not applied: ${response.safeCode}`);
                 }
             });
         }
     }
-    function dispatchNumericCommitted(shardId, identity, notification) {
+    function dispatchNumericCommitted(shardId, identity, notification, signal) {
         const worker = workers.get(shardId);
         if (!worker?.process)
             return Promise.reject(new Error("Committed numeric batch owner is unavailable"));
         return new Promise((resolve, reject) => {
+            const onAbort = () => {
+                const pending = pendingNumericCommitted.get(notification.requestId);
+                if (!pending)
+                    return;
+                pendingNumericCommitted.delete(notification.requestId);
+                clearTimeout(pending.timer);
+                pending.removeAbortListener();
+                reject(signal.reason instanceof Error
+                    ? signal.reason
+                    : new Error("Numeric claim ownership was lost"));
+            };
             const timer = setTimeout(() => {
+                const pending = pendingNumericCommitted.get(notification.requestId);
                 pendingNumericCommitted.delete(notification.requestId);
+                pending?.removeAbortListener();
                 reject(new Error("Committed numeric batch acknowledgement timed out"));
             }, SHARD_CONTROL_TIMEOUT_MS);
             timer.unref();
+            signal.addEventListener("abort", onAbort, { once: true });
             pendingNumericCommitted.set(notification.requestId, {
                 shardId,
                 identity,
                 resolve,
                 timer,
+                removeAbortListener: () => signal.removeEventListener("abort", onAbort),
             });
+            if (signal.aborted) {
+                onAbort();
+                return;
+            }
             void worker.send(notification).catch((error) => {
                 clearTimeout(timer);
+                const pending = pendingNumericCommitted.get(notification.requestId);
                 pendingNumericCommitted.delete(notification.requestId);
+                pending?.removeAbortListener();
                 reject(error instanceof Error
                     ? error
                     : new Error("Committed numeric delivery failed"));
@@ -720,6 +750,7 @@ export async function startShardSupervisor(environment, processEnvironment, work
                 return;
             pendingNumericCommitted.delete(message.data.requestId);
             clearTimeout(pending.timer);
+            pending.removeAbortListener();
             pending.resolve(message.data);
             return;
         }
@@ -909,6 +940,25 @@ export async function startShardSupervisor(environment, processEnvironment, work
         const now = Date.now();
         const recent = (restartHistory.get(shardId) ?? []).filter((time) => now - time < RESTART_WINDOW_MS);
         if (recent.length >= MAX_RESTARTS_PER_WINDOW) {
+            openRestartCircuits.add(shardId);
+            const circuitCoordinator = coordinator;
+            if (circuitCoordinator)
+                void relinquishRestartCircuitOwnership({
+                    shardId,
+                    localLeases,
+                    release: (lease) => circuitCoordinator.releaseShard(lease),
+                })
+                    .then((released) => {
+                    if (!released)
+                        logger.warn({ event: "shard.circuit_lease_release_stale", shardId }, "Shard restart circuit lease was already reassigned");
+                })
+                    .catch((error) => logger.warn({
+                    event: "shard.circuit_lease_release_failed",
+                    shardId,
+                    error: safeError(error),
+                }, "Shard restart circuit lease will expire without renewal"));
+            else
+                localLeases.delete(shardId);
             logger.error({ event: "shard.circuit_open", shardId, restartCount: recent.length }, "Shard worker restart circuit opened");
             return;
         }
@@ -998,7 +1048,9 @@ export async function startShardSupervisor(environment, processEnvironment, work
         try {
             const occupied = new Set((await withTimeout(coordinator.getShardLeases(shardCount), LEASE_VALIDATION_TIMEOUT_MS).catch(() => [])).map(({ shardId }) => shardId));
             const candidates = Array.from({ length: shardCount }, (_, shardId) => shardId)
-                .filter((shardId) => !localLeases.has(shardId) && !occupied.has(shardId))
+                .filter((shardId) => !localLeases.has(shardId) &&
+                !occupied.has(shardId) &&
+                !openRestartCircuits.has(shardId))
                 .slice(0, capacity);
             const results = await Promise.all(candidates.map(async (shardId) => {
                 const lease = await withTimeout(coordinator.claimShard(generationDescriptor, shardId, deploymentOrder), LEASE_VALIDATION_TIMEOUT_MS).catch(() => undefined);

package/dist/support-module.js

@@ -1,5 +1,6 @@
 import { defineModule, HELYX_SERVICE_NAMES, } from "@helyx/sdk";
 import { randomUUID } from "node:crypto";
+import { INTERNAL_MODULE_MANIFEST_FIELDS } from "./internal-module-manifest.js";
 const MODAL_ID = "helyx.support.bug";
 const MODULE_FIELD_ID = "helyx.support.bug.module";
 const SUMMARY_FIELD_ID = "helyx.support.bug.summary";
@@ -11,7 +12,7 @@ const HOUR_SECONDS = 60 * 60;
 export function createSupportModule(transport) {
     return defineModule({
         manifest: {
-            manifestVersion: "0.1",
+            ...INTERNAL_MODULE_MANIFEST_FIELDS,
             releaseStatus: "live",
             id: MODULE_ID,
             name: "Helyx Support",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helyx/bot",
-  "version": "0.4.35",
+  "version": "0.4.37",
   "description": "Production Discord runtime for the modular Helyx platform.",
   "keywords": [
     "helyx",