@helyx/bot 0.4.37 → 0.4.39

package/dist/guild-command-worker.js

@@ -0,0 +1,443 @@
+import { GuildCommandBudgetExceededError, GuildCommandRepository, InstallationRepository, ModuleAvailabilityRepository, } from "@helyx/database";
+import { randomUUID } from "node:crypto";
+import { diffDryRunGuildCommands, diffGuildCommands, projectGuildCommands, } from "./guild-command-projection.js";
+const DEFAULT_POLL_INTERVAL_MS = 1_000;
+const DEFAULT_LEASE_MS = 45_000;
+const DEFAULT_CLAIM_LIMIT = 4;
+const DEFAULT_CONCURRENCY = 2;
+const DEFAULT_STOP_TIMEOUT_MS = 5_000;
+export class GuildCommandWorker {
+    #repository;
+    #database;
+    #modules;
+    #logger;
+    #workerId;
+    #fencingToken;
+    #pollIntervalMs;
+    #leaseMs;
+    #claimLimit;
+    #concurrency;
+    #stopTimeoutMs;
+    #discord;
+    #internalGuildId;
+    #controller = new AbortController();
+    #active = new Set();
+    #timer;
+    #claiming;
+    #stopping = false;
+    constructor(input) {
+        if (input.fencingToken < 1n)
+            throw new Error("Guild command worker fencing token must be positive");
+        this.#database = input.database;
+        this.#repository = new GuildCommandRepository(input.database);
+        this.#modules = input.modules;
+        this.#logger = input.logger;
+        this.#workerId = input.workerId ?? `guild-command:${randomUUID()}`;
+        this.#fencingToken = input.fencingToken;
+        this.#discord = input.discord ?? null;
+        this.#internalGuildId = input.internalGuildId;
+        this.#pollIntervalMs = boundedInteger(input.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS, 100, 60_000, "poll interval");
+        this.#leaseMs = boundedInteger(input.leaseMs ?? DEFAULT_LEASE_MS, 5_000, 300_000, "lease duration");
+        this.#claimLimit = boundedInteger(input.claimLimit ?? DEFAULT_CLAIM_LIMIT, 1, 25, "claim limit");
+        this.#concurrency = boundedInteger(input.concurrency ?? DEFAULT_CONCURRENCY, 1, 8, "worker concurrency");
+        this.#stopTimeoutMs = boundedInteger(input.stopTimeoutMs ?? DEFAULT_STOP_TIMEOUT_MS, 100, 30_000, "stop timeout");
+    }
+    start() {
+        if (this.#timer || this.#stopping)
+            throw new Error("Guild command worker is already started or stopped");
+        this.#runSoon();
+        this.#timer = setInterval(() => this.#runSoon(), this.#pollIntervalMs);
+        this.#timer.unref();
+    }
+    async runOnce() {
+        if (this.#stopping)
+            return;
+        const capacity = Math.max(0, this.#concurrency - this.#active.size);
+        if (capacity === 0)
+            return;
+        const requests = await this.#repository.claimDue({
+            workerId: this.#workerId,
+            fencingToken: this.#fencingToken,
+            leaseMs: this.#leaseMs,
+            limit: Math.min(capacity, this.#claimLimit),
+        });
+        for (const request of requests) {
+            const task = this.#execute(request)
+                .catch((error) => {
+                this.#logger.error({
+                    event: "guild_commands.reconcile_failed",
+                    requestId: request.requestId,
+                    guildId: request.guildId,
+                    errorName: error instanceof Error ? error.name : "Error",
+                }, "Guild command reconciliation failed");
+            })
+                .finally(() => this.#active.delete(task));
+            this.#active.add(task);
+        }
+    }
+    async stop() {
+        if (!this.#stopping) {
+            this.#stopping = true;
+            if (this.#timer)
+                clearInterval(this.#timer);
+            this.#timer = undefined;
+            this.#controller.abort();
+        }
+        const active = [...this.#active];
+        if (this.#claiming)
+            active.push(this.#claiming);
+        if (active.length === 0)
+            return;
+        const drained = await Promise.race([
+            Promise.allSettled(active).then(() => true),
+            wait(this.#stopTimeoutMs).then(() => false),
+        ]);
+        if (!drained) {
+            this.#logger.warn({ event: "guild_commands.worker_stop_deadline_exceeded" }, "Guild command worker is still draining");
+        }
+    }
+    async getHealth() {
+        return {
+            ...(await this.#repository.getHealth()),
+            mode: this.#discord ? "live" : "dry_run",
+            discordWritesEnabled: Boolean(this.#discord),
+            workerRunning: Boolean(this.#timer) && !this.#stopping,
+            activeCount: this.#active.size,
+        };
+    }
+    #runSoon() {
+        if (this.#claiming || this.#stopping)
+            return;
+        this.#claiming = this.runOnce()
+            .catch((error) => {
+            this.#logger.error({
+                event: "guild_commands.claim_failed",
+                errorName: error instanceof Error ? error.name : "Error",
+            }, "Guild command claim failed");
+        })
+            .finally(() => {
+            this.#claiming = undefined;
+        });
+    }
+    async #execute(request) {
+        if (this.#controller.signal.aborted)
+            return;
+        let leaseAlive = true;
+        const renew = () => {
+            if (!leaseAlive || this.#controller.signal.aborted)
+                return;
+            void this.#repository
+                .heartbeat({
+                requestId: request.requestId,
+                workerId: this.#workerId,
+                leaseGeneration: request.leaseGeneration,
+                fencingToken: this.#fencingToken,
+                leaseMs: this.#leaseMs,
+            })
+                .then((alive) => {
+                leaseAlive = alive;
+            })
+                .catch(() => {
+                leaseAlive = false;
+            });
+        };
+        const heartbeat = setInterval(renew, Math.max(1_000, Math.floor(this.#leaseMs / 3)));
+        heartbeat.unref();
+        try {
+            const [enabled, availability, state] = await Promise.all([
+                new InstallationRepository(this.#database).listEnabledModules(request.guildId),
+                new ModuleAvailabilityRepository(this.#database).list(),
+                this.#repository.getState(request.applicationId, request.guildId),
+            ]);
+            if (!state || state.desiredRevision !== request.requestedRevision) {
+                await this.#repository.completeDryRun({
+                    request,
+                    workerId: this.#workerId,
+                    fencingToken: this.#fencingToken,
+                    definitionHash: "0".repeat(64),
+                    rootCount: 0,
+                    subcommandCount: 0,
+                    createdCount: 0,
+                    updatedCount: 0,
+                    removedCount: 0,
+                    noChange: false,
+                });
+                return;
+            }
+            const projection = projectGuildCommands({
+                modules: this.#modules,
+                enabledModuleIds: new Set(enabled.map(({ moduleId }) => moduleId)),
+                globallyDisabledModuleIds: new Set(availability
+                    .filter(({ globallyDisabled }) => globallyDisabled)
+                    .map(({ moduleId }) => moduleId)),
+                profile: state.visibilityProfile,
+                includeInternalCommands: request.guildId === this.#internalGuildId,
+            });
+            if (this.#discord?.canWrite(request.guildId) &&
+                leaseAlive &&
+                !this.#controller.signal.aborted) {
+                await this.#executeLive(request, projection);
+                return;
+            }
+            const diff = diffDryRunGuildCommands(projection, state.desiredDefinitionHash);
+            const outcome = await this.#completeDryRun(request, projection, diff);
+            const noChange = hasNoChanges(diff);
+            this.#logger.info({
+                event: noChange
+                    ? "guild_commands.dry_run_no_change"
+                    : "guild_commands.dry_run_completed",
+                requestId: request.requestId,
+                guildId: request.guildId,
+                revision: request.requestedRevision.toString(),
+                rootCount: projection.rootCount,
+                subcommandCount: projection.subcommandCount,
+                predictedCreates: diff.create.length,
+                noChange,
+                outcome,
+            }, noChange
+                ? "Guild command dry run found no changes"
+                : "Guild command dry run completed");
+        }
+        catch (error) {
+            if (!this.#controller.signal.aborted && leaseAlive) {
+                const safeCode = error instanceof GuildCommandLiveError
+                    ? error.safeCode
+                    : "projection_failed";
+                if (safeCode === "discord_observation_failed" ||
+                    safeCode === "discord_rate_limited" ||
+                    safeCode === "discord_write_ambiguous" ||
+                    safeCode === "discord_write_mismatch")
+                    await this.#repository.defer({
+                        request,
+                        workerId: this.#workerId,
+                        fencingToken: this.#fencingToken,
+                        delayMs: retryDelay(request.attemptCount),
+                        safeCode,
+                    });
+                else if (safeCode === "creation_budget_exhausted")
+                    await this.#repository.markBlocked({
+                        request,
+                        workerId: this.#workerId,
+                        fencingToken: this.#fencingToken,
+                        safeCode,
+                    });
+                else
+                    await this.#repository.markFailed({
+                        request,
+                        workerId: this.#workerId,
+                        fencingToken: this.#fencingToken,
+                        safeCode,
+                    });
+            }
+            throw error;
+        }
+        finally {
+            clearInterval(heartbeat);
+        }
+    }
+    async #completeDryRun(request, projection, diff) {
+        if (diff.conflicts.length > 0)
+            throw new Error("Guild command projection produced conflicts");
+        if (this.#controller.signal.aborted)
+            return "fenced";
+        return this.#repository.completeDryRun({
+            request,
+            workerId: this.#workerId,
+            fencingToken: this.#fencingToken,
+            definitionHash: projection.definitionHash,
+            rootCount: projection.rootCount,
+            subcommandCount: projection.subcommandCount,
+            createdCount: diff.create.length,
+            updatedCount: diff.update.length,
+            removedCount: diff.remove.length,
+            noChange: hasNoChanges(diff),
+        });
+    }
+    async #executeLive(request, projection) {
+        if (!this.#discord)
+            throw new GuildCommandLiveError("discord_write_disabled");
+        const result = await reconcileLiveGuildCommands({
+            repository: this.#repository,
+            discord: this.#discord,
+            request,
+            projection,
+            workerId: this.#workerId,
+            fencingToken: this.#fencingToken,
+            leaseMs: this.#leaseMs,
+            verificationDelayMs: 1_000,
+            signal: this.#controller.signal,
+        });
+        if (!result)
+            return;
+        this.#logger.info({
+            event: "guild_commands.live_reconciled",
+            requestId: request.requestId,
+            guildId: request.guildId,
+            revision: request.requestedRevision.toString(),
+            rootCount: projection.rootCount,
+            createdCount: result.diff.create.length,
+            updatedCount: result.diff.update.length,
+            removedCount: result.diff.remove.length,
+            outcome: result.outcome,
+        }, "Guild commands reconciled");
+    }
+}
+export async function reconcileLiveGuildCommands(input) {
+    let observed;
+    try {
+        observed = await input.discord.observe(input.request.guildId);
+    }
+    catch (error) {
+        throw new GuildCommandLiveError(isDiscordRateLimit(error)
+            ? "discord_rate_limited"
+            : "discord_observation_failed");
+    }
+    const diff = diffGuildCommands(input.projection, observed);
+    if (diff.conflicts.length > 0)
+        throw new GuildCommandLiveError("discord_command_conflict");
+    let ledgerId;
+    if (diff.create.length > 0) {
+        try {
+            ledgerId = (await input.repository.reserveCreations({
+                applicationId: input.request.applicationId,
+                guildId: input.request.guildId,
+                requestId: input.request.requestId,
+                correlationId: input.request.correlationId,
+                bucket: input.request.budgetClass,
+                commandCount: diff.create.length,
+                reservationMs: Math.min(300_000, Math.max(60_000, input.leaseMs * 2)),
+                reasonCode: "guild_command_create",
+            })).ledgerId;
+        }
+        catch (error) {
+            if (error instanceof GuildCommandBudgetExceededError)
+                throw new GuildCommandLiveError("creation_budget_exhausted");
+            throw error;
+        }
+    }
+    const current = await input.repository.isLeaseCurrent({
+        request: input.request,
+        workerId: input.workerId,
+        fencingToken: input.fencingToken,
+    });
+    if (!current || input.signal?.aborted) {
+        if (ledgerId)
+            await input.repository.settleCreation({
+                ledgerId,
+                state: "released",
+                observedReference: "write_not_started",
+            });
+        return null;
+    }
+    if (!hasNoChanges(diff)) {
+        let written;
+        try {
+            written = await input.discord.replace(input.request.guildId, input.projection.definitions);
+        }
+        catch {
+            try {
+                written = await input.discord.observe(input.request.guildId);
+            }
+            catch {
+                if (ledgerId)
+                    await input.repository.settleCreation({
+                        ledgerId,
+                        state: "committed",
+                        observedReference: `discord:guild:${input.request.guildId}:write_ambiguous`,
+                    });
+                throw new GuildCommandLiveError("discord_write_ambiguous");
+            }
+        }
+        let verifiedDiff = diffGuildCommands(input.projection, written);
+        if (!hasNoChanges(verifiedDiff)) {
+            try {
+                await waitForPropagation(input.verificationDelayMs ?? 0, input.signal);
+                written = await input.discord.observe(input.request.guildId);
+                verifiedDiff = diffGuildCommands(input.projection, written);
+            }
+            catch {
+                if (ledgerId)
+                    await input.repository.settleCreation({
+                        ledgerId,
+                        state: "committed",
+                        observedReference: `discord:guild:${input.request.guildId}:verification_ambiguous`,
+                    });
+                throw new GuildCommandLiveError("discord_write_ambiguous");
+            }
+        }
+        if (!hasNoChanges(verifiedDiff)) {
+            if (ledgerId)
+                await input.repository.settleCreation({
+                    ledgerId,
+                    state: "committed",
+                    observedReference: `discord:guild:${input.request.guildId}:roots:${written.length}:verification_mismatch`,
+                });
+            throw new GuildCommandLiveError("discord_write_mismatch");
+        }
+    }
+    const outcome = await input.repository.completeLive({
+        request: input.request,
+        workerId: input.workerId,
+        fencingToken: input.fencingToken,
+        definitionHash: input.projection.definitionHash,
+        rootCount: input.projection.rootCount,
+        subcommandCount: input.projection.subcommandCount,
+        observedDefinitionHash: input.projection.definitionHash,
+        installedRootCount: input.projection.rootCount,
+        createdCount: diff.create.length,
+        updatedCount: diff.update.length,
+        removedCount: diff.remove.length,
+        createdCommands: diff.create.map(({ name }) => `/${name}`),
+        updatedCommands: diff.update.map(({ name }) => `/${name}`),
+        removedCommands: diff.remove.map(({ name }) => `/${name}`),
+        ...(ledgerId ? { ledgerId } : {}),
+        observedReference: `discord:guild:${input.request.guildId}:roots:${input.projection.rootCount}`,
+    });
+    return { outcome, diff };
+}
+class GuildCommandLiveError extends Error {
+    safeCode;
+    constructor(safeCode) {
+        super(safeCode);
+        this.safeCode = safeCode;
+        this.name = "GuildCommandLiveError";
+    }
+}
+function hasNoChanges(diff) {
+    return (!diff.observationRequired &&
+        diff.create.length === 0 &&
+        diff.update.length === 0 &&
+        diff.remove.length === 0 &&
+        diff.conflicts.length === 0);
+}
+function retryDelay(attemptCount) {
+    return Math.min(60_000, 2_000 * 2 ** Math.max(0, attemptCount - 1));
+}
+function isDiscordRateLimit(error) {
+    if (!error || typeof error !== "object")
+        return false;
+    const candidate = error;
+    return candidate.status === 429 || candidate.statusCode === 429;
+}
+function boundedInteger(value, minimum, maximum, label) {
+    if (!Number.isSafeInteger(value) || value < minimum || value > maximum)
+        throw new Error(`${label} must be between ${minimum} and ${maximum}`);
+    return value;
+}
+function wait(milliseconds) {
+    return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+function waitForPropagation(milliseconds, signal) {
+    if (milliseconds <= 0 || signal?.aborted)
+        return Promise.resolve();
+    return new Promise((resolve) => {
+        const timeout = setTimeout(finish, milliseconds);
+        signal?.addEventListener("abort", finish, { once: true });
+        function finish() {
+            clearTimeout(timeout);
+            signal?.removeEventListener("abort", finish);
+            resolve();
+        }
+    });
+}
+//# sourceMappingURL=guild-command-worker.js.map

package/dist/guilds.d.ts

@@ -21,5 +21,5 @@ export interface GuildTrackingPreparation {
     health(): GuildReconciliationHealth;
     stop(): void;
 }
-export declare function installGuildTracking(client: Client, database: DatabasePool, modules: readonly ModuleDefinition[], logger: InteractionLogger, enabledByDefaultModuleIds?: ReadonlySet<string>, ownership?: GuildShardOwnership, reconciliationIntervalMs?: number, shardCount?: number): GuildTrackingPreparation;
+export declare function installGuildTracking(client: Client, database: DatabasePool, modules: readonly ModuleDefinition[], logger: InteractionLogger, enabledByDefaultModuleIds?: ReadonlySet<string>, ownership?: GuildShardOwnership, reconciliationIntervalMs?: number, shardCount?: number, applicationId?: string): GuildTrackingPreparation;
 //# sourceMappingURL=guilds.d.ts.map

package/dist/guilds.js

@@ -1,8 +1,8 @@
-import { InstallationRepository, } from "@helyx/database";
+import { InstallationRepository, GuildCommandRepository, withTransaction, } from "@helyx/database";
 import { Events } from "discord.js";
 const RECONCILIATION_PAGE_SIZE = 250;
 const DEFAULT_RECONCILIATION_INTERVAL_MS = 15 * 60 * 1_000;
-export function installGuildTracking(client, database, modules, logger, enabledByDefaultModuleIds = new Set(), ownership, reconciliationIntervalMs = DEFAULT_RECONCILIATION_INTERVAL_MS, shardCount) {
+export function installGuildTracking(client, database, modules, logger, enabledByDefaultModuleIds = new Set(), ownership, reconciliationIntervalMs = DEFAULT_RECONCILIATION_INTERVAL_MS, shardCount, applicationId) {
     if (!Number.isSafeInteger(reconciliationIntervalMs) ||
         reconciliationIntervalMs < 60_000)
         throw new Error("Guild reconciliation interval must be at least one minute");
@@ -19,33 +19,46 @@ export function installGuildTracking(client, database, modules, logger, enabledB
         rejectedFences: 0,
         unavailableGuilds: 0,
     };
-    async function syncGuild(guild) {
-        const record = await repository.syncGuild({
-            guildId: guild.id,
-            ownerUserId: guild.ownerId,
-            name: guild.name,
-            ...(ownership ? { ownership } : {}),
-        });
-        if (ownership &&
-            (record.shardGeneration !== ownership.generation ||
-                record.shardDeploymentOrder !== (ownership.deploymentOrder ?? 1n) ||
-                record.shardSupervisorId !== ownership.supervisorId ||
-                record.shardId !== ownership.shardId ||
-                record.shardFencingToken !== ownership.fencingToken))
-            throw new Error("Guild observation was rejected by a newer shard owner");
-        for (const module of modules) {
-            await repository.ensureModule({
+    async function syncGuild(guild, firstObservation = false) {
+        const persist = async (connection) => {
+            const installations = new InstallationRepository(connection);
+            const record = await installations.syncGuild({
                 guildId: guild.id,
-                moduleId: module.manifest.id,
-                moduleVersion: module.manifest.version,
-                configurationSchemaVersion: module.configuration?.schemaVersion ?? 1,
-                enabledBy: guild.ownerId,
-                enabledByDefault: enabledByDefaultModuleIds.has(module.manifest.id),
+                ownerUserId: guild.ownerId,
+                name: guild.name,
+                ...(ownership ? { ownership } : {}),
             });
-        }
+            if (ownership &&
+                (record.shardGeneration !== ownership.generation ||
+                    record.shardDeploymentOrder !== (ownership.deploymentOrder ?? 1n) ||
+                    record.shardSupervisorId !== ownership.supervisorId ||
+                    record.shardId !== ownership.shardId ||
+                    record.shardFencingToken !== ownership.fencingToken))
+                throw new Error("Guild observation was rejected by a newer shard owner");
+            if (firstObservation && applicationId)
+                await new GuildCommandRepository(connection).ensureState({
+                    applicationId,
+                    guildId: guild.id,
+                    setupState: "incomplete",
+                });
+            for (const module of modules) {
+                await installations.ensureModule({
+                    guildId: guild.id,
+                    moduleId: module.manifest.id,
+                    moduleVersion: module.manifest.version,
+                    configurationSchemaVersion: module.configuration?.schemaVersion ?? 1,
+                    enabledBy: guild.ownerId,
+                    enabledByDefault: enabledByDefaultModuleIds.has(module.manifest.id),
+                });
+            }
+        };
+        if (firstObservation)
+            await withTransaction(database, persist);
+        else
+            await persist(database);
     }
     client.on(Events.GuildCreate, (guild) => {
-        void syncGuild(guild).catch((error) => {
+        void syncGuild(guild, true).catch((error) => {
             logger.error({
                 event: "guild.sync_failed",
                 guildId: guild.id,
@@ -56,7 +69,9 @@ export function installGuildTracking(client, database, modules, logger, enabledB
     client.on(Events.GuildDelete, (guild) => {
         void repository
             .markGuildRemoved(guild.id, ownership)
-            .then((changed) => {
+            .then(async (changed) => {
+            if (changed && applicationId)
+                await new GuildCommandRepository(database).cancelGuildWork(applicationId, guild.id);
             logger.info({
                 event: "guild.removed",
                 outcome: changed ? "changed" : "rejected_or_duplicate",