@helyx/bot 0.4.11 → 0.4.13

package/dist/shard-supervisor.js

@@ -1,5 +1,5 @@
 import { safeError, ServiceRegistry } from "@helyx/core";
-import { checkDatabaseConnection, createDatabasePool, InteractionSessionRepository, RateLimitRepository, runConfiguredModuleMigrations, runCoreMigrations, RuntimeStateRepository, } from "@helyx/database";
+import { checkDatabaseConnection, createDatabasePool, AuditRepository, InteractionSessionRepository, RateLimitRepository, runConfiguredModuleMigrations, runCoreMigrations, RuntimeStateRepository, } from "@helyx/database";
 import { createPlatformServices, PostgresPrivacyProviderExecutor, } from "@helyx/platform";
 import { HELYX_SERVICE_NAMES } from "@helyx/sdk";
 import { Client, ShardingManager } from "discord.js";
@@ -29,12 +29,25 @@ import { encodeNumericIpcValue } from "./numeric-write-behind-proxy.js";
 const LEASE_RENEW_INTERVAL_MS = 10_000;
 const STARTUP_LEASE_RENEW_INTERVAL_MS = 5_000;
 const LEASE_VALIDATION_TIMEOUT_MS = 1_000;
+const LEASE_RENEW_ATTEMPTS = 2;
 const RETENTION_CLEANUP_INTERVAL_MS = 60 * 60 * 1_000;
 const STARTUP_WAIT_MS = 60_000;
 const WORKER_SHUTDOWN_MS = 5_000;
 const MAX_RESTARTS_PER_WINDOW = 5;
 const RESTART_WINDOW_MS = 10 * 60 * 1_000;
-export const STEADY_STATE_LEASE_LOSS_BOUND_MS = LEASE_RENEW_INTERVAL_MS + LEASE_VALIDATION_TIMEOUT_MS + WORKER_SHUTDOWN_MS;
+export const STEADY_STATE_LEASE_LOSS_BOUND_MS = LEASE_RENEW_INTERVAL_MS +
+    LEASE_VALIDATION_TIMEOUT_MS * LEASE_RENEW_ATTEMPTS +
+    WORKER_SHUTDOWN_MS;
+const FATAL_SUPERVISOR_SHUTDOWN_REASONS = new Set([
+    "generation-lease-lost",
+    "leader-lease-lost",
+    "leader-promotion-failed",
+    "rebalance-failed",
+    "coordination-unavailable",
+]);
+export function supervisorShutdownExitCode(reason) {
+    return FATAL_SUPERVISOR_SHUTDOWN_REASONS.has(reason) ? 1 : undefined;
+}
 export function createStartupWorkerHeartbeatMaintenance(heartbeat, intervalMs = STARTUP_LEASE_RENEW_INTERVAL_MS) {
     const timer = setInterval(heartbeat, intervalMs);
     timer.unref();
@@ -150,6 +163,11 @@ export async function startShardSupervisor(environment, processEnvironment, work
         });
         const commandClient = new Client({ intents: [] });
         const unavailableResources = {
+            listGuildPresence: (guildIds) => Promise.resolve(guildIds.map((guildId) => ({
+                guildId,
+                state: "unknown",
+                observedAt: new Date().toISOString(),
+            }))),
             getGuildResources: () => Promise.reject(new Error("Guild resources are worker-owned")),
             sendToChannel: () => Promise.reject(new Error("Guild resources are worker-owned")),
             deleteChannelMessage: () => Promise.reject(new Error("Guild resources are worker-owned")),
@@ -883,8 +901,19 @@ export async function startShardSupervisor(environment, processEnvironment, work
         try {
             if (!coordinator)
                 return;
-            const renewal = await renewSteadyStateOwnership(coordinator, generationDescriptor, deploymentOrder, localLeases, leaderLease).catch(() => undefined);
-            if (!renewal?.generationRenewed) {
+            let renewal;
+            try {
+                renewal = await retryOwnershipRenewal(() => renewSteadyStateOwnership(coordinator, generationDescriptor, deploymentOrder, localLeases, leaderLease));
+            }
+            catch (error) {
+                logger.error({
+                    event: "shard.coordination_renewal_failed",
+                    error: safeError(error),
+                }, "Shard coordination could not be renewed after a bounded retry");
+                await shutdown("coordination-unavailable");
+                return;
+            }
+            if (!renewal.generationRenewed) {
                 logger.error({ event: "shard.generation_lease_lost" }, "Shard generation lease was lost; stopping this supervisor safely");
                 await shutdown("generation-lease-lost");
                 return;
@@ -1164,6 +1193,9 @@ export async function startShardSupervisor(environment, processEnvironment, work
         if (shuttingDown)
             return;
         shuttingDown = true;
+        const shutdownExitCode = supervisorShutdownExitCode(reason);
+        if (shutdownExitCode !== undefined)
+            process.exitCode = shutdownExitCode;
         if (leaseTimer)
             clearInterval(leaseTimer);
         if (startupLeaseTimer)
@@ -1450,15 +1482,17 @@ function failedCloud(requestId, message) {
 }
 async function cleanupExpiredCoreRecords(database, logger) {
     try {
-        const [sessions, rateLimits] = await Promise.all([
+        const [sessions, rateLimits, verificationAudit] = await Promise.all([
             new InteractionSessionRepository(database).deleteExpired(),
             new RateLimitRepository(database).deleteExpired(),
+            new AuditRepository(database).deleteModuleEventsOlderThanMonths("helyx.verification", 12),
         ]);
-        if (sessions > 0 || rateLimits > 0)
+        if (sessions > 0 || rateLimits > 0 || verificationAudit > 0)
             logger.info({
                 event: "database.retention_cleaned",
                 interactionSessions: sessions,
                 rateLimitWindows: rateLimits,
+                verificationAuditEvents: verificationAudit,
             }, "Expired core records removed");
     }
     catch (error) {
@@ -1509,6 +1543,20 @@ async function waitForDeath(shard, timeoutMs) {
 export function boundedRespawnDelay(attempt) {
     return Math.min(30_000, 1_000 * 2 ** Math.max(0, attempt - 1));
 }
+export async function retryOwnershipRenewal(renew, attempts = LEASE_RENEW_ATTEMPTS) {
+    if (!Number.isSafeInteger(attempts) || attempts < 1)
+        throw new Error("Ownership renewal attempts must be a positive integer");
+    let lastError;
+    for (let attempt = 0; attempt < attempts; attempt += 1) {
+        try {
+            return await renew();
+        }
+        catch (error) {
+            lastError = error;
+        }
+    }
+    throw lastError;
+}
 export function selectFencedShardHealth(input) {
     const liveByShard = new Map(input.liveLeases.map((lease) => [lease.shardId, lease]));
     const selected = new Map();

package/dist/shard-worker.d.ts

@@ -35,7 +35,10 @@ export declare function parseShardWorkerEnvironment(processEnvironment: NodeJS.P
     HELYX_MODULE_PACKAGES: string[];
     HELYX_MODULE_CATALOGUE_PATH: string | undefined;
     HELYX_SUPPORT_STAFF_CHANNEL_ID: string | undefined;
+    HELYX_VERIFICATION_ENABLED: boolean;
+    HELYX_VERIFICATION_SIGNING_PUBLIC_KEYS: Record<string, string>;
     HELYX_BOT_CONTROL_PORT: number;
+    HELYX_GUILD_RECONCILIATION_INTERVAL_SECONDS: number;
     HELYX_PRIVACY_ENABLED: boolean;
     HELYX_MESSAGE_CONTENT_ENABLED: boolean;
     HELYX_CLOUD_CONNECT_ENABLED: boolean;
@@ -52,6 +55,9 @@ export declare function parseShardWorkerEnvironment(processEnvironment: NodeJS.P
     NODE_ENV: "development" | "test" | "production";
     HELYX_BOT_CONTROL_SECRET?: string | undefined;
     HELYX_SUPPORT_SERVICE_SECRET?: string | undefined;
+    HELYX_VERIFICATION_INSTALLATION_ID?: string | undefined;
+    HELYX_VERIFICATION_URL?: string | undefined;
+    HELYX_VERIFICATION_INTERNAL_SECRET?: string | undefined;
     HELYX_PRIVACY_EXECUTOR_DATABASE_URL?: string | undefined;
     HELYX_PRIVACY_EXECUTOR_DATABASE_SSL?: "disable" | "require" | "verify-full" | undefined;
     HELYX_SHARD_COUNT?: number | undefined;

package/dist/shard-worker.js

@@ -1,6 +1,6 @@
 import { parseBotEnvironment } from "@helyx/config";
 import { safeError, ServiceRegistry } from "@helyx/core";
-import { checkDatabaseConnection, createDatabasePool } from "@helyx/database";
+import { checkDatabaseConnection, createDatabasePool, VerificationSetupRepository, } from "@helyx/database";
 import { createRuntime } from "@helyx/framework";
 import { createPlatformServices } from "@helyx/platform";
 import { HELYX_SERVICE_NAMES } from "@helyx/sdk";
@@ -9,8 +9,9 @@ import { monitorEventLoopDelay } from "node:perf_hooks";
 import pino from "pino";
 import { createCloudManagementModule } from "./cloud-management-module.js";
 import { BotControlService, DiscordGuildResourceDirectory, mapBotControlError, } from "./control-server.js";
-import { DiscordMessageDeliveryService, DiscordMemberEnforcementService, DiscordMemberRoleService, DiscordResourceInspection, installModuleEvents, } from "./events.js";
-import { installGuildTracking } from "./guilds.js";
+import { HttpVerificationService } from "./verification-service.js";
+import { DiscordMessageDeliveryService, DiscordMemberEnforcementService, DiscordMemberRoleService, DiscordVerificationMemberService, DiscordVerificationSetupService, DiscordResourceInspection, installModuleEvents, } from "./events.js";
+import { installGuildTracking, } from "./guilds.js";
 import { installInteractions, } from "./interactions.js";
 import { createManagementModule } from "./management-module.js";
 import { DiscordModuleLoggingService } from "./module-logging.js";
@@ -73,6 +74,7 @@ export async function startShardWorker(processEnvironment = process.env, identit
     let shuttingDown = false;
     let runtimeStarted = false;
     let moduleEvents;
+    let guildTracking;
     let client;
     let cloudProxy;
     let shardHealthProvider;
@@ -84,6 +86,7 @@ export async function startShardWorker(processEnvironment = process.env, identit
             return;
         shuttingDown = true;
         status = "stopped";
+        guildTracking?.stop();
         if (healthTimer)
             clearInterval(healthTimer);
         leaseHeartbeatWatchdog?.stop();
@@ -168,6 +171,18 @@ export async function startShardWorker(processEnvironment = process.env, identit
         })) {
             platformServices.set(name, service);
         }
+        if (environment.HELYX_VERIFICATION_ENABLED &&
+            environment.HELYX_VERIFICATION_URL &&
+            environment.HELYX_VERIFICATION_INTERNAL_SECRET) {
+            platformServices.set(HELYX_SERVICE_NAMES.verification, new HttpVerificationService({
+                baseUrl: environment.HELYX_VERIFICATION_URL,
+                internalSecret: environment.HELYX_VERIFICATION_INTERNAL_SECRET,
+                installationId: environment.HELYX_VERIFICATION_INSTALLATION_ID,
+                publicSigningKeys: environment.HELYX_VERIFICATION_SIGNING_PUBLIC_KEYS,
+            }));
+            platformServices.set(HELYX_SERVICE_NAMES.verificationMembers, new DiscordVerificationMemberService(client));
+            platformServices.set(HELYX_SERVICE_NAMES.verificationSetup, new DiscordVerificationSetupService(client, new VerificationSetupRepository(database)));
+        }
         numericWriteBehind =
             identity.scope === "coordinated"
                 ? new WorkerNumericWriteBehindProxy(identity)
@@ -194,13 +209,15 @@ export async function startShardWorker(processEnvironment = process.env, identit
             ...(cloudManagementModule ? [cloudManagementModule.manifest.id] : []),
             ...(operationsModule ? [operationsModule.manifest.id] : []),
         ]);
-        const prepareGuilds = installGuildTracking(client, database, publicModules, logger, enabledCoreModuleIds, {
+        const shardOwnership = {
             generation: identity.generation,
             deploymentOrder: BigInt(identity.deploymentOrder ?? 1),
             supervisorId: identity.supervisorId,
             shardId: identity.shardId,
             fencingToken: BigInt(identity.fencingToken),
-        });
+        };
+        const prepareGuilds = installGuildTracking(client, database, publicModules, logger, enabledCoreModuleIds, shardOwnership, environment.HELYX_GUILD_RECONCILIATION_INTERVAL_SECONDS * 1_000, identity.shardCount);
+        guildTracking = prepareGuilds;
         const commandsReady = installInteractions(client, modules, shardWorkerCommandRegistration({
             ...(environment.DISCORD_INTERNAL_GUILD_ID
                 ? { internalGuildId: environment.DISCORD_INTERNAL_GUILD_ID }
@@ -218,7 +235,7 @@ export async function startShardWorker(processEnvironment = process.env, identit
                 guildId: environment.DISCORD_INTERNAL_GUILD_ID,
                 channelId: environment.HELYX_SUPPORT_STAFF_CHANNEL_ID,
             }
-            : undefined);
+            : undefined, shardOwnership, identity.shardCount);
         process.on("message", (raw) => {
             const parsed = supervisorMessageSchema.safeParse(raw);
             if (!parsed.success)

package/dist/verification-service.d.ts

@@ -0,0 +1,50 @@
+import { type ActionReceipt, type ChallengeCreateRequest, type ChallengeCreateResponse, type DashboardAttemptPage, type DashboardAttemptQuery, type DashboardSafeAttemptDetail, type SignedAssessmentEnvelope, type VerificationInstallationClaims } from "@helyx/verification-contracts";
+import type { VerificationResultContext, VerificationService } from "@helyx/sdk";
+export interface HttpVerificationServiceOptions {
+    baseUrl: string;
+    internalSecret: string;
+    publicSigningKeys: Readonly<Record<string, string>>;
+    installationId: string;
+    fetchImplementation?: typeof fetch;
+}
+export declare class HttpVerificationService implements VerificationService {
+    #private;
+    constructor(options: HttpVerificationServiceOptions);
+    getInstallationContext(): Promise<{
+        installation: VerificationInstallationClaims;
+    }>;
+    createChallenge(input: ChallengeCreateRequest): Promise<ChallengeCreateResponse>;
+    cancelChallenges(input: {
+        installationId: string;
+        guildId: string;
+        userId?: string;
+    }): Promise<{
+        cancelled: number;
+    }>;
+    listAttempts(input: DashboardAttemptQuery & {
+        installationId: string;
+        guildId: string;
+    }): Promise<DashboardAttemptPage>;
+    getAttempt(input: {
+        installationId: string;
+        guildId: string;
+        attemptId: string;
+    }): Promise<DashboardSafeAttemptDetail | null>;
+    decideAttempt(input: {
+        installationId: string;
+        guildId: string;
+        attemptId: string;
+        expectedVersion: number;
+        actorUserId: string;
+        decision: "approve" | "deny" | "retry" | "dismiss_relationship";
+        reason: string;
+    }): Promise<DashboardSafeAttemptDetail>;
+    registerResultHandler(handler: (context: VerificationResultContext) => Promise<ActionReceipt>): () => void;
+    acceptResult(envelopeInput: SignedAssessmentEnvelope): Promise<ActionReceipt>;
+    health(): Promise<{
+        status: "healthy" | "degraded" | "unavailable";
+        provider: "available" | "degraded" | "unavailable";
+        allowanceRemaining?: number;
+    }>;
+}
+//# sourceMappingURL=verification-service.d.ts.map

package/dist/verification-service.js

@@ -0,0 +1,166 @@
+import { actionReceiptSchema, challengeCreateRequestSchema, challengeCreateResponseSchema, dashboardAttemptPageSchema, dashboardAttemptQuerySchema, dashboardSafeAttemptDetailSchema, signedAssessmentEnvelopeSchema, } from "@helyx/verification-contracts";
+import { createPublicKey, verify } from "node:crypto";
+import { z } from "zod";
+const healthSchema = z
+    .object({
+    status: z.enum(["healthy", "degraded", "unavailable"]),
+    provider: z.enum(["available", "degraded", "unavailable"]),
+    allowanceRemaining: z.number().int().min(0).optional(),
+})
+    .strict();
+export class HttpVerificationService {
+    #baseUrl;
+    #internalSecret;
+    #publicSigningKeys;
+    #fetch;
+    #installationId;
+    #handler;
+    constructor(options) {
+        this.#baseUrl = options.baseUrl.replace(/\/$/u, "");
+        this.#internalSecret = options.internalSecret;
+        this.#publicSigningKeys = { ...options.publicSigningKeys };
+        this.#installationId = options.installationId;
+        this.#fetch = options.fetchImplementation ?? fetch;
+    }
+    getInstallationContext() {
+        return Promise.resolve({
+            installation: {
+                installationId: this.#installationId,
+                sourceClassification: "helyx_hosted",
+                capabilities: [
+                    "challenge.create.v1",
+                    "assessment.receive.v1",
+                    "action.receipt.write.v1",
+                    "member.current.read.v1",
+                    "ban.current.read.v1",
+                    "service.health.read.v1",
+                ],
+                authenticatedAt: new Date().toISOString(),
+            },
+        });
+    }
+    async createChallenge(input) {
+        return challengeCreateResponseSchema.parse(await this.#request("/internal/v1/challenges", {
+            method: "POST",
+            body: challengeCreateRequestSchema.parse(input),
+        }));
+    }
+    async cancelChallenges(input) {
+        return z
+            .object({ cancelled: z.number().int().min(0) })
+            .strict()
+            .parse(await this.#request("/internal/v1/challenges/cancel", {
+            method: "POST",
+            body: input,
+        }));
+    }
+    async listAttempts(input) {
+        const { installationId, guildId, ...query } = input;
+        return dashboardAttemptPageSchema.parse(await this.#request("/internal/v1/attempts/query", {
+            method: "POST",
+            body: {
+                installationId,
+                guildId,
+                query: dashboardAttemptQuerySchema.parse(query),
+            },
+        }));
+    }
+    async getAttempt(input) {
+        const result = await this.#request(`/internal/v1/attempts/${encodeURIComponent(input.attemptId)}`, { method: "POST", body: input }, true);
+        return result === null
+            ? null
+            : dashboardSafeAttemptDetailSchema.parse(result);
+    }
+    async decideAttempt(input) {
+        return dashboardSafeAttemptDetailSchema.parse(await this.#request(`/internal/v1/attempts/${encodeURIComponent(input.attemptId)}/decision`, { method: "POST", body: input }));
+    }
+    registerResultHandler(handler) {
+        if (this.#handler !== undefined)
+            throw new Error("Verification result handler is already registered");
+        this.#handler = handler;
+        return () => {
+            if (this.#handler === handler)
+                this.#handler = undefined;
+        };
+    }
+    async acceptResult(envelopeInput) {
+        const envelope = signedAssessmentEnvelopeSchema.parse(envelopeInput);
+        if (envelope.payload.deploymentSourceClassification !== "helyx_hosted" ||
+            envelope.payload.evidenceSources.discord !== "helyx_hosted" ||
+            envelope.payload.installationId !== this.#installationId)
+            throw new Error("Verification assessment is not bound to hosted Helyx");
+        const publicKey = this.#publicSigningKeys[envelope.keyId];
+        if (publicKey === undefined || !verifyEnvelope(envelope, publicKey))
+            throw new Error("Verification assessment signature is invalid");
+        const expiresAt = Date.parse(envelope.payload.expiresAt);
+        const issuedAt = Date.parse(envelope.payload.issuedAt);
+        const now = Date.now();
+        if (issuedAt > now + 30_000 || expiresAt <= now)
+            throw new Error("Verification assessment is outside its valid lifetime");
+        if (!this.#handler)
+            throw new Error("Verification result handler is unavailable");
+        return actionReceiptSchema.parse(await this.#handler({
+            envelope,
+            correlationId: envelope.payload.attemptId,
+        }));
+    }
+    async health() {
+        const health = healthSchema.parse(await this.#request("/internal/v1/health", { method: "GET" }));
+        return {
+            status: health.status,
+            provider: health.provider,
+            ...(health.allowanceRemaining === undefined
+                ? {}
+                : { allowanceRemaining: health.allowanceRemaining }),
+        };
+    }
+    async #request(path, input, allowNotFound = false) {
+        let response;
+        try {
+            response = await this.#fetch(`${this.#baseUrl}${path}`, {
+                method: input.method,
+                headers: {
+                    authorization: `Bearer ${this.#internalSecret}`,
+                    accept: "application/json",
+                    ...(input.body === undefined
+                        ? {}
+                        : { "content-type": "application/json" }),
+                },
+                ...(input.body === undefined
+                    ? {}
+                    : { body: JSON.stringify(input.body) }),
+                signal: AbortSignal.timeout(10_000),
+            });
+        }
+        catch {
+            throw new Error("Verification service is unavailable");
+        }
+        if (allowNotFound && response.status === 404)
+            return null;
+        if (!response.ok)
+            throw new Error(response.status >= 500
+                ? "Verification service is unavailable"
+                : "Verification request was rejected");
+        try {
+            return await response.json();
+        }
+        catch {
+            throw new Error("Verification service returned an invalid response");
+        }
+    }
+}
+function verifyEnvelope(envelope, publicKeySpkiBase64) {
+    let key;
+    try {
+        key = createPublicKey({
+            key: Buffer.from(publicKeySpkiBase64, "base64"),
+            format: "der",
+            type: "spki",
+        });
+    }
+    catch {
+        return false;
+    }
+    return verify(null, Buffer.from(JSON.stringify(envelope.payload), "utf8"), key, Buffer.from(envelope.signature, "base64url"));
+}
+//# sourceMappingURL=verification-service.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helyx/bot",
-  "version": "0.4.11",
+  "version": "0.4.13",
   "description": "Production Discord runtime for the modular Helyx platform.",
   "keywords": [
     "helyx",
@@ -48,14 +48,15 @@
     "start": "node --env-file-if-exists=../../.env dist/index.js"
   },
   "dependencies": {
-    "@helyx/config": "0.4.3",
-    "@helyx/control-protocol": "0.2.0",
-    "@helyx/core": "0.1.11",
-    "@helyx/database": "0.5.0",
-    "@helyx/framework": "0.1.11",
-    "@helyx/module-manifest": "0.5.0",
-    "@helyx/platform": "0.3.0",
-    "@helyx/sdk": "0.4.0",
+    "@helyx/config": "0.5.0",
+    "@helyx/control-protocol": "0.3.0",
+    "@helyx/core": "0.1.12",
+    "@helyx/database": "0.6.0",
+    "@helyx/framework": "0.1.12",
+    "@helyx/module-manifest": "0.6.0",
+    "@helyx/platform": "0.4.0",
+    "@helyx/sdk": "0.5.0",
+    "@helyx/verification-contracts": "0.1.0",
     "discord.js": "14.27.0",
     "fastify": "5.10.0",
     "pino": "10.3.1",