@neontechspace/fluxerly 1000.0.0-nightly.12

package/src/internal/scheduled-tasks.ts

@@ -0,0 +1,217 @@
+/**
+ * Scheduled tasks: Application work that client.schedule runs once or repeatedly within the client's lifetime.
+ * Invariant: Every wait uses the client's logical scheduler, and each task runs in a fiber of its own, never in an event
+ * handler or command slot. A failed run reaches the client failure reporter, and a repeating task keeps its schedule. A
+ * draining shutdown starts no new run and waits for running ones, and shutdown interrupts the rest, awaits their cleanup
+ * and records in one summary how many tasks it ended with a run waiting or in progress. Implements [SDK contracts: User-handler failures](/docs/SDK-CONTRACTS.md#user-handler-failures)
+ */
+import * as Cause from "effect/Cause"
+import type * as Context from "effect/Context"
+import * as Deferred from "effect/Deferred"
+import * as Effect from "effect/Effect"
+import * as Exit from "effect/Exit"
+import * as Fiber from "effect/Fiber"
+import { ClientClosedError, ConfigurationError } from "#sdk/errors"
+import { record } from "./decode/primitives.js"
+import type { FailureReporter } from "./failures.js"
+import type { LogicalScheduler } from "./logical-scheduler.js"
+import type { ClientLogger } from "./logging.js"
+import { unsupportedKeyHint } from "./suggest.js"
+
+/** Longest delay or interval, the longest single timer delay */
+const maximumMs = 2_147_483_647
+const optionKeys = ["delayMs", "intervalMs"]
+
+/** When a task first runs, and for a repeating task how long it waits after each run ends */
+export interface ScheduleTiming {
+    readonly delayMs: number
+    readonly intervalMs: number | undefined
+}
+
+function milliseconds(value: unknown, minimum: number): boolean {
+    return typeof value === "number" && Number.isSafeInteger(value) && value >= minimum && value <= maximumMs
+}
+
+/**
+ * Read client.schedule options. Misuse returns ConfigurationError without the rejected value.
+ * Reading the options is application input, so the caller runs this under a defect boundary
+ */
+export function scheduleTiming(options: unknown): ScheduleTiming | ConfigurationError {
+    if (options === undefined) return { delayMs: 0, intervalMs: undefined }
+    if (!record(options)) return new ConfigurationError("configuration", "Schedule options must be an object")
+    const unsupported = Object.keys(options).find((key) => !optionKeys.includes(key))
+    if (unsupported !== undefined)
+        return new ConfigurationError("configuration", `Unsupported schedule option ${JSON.stringify(unsupported)}`, {
+            hint: unsupportedKeyHint(unsupported, optionKeys),
+        })
+    const { delayMs, intervalMs } = options
+    if (delayMs !== undefined && !milliseconds(delayMs, 0))
+        return new ConfigurationError(
+            "delayMs",
+            'The option "delayMs" must be an integer from 0 through 2,147,483,647 milliseconds',
+        )
+    if (intervalMs !== undefined && !milliseconds(intervalMs, 1))
+        return new ConfigurationError(
+            "intervalMs",
+            'The option "intervalMs" must be an integer from 1 through 2,147,483,647 milliseconds',
+        )
+    return { delayMs: (delayMs ?? intervalMs ?? 0) as number, intervalMs: intervalMs as number | undefined }
+}
+
+/** The client's scheduled tasks. Each one is a fiber that waits on the logical scheduler and then runs its work */
+export class ScheduledTasks {
+    /** Each task's fiber, with whether the application closed it and whether a run is in progress */
+    readonly #tasks = new Map<Fiber.Fiber<void>, { closed: boolean; running: boolean }>()
+    /** Task runs in progress, which a draining shutdown waits for */
+    #running = 0
+    /** Released and replaced whenever a run ends, so a drain can wait for progress */
+    #progress = Deferred.makeUnsafe<void>()
+    /** Set when shutdown begins, draining or not: No task starts another run and no task can be added */
+    #closing = false
+    /** Tasks that ended during a draining shutdown with runs still to come, reported when the tasks stop */
+    #ended = 0
+    #stopped = false
+
+    constructor(
+        private readonly logical: LogicalScheduler,
+        private readonly failures: FailureReporter,
+        private readonly logging: ClientLogger,
+    ) {}
+
+    /** Task runs in progress */
+    get running(): number {
+        return this.#running
+    }
+
+    /** Wait until the next run ends */
+    progress(): Effect.Effect<void> {
+        return Effect.suspend(() => Deferred.await(this.#progress))
+    }
+
+    /** Whether the fiber runs a task, so a task that requests shutdown does not wait for itself */
+    owns(fiberId: number): boolean {
+        for (const fiber of this.#tasks.keys()) if (fiber.id === fiberId) return true
+        return false
+    }
+
+    /**
+     * Start a task in a fiber of its own with the given services. It first runs after timing.delayMs and, when it repeats,
+     * again intervalMs after each run ends. Returns a close function that cancels later runs without interrupting a run
+     * in progress. Throws ClientClosedError once shutdown has begun
+     */
+    start(run: Effect.Effect<unknown, unknown>, timing: ScheduleTiming, context: Context.Context<never>): () => void {
+        if (this.#closing) throw new ClientClosedError()
+        const owner = this
+        const state = { closed: false, running: false }
+        const wait = (delayMs: number) =>
+            Effect.callback<void>((resume) => {
+                const timer = owner.logical.set(() => resume(Effect.void), delayMs, "scheduled task")
+                return Effect.sync(() => owner.logical.clear(timer))
+            })
+        // A failed run is reported and the task continues. Interruption ends the task, after reporting any defect with it
+        const once = Effect.uninterruptibleMask((restore) => {
+            state.running = true
+            owner.#running++
+            return restore(run).pipe(
+                Effect.catchCause((cause) => {
+                    const report = Cause.hasInterruptsOnly(cause)
+                        ? Effect.void
+                        : Effect.withFiber((fiber) =>
+                              Effect.sync(() => owner.failures.report({ kind: "task", cause }, fiber.context)),
+                          )
+                    return Cause.hasInterrupts(cause) ? report.pipe(Effect.andThen(Effect.interrupt)) : report
+                }),
+                Effect.ensuring(
+                    Effect.sync(() => {
+                        state.running = false
+                        owner.#running--
+                        owner.#signalProgress()
+                    }),
+                ),
+            )
+        })
+        const task = Effect.gen(function* () {
+            let delayMs = timing.delayMs
+            while (true) {
+                yield* wait(delayMs)
+                if (state.closed) return
+                // During a draining shutdown a due run never starts, and stop reports the task as cancelled
+                if (owner.#closing) {
+                    owner.#ended++
+                    return
+                }
+                yield* once
+                if (timing.intervalMs === undefined || state.closed) return
+                // Likewise a repeating task whose run ended during the drain gets no later run
+                if (owner.#closing) {
+                    owner.#ended++
+                    return
+                }
+                delayMs = timing.intervalMs
+            }
+        })
+        const fiber: Fiber.Fiber<void> = Effect.runForkWith(context)(
+            task.pipe(Effect.ensuring(Effect.sync(() => void owner.#tasks.delete(fiber)))),
+        )
+        this.#tasks.set(fiber, state)
+        return () => {
+            if (state.closed) return
+            state.closed = true
+            // A run in progress finishes first, and the task then ends instead of waiting for another run
+            if (!state.running) fiber.interruptUnsafe()
+        }
+    }
+
+    /** Start no new run and accept no new task, while runs in progress continue. A draining shutdown begins with this */
+    seal(): void {
+        this.#closing = true
+    }
+
+    /** Stop every task and wait until their fibers end. Fails only with defects raised while they stopped */
+    shutdown(): Effect.Effect<void> {
+        return Effect.suspend(() => {
+            this.stop()
+            return Effect.forEach([...this.#tasks.keys()], (fiber) => Fiber.await(fiber), {
+                concurrency: "unbounded",
+            }).pipe(
+                Effect.flatMap((exits) => {
+                    const reasons = exits.flatMap((exit) =>
+                        Exit.isFailure(exit) ? exit.cause.reasons.filter((reason) => reason._tag === "Die") : [],
+                    )
+                    return reasons.length ? Effect.failCause(Cause.fromReasons<never>(reasons)) : Effect.void
+                }),
+            )
+        })
+    }
+
+    /**
+     * Accept no new task and interrupt every task, including runs in progress, without waiting. The first call records
+     * in one Info record how many tasks shutdown ended with a run waiting or in progress, so none ends unrecorded
+     */
+    stop(): void {
+        this.#closing = true
+        if (this.#stopped) return
+        this.#stopped = true
+        const running = this.#running
+        let cancelled = this.#ended
+        for (const [fiber, state] of this.#tasks) {
+            // A task the application closed has no run waiting, unless its last run is still in progress
+            if (state.running || !state.closed) cancelled++
+            fiber.interruptUnsafe()
+        }
+        if (cancelled === 0) return
+        this.logging.log({
+            level: "info",
+            category: "lifecycle",
+            code: "lifecycle.tasksCancelled",
+            message: `The client is closing, so it cancelled ${cancelled} scheduled ${cancelled === 1 ? "task" : "tasks"} that had a run waiting or in progress${running ? `, including ${running} ${running === 1 ? "run" : "runs"} in progress` : ""}`,
+            fields: { cancelled, running },
+        })
+    }
+
+    #signalProgress() {
+        const released = this.#progress
+        this.#progress = Deferred.makeUnsafe<void>()
+        Deferred.doneUnsafe(released, Effect.void)
+    }
+}

package/src/internal/sharding.ts

@@ -0,0 +1,270 @@
+/**
+ * Shard plan: Validation of the fixed local shard assignment, automatic plan sizing, scaling hooks and guild-to-shard
+ * routing.
+ * Invariant: A plan is immutable once known and decides guild ownership, never cache contents or discovery hints. An
+ * automatic plan is computed at the first connect from the bot's guild count, and omitted settings start with one shard.
+ * The client replaces either with a new, larger automatic plan only when Fluxer closes a shard with 4011 (sharding
+ * required), and an explicit plan never changes.
+ * Implements [SDK contracts: Connection and recovery](/docs/SDK-CONTRACTS.md#connection-and-recovery)
+ */
+import { ConfigurationError } from "#sdk/errors"
+import type { IdentifyCoordinator, SessionSnapshot, SessionStore } from "#sdk/sharding"
+import { settingError } from "./client/configuration-fields.js"
+import { record } from "./decode/primitives.js"
+import { maxGuildsPerShard, sessionRetentionMs } from "./protocol/gateway.js"
+import { unsupportedKeyHint } from "./suggest.js"
+
+/** Largest total shard count Fluxer accepts */
+export const maximumShardCount = 16_384
+
+/**
+ * Average guilds per shard that automatic sharding aims for, four fifths of Fluxer's per-shard ceiling.
+ * Fluxer counts the guilds routed to each shard, not the average. The validate_session_guild_count function in
+ * fluxer_gateway/src/gateway/gateway_sharding.erl at fluxerapp/fluxer commit 858a2d9e2b987330edd81711bb53e4f7edc0bbcc
+ * rejects Identify with 4011 when one shard's own guilds exceed 2,500. Routing by (id >> 22) % totalShards is uneven,
+ * so a plan sized to the ceiling almost always overfills a shard above about 20,000 guilds. At 2,000 the fullest of
+ * 16,384 shards stays near 2,200 and leaves room for growth before a 4011 closure makes the client move to a larger plan
+ */
+export const guildsPerShard = (maxGuildsPerShard * 4) / 5
+
+export interface ShardPlan {
+    readonly totalShards: number
+    readonly shardIds: readonly number[]
+    readonly identifyShards: boolean
+}
+
+/** Validated sharding settings: A known plan or "auto", plus the optional scaling hooks */
+export interface ShardingConfiguration {
+    readonly plan: ShardPlan | "auto"
+    /** Whether a 4011 (sharding required) closure moves to a larger automatic plan: For "auto" and omitted settings */
+    readonly reshard: boolean
+    readonly identify: IdentifyCoordinator | undefined
+    readonly sessions: SessionStore | undefined
+    /** Whether shards that resume a stored session refill the guild, role and channel caches through REST */
+    readonly refillCaches: boolean
+}
+
+const defaultShardPlan = immutablePlan(1, [0])
+const defaultSharding: ShardingConfiguration = Object.freeze({
+    plan: defaultShardPlan,
+    reshard: true,
+    identify: undefined,
+    sessions: undefined,
+    refillCaches: true,
+})
+
+function hook<K extends string>(
+    value: unknown,
+    field: "identify" | "sessions",
+    methods: readonly K[],
+): Record<K, (...args: never[]) => unknown> | undefined | ConfigurationError {
+    if (value === undefined) return undefined
+    // Methods may come from a class prototype, and other members such as a store's own state are left alone
+    if (!record(value) || methods.some((method) => typeof value[method] !== "function"))
+        return settingError(
+            field,
+            field === "identify"
+                ? 'The option "sharding.identify" must be an object with a permit function'
+                : 'The option "sharding.sessions" must be an object with load and save functions',
+        )
+    // Bind the methods now so later caller mutation cannot change the hooks this client calls
+    return Object.freeze(
+        Object.fromEntries(
+            methods.map((method) => [method, (value[method] as (...args: never[]) => unknown).bind(value)]),
+        ),
+    ) as Record<K, (...args: never[]) => unknown>
+}
+
+/** Validate and copy the one-time sharding settings without changing caller-owned input */
+export function parseSharding(input: unknown): ShardingConfiguration | ConfigurationError {
+    if (input === undefined) return defaultSharding
+    if (input === "auto")
+        return Object.freeze({
+            plan: "auto",
+            reshard: true,
+            identify: undefined,
+            sessions: undefined,
+            refillCaches: true,
+        })
+    if (!record(input)) return new ConfigurationError("sharding", 'Sharding settings must be an object or "auto"')
+    const shardingKeys = ["totalShards", "shardIds", "identify", "sessions", "refillCaches"]
+    const unsupported = Reflect.ownKeys(input).find((key) => typeof key !== "string" || !shardingKeys.includes(key))
+    if (unsupported !== undefined)
+        return new ConfigurationError(
+            "sharding",
+            typeof unsupported === "string"
+                ? `Unsupported sharding setting ${JSON.stringify(unsupported)}`
+                : "Sharding settings must not contain symbol keys",
+            { hint: unsupportedKeyHint(typeof unsupported === "string" ? unsupported : "", shardingKeys, "settings") },
+        )
+    const identify = hook(input.identify, "identify", ["permit"])
+    if (identify instanceof ConfigurationError) return identify
+    const sessions = hook(input.sessions, "sessions", ["load", "save"])
+    if (sessions instanceof ConfigurationError) return sessions
+    const refillCaches = input.refillCaches ?? true
+    if (typeof refillCaches !== "boolean")
+        return new ConfigurationError("sharding", 'The option "sharding.refillCaches" must be a boolean')
+    const hooks = {
+        identify: identify as IdentifyCoordinator | undefined,
+        sessions: sessions as SessionStore | undefined,
+        refillCaches,
+    }
+    const totalShards = input.totalShards
+    if (totalShards === "auto") {
+        if (input.shardIds !== undefined)
+            return new ConfigurationError(
+                "shardIds",
+                "Automatic sharding assigns every shard, so omit sharding.shardIds",
+                {
+                    hint: "Set sharding.totalShards to a number to run only some shards",
+                },
+            )
+        return Object.freeze({ plan: "auto", reshard: true, ...hooks })
+    }
+    if (
+        typeof totalShards !== "number" ||
+        !Number.isSafeInteger(totalShards) ||
+        totalShards < 1 ||
+        totalShards > maximumShardCount
+    ) {
+        return new ConfigurationError(
+            "totalShards",
+            'The option "sharding.totalShards" must be "auto" or an integer from 1 through 16,384',
+        )
+    }
+
+    const inputShardIds = input.shardIds
+    if (inputShardIds === undefined) {
+        return Object.freeze({
+            plan: immutablePlan(
+                totalShards,
+                Array.from({ length: totalShards }, (_, shardId) => shardId),
+            ),
+            reshard: false,
+            ...hooks,
+        })
+    }
+    if (!Array.isArray(inputShardIds) || inputShardIds.length === 0)
+        return new ConfigurationError("shardIds", 'The option "sharding.shardIds" must be a non-empty array')
+
+    const shardIds: number[] = []
+    const seen = new Set<number>()
+    for (const shardId of inputShardIds) {
+        if (
+            typeof shardId !== "number" ||
+            !Number.isSafeInteger(shardId) ||
+            shardId < 0 ||
+            shardId >= totalShards ||
+            seen.has(shardId)
+        ) {
+            return new ConfigurationError(
+                "shardIds",
+                `The option "sharding.shardIds" must contain unique integers from 0 through ${totalShards - 1}`,
+            )
+        }
+        seen.add(shardId)
+        shardIds.push(shardId)
+    }
+    return Object.freeze({ plan: immutablePlan(totalShards, shardIds), reshard: false, ...hooks })
+}
+
+/** The plan automatic sharding picks for a guild count: Every shard, at most guildsPerShard guilds each on average */
+export function automaticShardPlan(guildCount: number): ShardPlan {
+    return fullShardPlan(Math.min(maximumShardCount, Math.max(1, Math.ceil(guildCount / guildsPerShard))))
+}
+
+/**
+ * The plan an automatic client moves to after Fluxer closed one of its shards with 4011 (sharding required): The plan
+ * for the new guild count, or one shard more than before when that count still fits the old total, because Fluxer
+ * found a shard above its ceiling whatever the average. Callers check that the old total is below maximumShardCount
+ */
+export function largerShardPlan(guildCount: number, previousTotalShards: number): ShardPlan {
+    const counted = automaticShardPlan(guildCount)
+    return counted.totalShards > previousTotalShards
+        ? counted
+        : fullShardPlan(Math.min(maximumShardCount, previousTotalShards + 1))
+}
+
+/** Moves to a larger automatic plan allowed within reshardWindowMs, for a client or a supervisor */
+const reshardLimit = 3
+const reshardWindowMs = 3_600_000
+
+/**
+ * Admit a move to a larger automatic plan at now, recording it in history, or return why it is refused: The plan
+ * already has the largest total, or reshardLimit moves happened within reshardWindowMs. The history keeps only moves
+ * inside the window
+ */
+export function admitReshard(history: number[], now: number, totalShards: number): string | undefined {
+    if (totalShards >= maximumShardCount)
+        return `the plan already has the largest total of ${maximumShardCount.toLocaleString("en-US")} shards`
+    const recent = history.filter((at) => now - at < reshardWindowMs)
+    history.splice(0, history.length, ...recent)
+    if (history.length >= reshardLimit) return `it already moved to a larger plan ${reshardLimit} times within an hour`
+    history.push(now)
+    return undefined
+}
+
+/** A plan of totalShards shards, all owned by this client */
+function fullShardPlan(totalShards: number): ShardPlan {
+    return immutablePlan(
+        totalShards,
+        Array.from({ length: totalShards }, (_, shardId) => shardId),
+    )
+}
+
+/** Route a guild snowflake to its planned shard. Callers provide a validated plan total */
+export function guildShardId(guildId: string, totalShards: number): number {
+    return Number((BigInt(guildId) >> 22n) % BigInt(totalShards))
+}
+
+function immutablePlan(totalShards: number, shardIds: readonly number[]): ShardPlan {
+    return Object.freeze({
+        totalShards,
+        shardIds: Object.freeze([...shardIds]),
+        identifyShards: totalShards > 1,
+    })
+}
+
+/** Allowed clock difference between the process that saved a snapshot and the one loading it */
+const snapshotClockSkewMs = 5_000
+
+/** Check a loaded snapshot: The frozen copy to resume, or why it cannot be resumed */
+export function checkSnapshot(
+    value: unknown,
+    gatewayUrl: string,
+    totalShards: number,
+    nowMs: number,
+): { readonly snapshot: SessionSnapshot } | { readonly problem: string } {
+    if (
+        !record(value) ||
+        typeof value.sessionId !== "string" ||
+        value.sessionId.length === 0 ||
+        value.sessionId.length > 256 ||
+        typeof value.sequence !== "number" ||
+        !Number.isSafeInteger(value.sequence) ||
+        value.sequence < 0 ||
+        typeof value.resumeUrl !== "string" ||
+        typeof value.savedAt !== "number" ||
+        !Number.isSafeInteger(value.savedAt) ||
+        typeof value.totalShards !== "number" ||
+        !Number.isSafeInteger(value.totalShards) ||
+        value.totalShards < 1
+    )
+        return { problem: "the snapshot is malformed" }
+    const age = nowMs - value.savedAt
+    if (age >= sessionRetentionMs) return { problem: "the snapshot is older than Fluxer's 60-second resume window" }
+    if (age < -snapshotClockSkewMs) return { problem: "the snapshot was saved in the future" }
+    // Resume sends the token, so it only goes to the gateway this client discovered itself
+    if (value.resumeUrl !== gatewayUrl) return { problem: "the snapshot belongs to a different gateway endpoint" }
+    // Fluxer binds a session to the shard tuple it identified with, so resuming it under another plan routes guilds to the wrong shard
+    if (value.totalShards !== totalShards) return { problem: "the snapshot belongs to a different shard plan" }
+    return {
+        snapshot: Object.freeze({
+            sessionId: value.sessionId,
+            sequence: value.sequence,
+            resumeUrl: value.resumeUrl,
+            savedAt: value.savedAt,
+            totalShards: value.totalShards,
+        }),
+    }
+}