chapterjs 0.1.2 → 0.1.4

package/dist/cli/build.js

@@ -0,0 +1,206 @@
+// `chapterjs build`: prepares the bot for production. Everything that can
+// go wrong with the files of the project goes wrong here, before anything
+// is online: types are checked, every file is run, then the whole bot is
+// compiled into one JavaScript file in `.chapterjs/build/`. `chapterjs
+// start` runs that, and only that: the bot in production is exactly what
+// was built.
+//
+// The compiling is done by esbuild, the one dependency of the framework. It
+// is only loaded here: running a bot never needs it.
+import { spawn } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { mkdir, readdir, readFile, rm, stat, writeFile, } from 'node:fs/promises';
+import { join, relative, sep } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { enableProjectLoader } from '../loader/hot.js';
+import { listFolder } from '../loader/loader.js';
+import { sourcesJoinVoice } from '../voice/usage.js';
+import { CONVENTIONS, createProject, hasSources, writeTypes, } from './project.js';
+/** Where the build of a project is, from its folder. */
+export const buildDir = (cwd) => join(cwd, '.chapterjs', 'build');
+/** The file the whole bot is compiled into, in the build. */
+export const BUNDLE = 'bot.js';
+/** One value for everything in a folder: it changes when a file does. */
+export async function fingerprint(dir) {
+    const hash = createHash('sha256');
+    const entries = await readdir(dir, { recursive: true, withFileTypes: true });
+    const files = entries
+        .filter(entry => entry.isFile())
+        .map(entry => join(entry.parentPath, entry.name))
+        .sort();
+    for (const file of files) {
+        hash.update(relative(dir, file).split(sep).join('/'));
+        hash.update('\0');
+        hash.update(await readFile(file));
+        hash.update('\0');
+    }
+    return hash.digest('hex');
+}
+/** The build of a project, or why there is none to run. */
+export async function readBuild(cwd, version) {
+    let info;
+    try {
+        info = JSON.parse(await readFile(join(buildDir(cwd), 'build.json'), 'utf8'));
+        await stat(join(buildDir(cwd), BUNDLE));
+    }
+    catch {
+        return {
+            problem: 'There is no build of your bot here.\nRun "chapterjs build" first (the build script of your project), then start again.',
+        };
+    }
+    if (info.version !== version) {
+        return {
+            problem: `This build was made with chapterjs ${info.version}, and this is chapterjs ${version}.\nRun "chapterjs build" again.`,
+        };
+    }
+    // On a host that only received the build, there are no sources to compare.
+    const src = join(cwd, 'src');
+    const hasSrc = (await stat(src).catch(() => null))?.isDirectory() === true;
+    return {
+        info,
+        stale: hasSrc && (await fingerprint(src)) !== info.sources,
+    };
+}
+/** Runs the TypeScript of the project on itself. `null`: there is none. */
+async function checkTypes(cwd) {
+    const tsc = join(cwd, 'node_modules', '.bin', 'tsc');
+    const [hasTsc, hasConfig] = await Promise.all([
+        stat(tsc).catch(() => null),
+        stat(join(cwd, 'tsconfig.json')).catch(() => null),
+    ]);
+    if (!hasTsc || !hasConfig)
+        return null;
+    return new Promise(resolve => {
+        const child = spawn(tsc, ['-b', '--pretty', 'false'], { cwd });
+        let output = '';
+        child.stdout.on('data', (chunk) => (output += chunk.toString()));
+        child.stderr.on('data', (chunk) => (output += chunk.toString()));
+        child.on('error', error => resolve({ ok: false, output: error.message }));
+        child.on('close', code => resolve({ ok: code === 0, output }));
+    });
+}
+/**
+ * Loads the bot a build compiled: every file of the project, already run.
+ * Errors that come out of it afterwards name the file and the line of the
+ * project, not of the compiled file: Node reads the map next to it.
+ */
+export async function loadBundle(cwd) {
+    process.setSourceMapsEnabled(true);
+    const module = (await import(pathToFileURL(join(buildDir(cwd), BUNDLE)).href));
+    return module.files;
+}
+/** Resolves with the exit code of the process. */
+export async function build(options) {
+    const { cwd, log } = options;
+    if (!(await hasSources(cwd, log)))
+        return 1;
+    const src = join(cwd, 'src');
+    const out = buildDir(cwd);
+    // Whatever happens next, the last build is not one to start any more.
+    await rm(out, { recursive: true, force: true });
+    const stopped = async (failures, why = "can't run") => {
+        log.error(`${failures === 1 ? 'This file' : `These ${failures} files`} ${why}, so your bot was not built. Fix ${failures === 1 ? 'it' : 'them'} and build again: chapterjs dev shows the same errors while you write.`);
+        await rm(out, { recursive: true, force: true });
+        return 1;
+    };
+    // 1. Every file must run, one by one: each says what is wrong with it.
+    enableProjectLoader(src, { reload: false });
+    const sources = createProject({ cwd, version: options.version, log });
+    const failures = await sources.load();
+    // 2. Types: what the editor underlines, for the whole project. Written
+    // once the files ran, so a language file only offers the commands they
+    // do not describe; written even when a file failed, so the editor
+    // follows.
+    await writeTypes(cwd, sources.commands, sources.messages, sources.languageDefaults);
+    if (failures.length > 0) {
+        for (const failure of failures)
+            sources.report(failure);
+        return stopped(failures.length);
+    }
+    const types = await checkTypes(cwd);
+    if (types === null) {
+        log.info('Types were not checked: TypeScript is not installed in this project (or it has no tsconfig.json).');
+    }
+    else if (!types.ok) {
+        log.error(`Your project has type errors, so it was not built:\n${types.output.trim()}`);
+        return 1;
+    }
+    else {
+        log.success('Types checked');
+    }
+    // 3. The whole bot in one file: every file of the conventional folders,
+    // with what it imports from the project. Packages stay where they are
+    // installed: the bot and the framework must share the same ones.
+    const found = (await Promise.all(CONVENTIONS.map(convention => listFolder(cwd, convention)))).flat();
+    const entry = [
+        ...found.map(({ path }, index) => `import * as m${index} from ${JSON.stringify(path)};`),
+        `export const files = [${found
+            .map(({ file }, index) => `{ file: ${JSON.stringify(file)}, exports: m${index} }`)
+            .join(', ')}];`,
+    ].join('\n');
+    await mkdir(out, { recursive: true });
+    // Loaded when it is needed: no other command pays for it.
+    const esbuild = await import('esbuild');
+    try {
+        await esbuild.build({
+            stdin: {
+                contents: entry,
+                resolveDir: cwd,
+                sourcefile: 'chapterjs-build.js',
+                loader: 'js',
+            },
+            absWorkingDir: cwd,
+            outfile: join(out, BUNDLE),
+            bundle: true,
+            platform: 'node',
+            format: 'esm',
+            target: 'node24',
+            packages: 'external',
+            // Compact, but with the names the developer wrote: an error must
+            // still read like their code. The map gives back files and lines.
+            minifyWhitespace: true,
+            minifySyntax: true,
+            minifyIdentifiers: false,
+            keepNames: true,
+            sourcemap: true,
+            sourcesContent: false,
+            logLevel: 'silent',
+        });
+    }
+    catch (error) {
+        const errors = error.errors ?? [];
+        if (errors.length === 0)
+            throw error;
+        for (const { text, location } of errors) {
+            log.error(`${location ? `${location.file}:${location.line} ` : ''}${text}`);
+        }
+        return stopped(errors.length, "can't be compiled");
+    }
+    // 4. What was compiled is run once, as production will run it.
+    const built = createProject({
+        cwd,
+        version: options.version,
+        log,
+        built: await loadBundle(cwd),
+    });
+    const broken = await built.load();
+    if (broken.length > 0) {
+        for (const failure of broken)
+            built.report(failure);
+        return stopped(broken.length, "can't run once compiled");
+    }
+    const info = {
+        version: options.version,
+        sources: await fingerprint(src),
+        commands: built.commands.size,
+        events: built.events.size,
+        voice: await sourcesJoinVoice(cwd),
+    };
+    await writeFile(join(out, 'build.json'), JSON.stringify(info, null, 2));
+    if (built.isEmpty())
+        log.info(built.summary());
+    const { size } = await stat(join(out, BUNDLE));
+    log.success(`Built in .chapterjs/build: ${built.isEmpty() ? 'nothing to run yet' : built.summary().replace(/ loaded$/, '')} (${Math.max(1, Math.round(size / 1024))} kB)`);
+    log.info('Run chapterjs start to put it online.');
+    return 0;
+}

package/dist/cli/cli.js

@@ -1,33 +1,116 @@
 #!/usr/bin/env node
+import { readFileSync } from 'node:fs';
+import { build } from './build.js';
+import { readAssignment } from './cluster.js';
 import { dev } from './dev.js';
-const HELP = `Usage: chapterjs <command>
+import { writeTypes } from './project.js';
+import { createLog } from './log.js';
+import { nodeTooOld } from './node-version.js';
+import { start } from './start.js';
+const { version } = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8'));
+const USAGE = `Usage: chapterjs <command>
 
-  dev                    Local development
-  build                  Validate and bundle the project`;
-const [command, ...args] = process.argv.slice(2);
-// package.json engines: global WebSocket and running .ts files without flags
-const [major = 0, minor = 0] = process.versions.node.split('.').map(Number);
-if (major < 22 || (major === 22 && minor < 18)) {
-    console.error(`✗ ChapterJS needs Node.js 22.18 or newer, you have ${process.version}. Install the latest LTS from https://nodejs.org`);
+Commands:
+  dev     Run your bot on your dev server and reload it when you save a file
+  build   Check your bot and prepare it for production
+  start   Run for everyone the bot that was built
+  sync    Write the types of your project (dev does it too)
+
+Options of start:
+  --processes <n>   How many processes to use (by default: as many as the
+                    size of your bot needs)`;
+const colors = process.stdout.isTTY === true && !process.env.NO_COLOR;
+const log = createLog(line => console.log(line), colors);
+const tooOld = nodeTooOld();
+if (tooOld) {
+    log.error(tooOld);
     process.exit(1);
 }
-try {
-    switch (command) {
-        case 'dev':
-            await dev();
-            break;
-        case 'build':
-            console.log('build');
-            break;
-        default:
-            if (command && command !== 'help' && command !== '--help') {
-                console.error(`✗ Unknown command "${command}".\n`);
-            }
-            console.log(HELP);
-            process.exit(command && command !== 'help' && command !== '--help' ? 1 : 0);
+const [command, ...rest] = process.argv.slice(2);
+/**
+ * Ctrl+C and a stop asked by the system both end the session cleanly, so
+ * the bot goes offline at once. A second one does not wait.
+ */
+function stopSignal() {
+    const controller = new AbortController();
+    for (const name of ['SIGINT', 'SIGTERM']) {
+        process.on(name, () => {
+            if (controller.signal.aborted)
+                process.exit(1);
+            controller.abort();
+        });
     }
+    return controller.signal;
 }
-catch (error) {
-    console.error(error instanceof Error ? `✗ ${error.message}` : error);
-    process.exit(1);
+/** `--processes 4` or `--processes=4`; `null` when it is not a count. */
+function readProcesses(args) {
+    const index = args.findIndex(arg => arg === '--processes' || arg.startsWith('--processes='));
+    if (index === -1)
+        return args.length === 0 ? undefined : null;
+    const arg = args[index];
+    const value = arg.includes('=')
+        ? arg.slice(arg.indexOf('=') + 1)
+        : args[index + 1];
+    const others = args.length - (arg.includes('=') ? 1 : 2);
+    const count = Number(value);
+    return others === 0 && Number.isInteger(count) && count >= 1 ? count : null;
+}
+if (command === 'dev') {
+    const controller = { signal: stopSignal() };
+    process.exitCode = await dev({
+        cwd: process.cwd(),
+        env: process.env,
+        version,
+        log,
+        interactive: process.stdin.isTTY === true && process.stdout.isTTY === true,
+        signal: controller.signal,
+    });
+    // Nothing may keep the process alive once the bot is disconnected.
+    process.exit();
+}
+else if (command === 'start') {
+    const processes = readProcesses(rest);
+    if (processes === null) {
+        log.error(`chapterjs start takes one option: --processes followed by a number, like "chapterjs start --processes 4". Got "${rest.join(' ')}".`);
+        process.exitCode = 1;
+    }
+    else {
+        const assignment = readAssignment(process.env);
+        process.exitCode = await start({
+            cwd: process.cwd(),
+            env: { ...process.env, CHAPTERJS_COLORS: colors ? '1' : '0' },
+            version,
+            // A process started by another prints for it: colors are its choice.
+            log: assignment
+                ? createLog(line => console.log(line), process.env.CHAPTERJS_COLORS === '1')
+                : log,
+            write: line => console.log(line),
+            signal: stopSignal(),
+            processes,
+            assignment,
+            script: process.argv[1],
+            args: ['start'],
+        });
+        process.exit();
+    }
+}
+else if (command === 'sync') {
+    // Run after installing, so the editor knows the types before the first
+    // `chapterjs dev`.
+    await writeTypes(process.cwd());
+    log.success('Types written to .chapterjs/');
+}
+else if (command === '--version' || command === '-v') {
+    console.log(version);
+}
+else if (command === undefined || command === '--help' || command === '-h') {
+    console.log(USAGE);
+}
+else if (command === 'build') {
+    process.exitCode = await build({ cwd: process.cwd(), version, log });
+    process.exit();
+}
+else {
+    log.error(`"${command}" is not a command.\n${USAGE}`);
+    process.exitCode = 1;
 }

package/dist/cli/cluster.d.ts

@@ -0,0 +1,70 @@
+import { type IdentifyGate } from '../gateway/identify-queue.js';
+/** How many shards one process runs when nothing else is asked. */
+export declare const SHARDS_PER_PROCESS = 4;
+/** The exit code of a process Discord refused for good: not started again. */
+export declare const REFUSED = 78;
+/** The share of one process, given to it when it starts. */
+export interface Assignment {
+    /** Which process this is, from 0. */
+    index: number;
+    /** How many processes run the bot. */
+    processes: number;
+    /** The shards it runs, out of `count`. */
+    ids: number[];
+    count: number;
+}
+/** The name of the variable a process finds its share in. */
+export declare const ASSIGNMENT_VARIABLE = "CHAPTERJS_PROCESS";
+/**
+ * How many processes run a bot of `shards` shards: what was asked, or one
+ * for every few shards, never more than the machine has CPUs.
+ */
+export declare function planProcesses(shards: number, asked?: number, cpus?: number): number;
+/** The shards of each process: as even as it gets, in order. */
+export declare function splitShards(count: number, processes: number): number[][];
+/** Reads the share of this process, if it was started by another one. */
+export declare function readAssignment(env: Record<string, string | undefined>): Assignment | null;
+/**
+ * In a process started by another: asks that one when a shard may
+ * identify, and hears when to stop.
+ */
+export declare function connectToPrimary(): {
+    gate: IdentifyGate;
+    ready(info: {
+        guilds: number;
+        user: string;
+    }): void;
+    refused(): void;
+    /** Resolves when this process is asked to stop, or is left alone. */
+    stopped: Promise<void>;
+};
+export interface ClusterOptions {
+    /** The file to run in each process, and its arguments. */
+    script: string;
+    args: readonly string[];
+    env: Record<string, string | undefined>;
+    shards: number;
+    processes: number;
+    maxConcurrency: number;
+    /** Aborted when the bot is asked to stop. */
+    signal: AbortSignal;
+    /** Shows a line printed by one of the processes. */
+    write: (line: string) => void;
+    /** Every process is connected, for the first time. */
+    onReady: (info: {
+        guilds: number;
+        user: string;
+    }) => void;
+    /** A process stopped by itself and is started again. */
+    onRestart: (index: number, why: string) => void;
+    /** Only tests change these. */
+    identifyInterval?: number | undefined;
+    restartDelay?: number;
+    stopTimeout?: number;
+}
+/**
+ * Starts the processes of the bot and looks after them until asked to
+ * stop. Resolves with the exit code: 0 when stopped, 1 when Discord refused
+ * the bot.
+ */
+export declare function runCluster(options: ClusterOptions): Promise<number>;

package/dist/cli/cluster.js

@@ -0,0 +1,213 @@
+// Several processes for one bot. Discord splits a bot into shards, each
+// receiving the events of some servers; a process can only use one CPU, so
+// a large bot spreads its shards over several processes. The first process
+// (the one the developer started) checks everything once, then starts the
+// others and looks after them; each of them runs the project for its share
+// of the shards.
+//
+// What processes must agree on goes through the first one: when a shard may
+// identify. A setup on several machines would give each process another
+// `IdentifyGate`, and nothing else here would change.
+// https://docs.discord.com/developers/events/gateway#sharding
+import { fork } from 'node:child_process';
+import { availableParallelism } from 'node:os';
+import { createInterface } from 'node:readline';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { IdentifyQueue } from '../gateway/identify-queue.js';
+/** How many shards one process runs when nothing else is asked. */
+export const SHARDS_PER_PROCESS = 4;
+/** The exit code of a process Discord refused for good: not started again. */
+export const REFUSED = 78;
+/** The name of the variable a process finds its share in. */
+export const ASSIGNMENT_VARIABLE = 'CHAPTERJS_PROCESS';
+/**
+ * How many processes run a bot of `shards` shards: what was asked, or one
+ * for every few shards, never more than the machine has CPUs.
+ */
+export function planProcesses(shards, asked, cpus = availableParallelism()) {
+    const wanted = asked ?? Math.min(cpus, Math.ceil(shards / SHARDS_PER_PROCESS));
+    return Math.max(1, Math.min(shards, Math.floor(wanted)));
+}
+/** The shards of each process: as even as it gets, in order. */
+export function splitShards(count, processes) {
+    const shares = [];
+    let next = 0;
+    for (let index = 0; index < processes; index++) {
+        const size = Math.ceil((count - next) / (processes - index));
+        shares.push(Array.from({ length: size }, (_, offset) => next + offset));
+        next += size;
+    }
+    return shares;
+}
+/** Reads the share of this process, if it was started by another one. */
+export function readAssignment(env) {
+    const raw = env[ASSIGNMENT_VARIABLE];
+    return raw ? JSON.parse(raw) : null;
+}
+/**
+ * In a process started by another: asks that one when a shard may
+ * identify, and hears when to stop.
+ */
+export function connectToPrimary() {
+    const waiting = new Map();
+    let lastId = 0;
+    const send = (message) => {
+        // The first process may be gone: there is nobody to tell any more.
+        if (process.connected)
+            process.send?.(message);
+    };
+    let stop;
+    const stopped = new Promise(resolve => (stop = resolve));
+    process.on('message', (message) => {
+        if (message.type === 'identify') {
+            waiting.get(message.id)?.();
+            waiting.delete(message.id);
+        }
+        else if (message.type === 'stop') {
+            stop();
+        }
+    });
+    // Without the process that started it, nothing looks after this one.
+    process.on('disconnect', stop);
+    return {
+        gate: {
+            wait: shard => new Promise(resolve => {
+                const id = ++lastId;
+                waiting.set(id, resolve);
+                send({ type: 'identify', shard, id });
+            }),
+        },
+        ready: info => send({ type: 'ready', ...info }),
+        refused: () => send({ type: 'refused' }),
+        stopped,
+    };
+}
+/**
+ * Sends a message to a process, and says whether it could. A process that
+ * is exiting has a channel that fails: the error goes to the callback
+ * instead of crashing the first process.
+ */
+function tell(child, message) {
+    if (!child.connected)
+        return false;
+    child.send(message, () => { });
+    return true;
+}
+/**
+ * Starts the processes of the bot and looks after them until asked to
+ * stop. Resolves with the exit code: 0 when stopped, 1 when Discord refused
+ * the bot.
+ */
+export async function runCluster(options) {
+    const { signal } = options;
+    const gate = new IdentifyQueue(options.maxConcurrency, options.identifyInterval);
+    const shares = splitShards(options.shards, options.processes);
+    const workers = new Map();
+    const stopped = new WeakSet();
+    const ready = new Map();
+    let announced = false;
+    let refused = false;
+    let end;
+    const ended = new Promise(resolve => (end = resolve));
+    const stopping = () => signal.aborted || refused;
+    const start = (index, delay) => {
+        const assignment = {
+            index,
+            processes: options.processes,
+            ids: shares[index],
+            count: options.shards,
+        };
+        const started = Date.now();
+        const child = fork(options.script, [...options.args], {
+            env: {
+                ...options.env,
+                [ASSIGNMENT_VARIABLE]: JSON.stringify(assignment),
+            },
+            stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        });
+        workers.set(index, child);
+        // A message sent while the process exits fails; it must not stop this
+        // one (see `tell`), and its exit is what matters.
+        child.on('error', () => { });
+        // Everything a process prints (the framework and the code of the
+        // project alike) is shown with which one said it.
+        for (const stream of [child.stdout, child.stderr]) {
+            createInterface({ input: stream }).on('line', line => options.write(`[${index + 1}] ${line}`));
+        }
+        child.on('message', (message) => {
+            if (message.type === 'identify') {
+                void gate.wait(message.shard).then(() => {
+                    tell(child, { type: 'identify', id: message.id });
+                });
+            }
+            else if (message.type === 'ready') {
+                ready.set(index, message);
+                if (!announced && ready.size === options.processes) {
+                    announced = true;
+                    options.onReady({
+                        guilds: [...ready.values()].reduce((sum, r) => sum + r.guilds, 0),
+                        user: message.user,
+                    });
+                }
+            }
+            else if (message.type === 'refused') {
+                refused = true;
+            }
+        });
+        child.on('exit', (code, killedBy) => {
+            workers.delete(index);
+            if (code === REFUSED)
+                refused = true;
+            if (stopping()) {
+                // The others have nothing to wait for either.
+                for (const other of workers.values())
+                    stopWorker(other);
+                if (workers.size === 0)
+                    end();
+                return;
+            }
+            // A process that ran for a while starts again at once; one that
+            // keeps falling waits longer each time.
+            const next = Date.now() - started > 60_000
+                ? (options.restartDelay ?? 1000)
+                : Math.min(60_000, Math.max(delay, 1) * 2);
+            options.onRestart(index, killedBy ? `stopped by ${killedBy}` : `exit code ${code}`);
+            void sleep(delay).then(() => {
+                if (stopping()) {
+                    if (workers.size === 0)
+                        end();
+                }
+                else {
+                    start(index, next);
+                }
+            });
+        });
+    };
+    const stopWorker = (child) => {
+        // Asked once: a process already stopping has nothing more to hear.
+        if (stopped.has(child))
+            return;
+        stopped.add(child);
+        if (!tell(child, { type: 'stop' }))
+            child.kill('SIGTERM');
+        // A process that does not stop is not waited for forever.
+        const timer = setTimeout(() => child.kill('SIGKILL'), options.stopTimeout ?? 15_000);
+        timer.unref();
+        child.once('exit', () => clearTimeout(timer));
+    };
+    for (let index = 0; index < options.processes; index++) {
+        start(index, options.restartDelay ?? 1000);
+    }
+    const stopAll = () => {
+        if (workers.size === 0)
+            end();
+        for (const child of workers.values())
+            stopWorker(child);
+    };
+    if (signal.aborted)
+        stopAll();
+    else
+        signal.addEventListener('abort', stopAll, { once: true });
+    await ended;
+    return refused ? 1 : 0;
+}

package/dist/cli/dev.d.ts

@@ -1 +1,18 @@
-export declare function dev(): Promise<void>;
+import type { Log } from './log.js';
+export interface DevOptions {
+    /** The folder of the project. */
+    cwd: string;
+    env: Record<string, string | undefined>;
+    /** The version of the framework. */
+    version: string;
+    log: Log;
+    /** Whether a person is watching the terminal. */
+    interactive: boolean;
+    /** Aborted when the user asks to stop (Ctrl+C). */
+    signal: AbortSignal;
+    /** Only tests change these. */
+    pollInterval?: number;
+    deferAfter?: number;
+}
+/** Runs until stopped. Resolves with the exit code of the process. */
+export declare function dev(options: DevOptions): Promise<number>;