@vex-chat/spire 1.0.1 → 1.0.2

package/src/Database.ts

@@ -21,6 +21,7 @@ import type { Migration, MigrationProvider } from "kysely";
 
 import { EventEmitter } from "events";
 import { pbkdf2Sync } from "node:crypto";
+import { statSync } from "node:fs";
 import * as fs from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
@@ -119,6 +120,34 @@ export interface InternalUserRecord extends UserRecord {
 export class Database extends EventEmitter {
     private db: Kysely<ServerDatabase>;
 
+    /** Underlying better-sqlite3 handle (file or :memory:). */
+    private readonly rawSqlite: InstanceType<typeof BetterSqlite3>;
+
+    /**
+     * In-process cache for {@link retrieveUserDeviceList} when queried for a
+     * single owner. Libvex calls GET /user/:id/devices on every outgoing
+     * `forward()`; stress runs otherwise hammer SQLite with identical reads.
+     * Cleared on device create/delete and when the DB is closed.
+     */
+    private readonly userDeviceListCache = new Map<
+        string,
+        { devices: Device[]; until: number }
+    >();
+
+    /**
+     * Short TTL cache for {@link retrieveEmojiList} (key = server id, stored as
+     * `emojis.owner`). The noise harness fires `emoji.retrieveList` as a
+     * follow-up on ~1/25 successful ops, so this path can contend on SQLite
+     * before heavier surfaces. Invalidated on emoji create/delete.
+     */
+    private readonly emojiListByServerCache = new Map<
+        string,
+        { rows: Emoji[]; until: number }
+    >();
+
+    private static readonly USER_DEVICE_LIST_CACHE_TTL_MS = 10_000;
+    private static readonly EMOJI_LIST_CACHE_TTL_MS = 10_000;
+
     constructor(options?: SpireOptions) {
         super();
 
@@ -138,22 +167,86 @@ export class Database extends EventEmitter {
                 break;
         }
 
-        const sqliteDb = new BetterSqlite3(filename);
-        sqliteDb.pragma("journal_mode = WAL");
-        sqliteDb.pragma("synchronous = NORMAL");
-        sqliteDb.pragma("busy_timeout = 5000");
-        sqliteDb.pragma("cache_size = -64000");
-        sqliteDb.pragma("temp_store = memory");
-        sqliteDb.pragma("foreign_keys = ON");
+        this.rawSqlite = new BetterSqlite3(filename);
+        this.rawSqlite.pragma("journal_mode = WAL");
+        this.rawSqlite.pragma("synchronous = NORMAL");
+        this.rawSqlite.pragma("busy_timeout = 5000");
+        this.rawSqlite.pragma("cache_size = -64000");
+        this.rawSqlite.pragma("temp_store = memory");
+        this.rawSqlite.pragma("foreign_keys = ON");
 
         this.db = new Kysely<ServerDatabase>({
-            dialect: new SqliteDialect({ database: sqliteDb }),
+            dialect: new SqliteDialect({ database: this.rawSqlite }),
         });
 
         void this.init();
     }
 
+    /**
+     * Dev-only snapshot of SQLite files + pragmas (same process as Spire).
+     * Not for production callers.
+     */
+    public getDevSqliteMonitor(): Record<string, unknown> {
+        const openedAs = this.rawSqlite.name;
+        const absPath =
+            openedAs === ":memory:"
+                ? ":memory:"
+                : path.isAbsolute(openedAs)
+                  ? openedAs
+                  : path.resolve(process.cwd(), openedAs);
+
+        const journalMode = this.rawSqlite.pragma("journal_mode", {
+            simple: true,
+        });
+        const synchronous = this.rawSqlite.pragma("synchronous", {
+            simple: true,
+        });
+        const busyTimeout = this.rawSqlite.pragma("busy_timeout", {
+            simple: true,
+        });
+        const cacheSize = this.rawSqlite.pragma("cache_size", { simple: true });
+        const pageCount = this.rawSqlite.pragma("page_count", { simple: true });
+        const pageSize = this.rawSqlite.pragma("page_size", { simple: true });
+        const freelistCount = this.rawSqlite.pragma("freelist_count", {
+            simple: true,
+        });
+
+        const out: Record<string, unknown> = {
+            absPath,
+            busyTimeout,
+            cacheSize,
+            freelistCount,
+            journalMode,
+            openedAs,
+            pageCount,
+            pageSize,
+            synchronous,
+        };
+
+        if (openedAs !== ":memory:") {
+            const filePaths = {
+                main: absPath,
+                shm: `${absPath}-shm`,
+                wal: `${absPath}-wal`,
+            };
+            out["filePaths"] = filePaths;
+            const sizes: Record<string, number> = {};
+            for (const [label, fp] of Object.entries(filePaths)) {
+                try {
+                    sizes[label] = statSync(fp).size;
+                } catch {
+                    sizes[label] = 0;
+                }
+            }
+            out["fileBytes"] = sizes;
+        }
+
+        return out;
+    }
+
     public async close(): Promise<void> {
+        this.userDeviceListCache.clear();
+        this.emojiListByServerCache.clear();
         await this.db.destroy();
     }
 
@@ -184,6 +277,7 @@ export class Database extends EventEmitter {
         };
 
         await this.db.insertInto("devices").values(device).execute();
+        this.userDeviceListCache.delete(owner);
 
         const medPreKeys = {
             deviceID: device.deviceID,
@@ -201,6 +295,7 @@ export class Database extends EventEmitter {
 
     public async createEmoji(emoji: Emoji): Promise<void> {
         await this.db.insertInto("emojis").values(emoji).execute();
+        this.emojiListByServerCache.delete(emoji.owner);
     }
 
     public async createFile(file: FileSQL): Promise<void> {
@@ -230,30 +325,31 @@ export class Database extends EventEmitter {
         resourceID: string,
         powerLevel: number,
     ): Promise<Permission> {
-        const permissionID = crypto.randomUUID();
-
-        // check if it already exists
-        const checkPermission = await this.db
-            .selectFrom("permissions")
-            .selectAll()
-            .where("userID", "=", userID)
-            .where("resourceID", "=", resourceID)
-            .execute();
-        const existing = checkPermission[0];
-        if (existing) {
-            return existing;
-        }
+        // Atomic check-then-insert inside a transaction so two concurrent
+        // callers cannot both find "no existing" and insert duplicates.
+        return this.db.transaction().execute(async (trx) => {
+            const checkPermission = await trx
+                .selectFrom("permissions")
+                .selectAll()
+                .where("userID", "=", userID)
+                .where("resourceID", "=", resourceID)
+                .execute();
+            const existing = checkPermission[0];
+            if (existing) {
+                return existing;
+            }
 
-        const permission: Permission = {
-            permissionID,
-            powerLevel,
-            resourceID,
-            resourceType,
-            userID,
-        };
+            const permission: Permission = {
+                permissionID: crypto.randomUUID(),
+                powerLevel,
+                resourceID,
+                resourceType,
+                userID,
+            };
 
-        await this.db.insertInto("permissions").values(permission).execute();
-        return permission;
+            await trx.insertInto("permissions").values(permission).execute();
+            return permission;
+        });
     }
 
     public async createServer(name: string, ownerID: string): Promise<Server> {
@@ -321,6 +417,12 @@ export class Database extends EventEmitter {
     }
 
     public async deleteDevice(deviceID: string): Promise<void> {
+        const ownerRow = await this.db
+            .selectFrom("devices")
+            .select("owner")
+            .where("deviceID", "=", deviceID)
+            .executeTakeFirst();
+
         await this.db
             .deleteFrom("preKeys")
             .where("deviceID", "=", deviceID)
@@ -336,13 +438,21 @@ export class Database extends EventEmitter {
             .set({ deleted: 1 })
             .where("deviceID", "=", deviceID)
             .execute();
+
+        if (ownerRow?.owner) {
+            this.userDeviceListCache.delete(ownerRow.owner);
+        }
     }
 
     public async deleteEmoji(emojiID: string): Promise<void> {
+        const existing = await this.retrieveEmoji(emojiID);
         await this.db
             .deleteFrom("emojis")
             .where("emojiID", "=", emojiID)
             .execute();
+        if (existing) {
+            this.emojiListByServerCache.delete(existing.owner);
+        }
     }
 
     public async deleteInvite(inviteID: string): Promise<void> {
@@ -405,31 +515,34 @@ export class Database extends EventEmitter {
     }
 
     public async getOTK(deviceID: string): Promise<null | PreKeysWS> {
-        const rows: PreKeysSQL[] = await this.db
-            .selectFrom("oneTimeKeys")
-            .selectAll()
-            .where("deviceID", "=", deviceID)
-            .orderBy("index")
-            .limit(1)
-            .execute();
-        const otkInfo = rows[0];
-        if (!otkInfo) {
-            return null;
-        }
-        const otk: PreKeysWS = {
-            deviceID: otkInfo.deviceID,
-            index: otkInfo.index,
-            publicKey: XUtils.decodeHex(otkInfo.publicKey),
-            signature: XUtils.decodeHex(otkInfo.signature),
-        };
+        // Atomic select-then-delete inside a transaction so two concurrent
+        // callers cannot dispense the same one-time key.
+        return this.db.transaction().execute(async (trx) => {
+            const rows: PreKeysSQL[] = await trx
+                .selectFrom("oneTimeKeys")
+                .selectAll()
+                .where("deviceID", "=", deviceID)
+                .orderBy("index")
+                .limit(1)
+                .execute();
+            const otkInfo = rows[0];
+            if (!otkInfo) {
+                return null;
+            }
 
-        // delete the otk
-        await this.db
-            .deleteFrom("oneTimeKeys")
-            .where("deviceID", "=", deviceID)
-            .where("index", "=", otk.index)
-            .execute();
-        return otk;
+            await trx
+                .deleteFrom("oneTimeKeys")
+                .where("deviceID", "=", deviceID)
+                .where("index", "=", otkInfo.index)
+                .execute();
+
+            return {
+                deviceID: otkInfo.deviceID,
+                index: otkInfo.index,
+                publicKey: XUtils.decodeHex(otkInfo.publicKey),
+                signature: XUtils.decodeHex(otkInfo.signature),
+            };
+        });
     }
 
     public async getOTKCount(deviceID: string): Promise<number> {
@@ -605,12 +718,25 @@ export class Database extends EventEmitter {
         return rows[0] ?? null;
     }
 
-    public async retrieveEmojiList(userID: string): Promise<Emoji[]> {
-        return this.db
+    public async retrieveEmojiList(serverID: string): Promise<Emoji[]> {
+        const now = Date.now();
+        const hit = this.emojiListByServerCache.get(serverID);
+        if (hit && hit.until > now) {
+            return hit.rows.map((r) => ({ ...r }));
+        }
+
+        const rows: Emoji[] = await this.db
             .selectFrom("emojis")
             .selectAll()
-            .where("owner", "=", userID)
+            .where("owner", "=", serverID)
             .execute();
+
+        this.emojiListByServerCache.set(serverID, {
+            rows: rows.map((r) => ({ ...r })),
+            until: now + Database.EMOJI_LIST_CACHE_TTL_MS,
+        });
+
+        return rows.map((r) => ({ ...r }));
     }
 
     public async retrieveFile(fileID: string): Promise<FileSQL | null> {
@@ -752,17 +878,25 @@ export class Database extends EventEmitter {
             .where("serverID", "=", serverID)
             .execute();
 
-        return rows.filter((invite: Invite) => {
-            const valid =
-                new Date(Date.now()).getTime() <
-                new Date(invite.expiration).getTime();
-
-            if (!valid) {
-                void this.deleteInvite(invite.inviteID);
+        const nowMs = Date.now();
+        const expiredIds: string[] = [];
+        const valid: Invite[] = [];
+        for (const row of rows) {
+            const expMs = new Date(row.expiration).getTime();
+            if (!Number.isFinite(expMs) || expMs <= nowMs) {
+                expiredIds.push(row.inviteID);
+                continue;
             }
-
-            return valid;
-        });
+            valid.push(row);
+        }
+        if (expiredIds.length > 0) {
+            await this.db
+                .deleteFrom("invites")
+                .where("serverID", "=", serverID)
+                .where("inviteID", "in", expiredIds)
+                .execute();
+        }
+        return valid;
     }
 
     public async retrieveServers(userID: string): Promise<Server[]> {
@@ -803,13 +937,36 @@ export class Database extends EventEmitter {
     }
 
     public async retrieveUserDeviceList(userIDs: string[]): Promise<Device[]> {
+        const now = Date.now();
+        if (userIDs.length === 1) {
+            const owner = userIDs[0];
+            if (typeof owner === "string") {
+                const hit = this.userDeviceListCache.get(owner);
+                if (hit && hit.until > now) {
+                    return hit.devices.map((d) => ({ ...d }));
+                }
+            }
+        }
+
         const rows = await this.db
             .selectFrom("devices")
             .selectAll()
             .where("owner", "in", userIDs)
             .where("deleted", "=", 0)
             .execute();
-        return rows.map(toDevice);
+        const devices = rows.map(toDevice);
+
+        if (userIDs.length === 1) {
+            const owner = userIDs[0];
+            if (typeof owner === "string") {
+                this.userDeviceListCache.set(owner, {
+                    devices: devices.map((d) => ({ ...d })),
+                    until: now + Database.USER_DEVICE_LIST_CACHE_TTL_MS,
+                });
+            }
+        }
+
+        return devices;
     }
 
     public async retrieveUsers(): Promise<InternalUserRecord[]> {

package/src/Spire.ts

@@ -4,6 +4,7 @@ import type { Server } from "http";
 import { EventEmitter } from "events";
 import { execSync } from "node:child_process";
 import * as fs from "node:fs";
+import { freemem, loadavg, totalmem } from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
@@ -31,7 +32,7 @@ import { z } from "zod/v4";
 import { ClientManager } from "./ClientManager.ts";
 import { Database, hashPasswordArgon2, verifyPassword } from "./Database.ts";
 import { initApp, protect } from "./server/index.ts";
-import { authLimiter } from "./server/rateLimit.ts";
+import { authLimiter, devApiKeySkipsRateLimits } from "./server/rateLimit.ts";
 import { censorUser, getParam, getUser } from "./server/utils.ts";
 import { getJwtSecret } from "./utils/jwtSecret.ts";
 import { msgpack } from "./utils/msgpack.ts";
@@ -190,13 +191,22 @@ export class Spire extends EventEmitter {
 
     private async bootstrapRequestCounter(): Promise<void> {
         const persistedTotal = await this.db.getRequestsTotal();
+
+        // Between the await above and this synchronous block, requests may
+        // have incremented both `requestsTotal` and `queuedRequestIncrements`.
+        // Capture the queue, mark loaded, then merge — never overwrite
+        // `requestsTotal` (which already includes in-flight increments).
         const startupIncrements = this.queuedRequestIncrements;
         this.queuedRequestIncrements = 0;
-        this.requestsTotal = persistedTotal + startupIncrements;
+        this.requestsTotalLoaded = true;
+
+        // Add the persisted baseline on top of whatever the middleware
+        // already counted in-memory, instead of overwriting it.
+        this.requestsTotal += persistedTotal;
+
         if (startupIncrements > 0) {
             await this.db.incrementRequestsTotal(startupIncrements);
         }
-        this.requestsTotalLoaded = true;
     }
 
     private createActionToken(scope: TokenScopes): ActionToken {
@@ -210,8 +220,9 @@ export class Spire extends EventEmitter {
     }
 
     private deleteActionToken(key: ActionToken) {
-        if (this.actionTokens.includes(key)) {
-            this.actionTokens.splice(this.actionTokens.indexOf(key), 1);
+        const idx = this.actionTokens.indexOf(key);
+        if (idx !== -1) {
+            this.actionTokens.splice(idx, 1);
         }
     }
 
@@ -286,11 +297,9 @@ export class Spire extends EventEmitter {
                     );
 
                     client.on("fail", () => {
-                        if (this.clients.includes(client)) {
-                            this.clients.splice(
-                                this.clients.indexOf(client),
-                                1,
-                            );
+                        const idx = this.clients.indexOf(client);
+                        if (idx !== -1) {
+                            this.clients.splice(idx, 1);
                         }
                     });
 
@@ -448,6 +457,68 @@ export class Spire extends EventEmitter {
             });
         });
 
+        /**
+         * Dev-only process snapshot (same gate as rate-limit bypass: `DEV_API_KEY`
+         * env + `x-dev-api-key` header). Returns 404 when not enabled or key wrong
+         * so the route is not advertised to anonymous callers.
+         * Lets local stress / `sample <pid>` workflows see RSS, WS count, etc.
+         */
+        this.api.get("/status/process", (req, res) => {
+            if (!devApiKeySkipsRateLimits(req)) {
+                res.sendStatus(404);
+                return;
+            }
+            const mu = process.memoryUsage();
+            const ru = process.resourceUsage();
+            res.json({
+                activeRequestsApprox: this.requestsTotal,
+                dbReady: this.dbReady,
+                hostOs: {
+                    freemem: freemem(),
+                    loadavg: loadavg(),
+                    totalmem: totalmem(),
+                },
+                memory: {
+                    arrayBuffers: mu.arrayBuffers,
+                    external: mu.external,
+                    heapTotal: mu.heapTotal,
+                    heapUsed: mu.heapUsed,
+                    rss: mu.rss,
+                },
+                pid: process.pid,
+                resourceUsage: {
+                    fsRead: ru.fsRead,
+                    fsWrite: ru.fsWrite,
+                    maxRSS: ru.maxRSS,
+                    systemMicros: ru.systemCPUTime,
+                    userMicros: ru.userCPUTime,
+                },
+                uptimeSeconds: Math.floor(process.uptime()),
+                websocketClients: this.wss.clients.size,
+            });
+        });
+
+        /**
+         * Dev-only SQLite file + pragma snapshot (same gate as `/status/process`).
+         */
+        this.api.get("/status/sqlite", (req, res) => {
+            if (!devApiKeySkipsRateLimits(req)) {
+                res.sendStatus(404);
+                return;
+            }
+            if (!this.dbReady) {
+                res.status(503).json({ dbReady: false, ok: false });
+                return;
+            }
+            try {
+                const sqlite = this.db.getDevSqliteMonitor();
+                res.json({ ok: true, sqlite });
+            } catch (err: unknown) {
+                const msg = err instanceof Error ? err.message : String(err);
+                res.status(500).json({ error: msg, ok: false });
+            }
+        });
+
         this.api.post("/goodbye", protect, (req, res) => {
             jwt.sign({ user: req.user }, getJwtSecret(), { expiresIn: -1 });
             res.sendStatus(200);
@@ -763,7 +834,10 @@ export class Spire extends EventEmitter {
         data?: unknown,
         deviceID?: string,
     ): void {
-        for (const client of this.clients) {
+        // Snapshot the array so that a synchronous `fail` → splice inside
+        // client.send() doesn't corrupt the iteration.
+        const snapshot = this.clients.slice();
+        for (const client of snapshot) {
             if (deviceID) {
                 if (client.getDevice().deviceID === deviceID) {
                     const msg: NotifyMsg = {

package/src/server/index.ts

@@ -197,11 +197,25 @@ export const initApp = (
     api.use(checkAuth);
     api.use(checkDevice);
 
-    const allowedOrigins = process.env["CORS_ORIGINS"]?.split(",") ?? [];
+    // Browser clients (web, Tauri, Capacitor, etc.) hit Spire from arbitrary
+    // origins when self-hosted or embedded in third-party apps. libvex uses
+    // `Authorization: Bearer` only (no cookies), so reflecting `Origin` is the
+    // usual bearer-API pattern. Set `CORS_ORIGINS` to a comma-separated allowlist
+    // when an operator wants to restrict which frontends may call the API.
+    const corsRaw = process.env["CORS_ORIGINS"];
+    const corsOrigins = corsRaw
+        ? corsRaw
+              .split(",")
+              .map((o) => o.trim())
+              .filter((o) => o.length > 0)
+        : [];
     api.use(
         cors({
             credentials: true,
-            origin: allowedOrigins.length > 0 ? allowedOrigins : false,
+            origin:
+                corsOrigins.length > 0
+                    ? corsOrigins
+                    : true /* reflect request Origin */,
         }),
     );
 

package/src/server/rateLimit.ts

@@ -1,3 +1,37 @@
+import type { Request } from "express";
+
+import { timingSafeEqual } from "node:crypto";
+
+import rateLimit, { ipKeyGenerator } from "express-rate-limit";
+
+/** HTTP header carrying the dev API key (must match {@link process.env.DEV_API_KEY}). */
+export const DEV_API_KEY_HEADER = "x-dev-api-key";
+
+/**
+ * When `DEV_API_KEY` is set in the environment, any request whose
+ * `x-dev-api-key` header matches (constant-time) skips all in-process rate
+ * limiters. Dev / load-testing escape hatch only — never set in production.
+ * (Future: first-class API keys with scopes may reuse this header name.)
+ */
+export function devApiKeySkipsRateLimits(req: Request): boolean {
+    const configured = process.env["DEV_API_KEY"]?.trim() ?? "";
+    if (configured.length === 0) {
+        return false;
+    }
+    const presented = req.get(DEV_API_KEY_HEADER);
+    if (!presented || presented.length !== configured.length) {
+        return false;
+    }
+    try {
+        return timingSafeEqual(
+            Buffer.from(presented, "utf8"),
+            Buffer.from(configured, "utf8"),
+        );
+    } catch {
+        return false;
+    }
+}
+
 /**
  * Rate limiting middleware.
  *
@@ -21,9 +55,6 @@
  * `trust proxy` must be set on the Express app (see Spire.ts) so
  * `req.ip` returns the real client address, not the immediate proxy.
  */
-import type { Request } from "express";
-
-import rateLimit, { ipKeyGenerator } from "express-rate-limit";
 
 /**
  * Bucket requests by the real client IP, IPv6-safe.
@@ -46,6 +77,7 @@ export const globalLimiter = rateLimit({
     keyGenerator: keyByIp,
     legacyHeaders: false,
     limit: 3000,
+    skip: devApiKeySkipsRateLimits,
     standardHeaders: "draft-7",
     windowMs: 15 * 60 * 1000,
 });
@@ -63,6 +95,7 @@ export const authLimiter = rateLimit({
     keyGenerator: keyByIp,
     legacyHeaders: false,
     limit: 50,
+    skip: devApiKeySkipsRateLimits,
     skipSuccessfulRequests: true,
     standardHeaders: "draft-7",
     windowMs: 15 * 60 * 1000,
@@ -81,6 +114,7 @@ export const uploadLimiter = rateLimit({
     keyGenerator: keyByIp,
     legacyHeaders: false,
     limit: 200,
+    skip: devApiKeySkipsRateLimits,
     standardHeaders: "draft-7",
     windowMs: 60 * 1000,
 });

package/src/server/user.ts

@@ -31,9 +31,13 @@ export const getUserRouter = (
     });
 
     router.get("/:id/devices", protect, async (req, res) => {
-        const deviceList = await db.retrieveUserDeviceList([
-            getParam(req, "id"),
-        ]);
+        const id = getParam(req, "id");
+        const user = await db.retrieveUser(id);
+        if (!user) {
+            res.sendStatus(404);
+            return;
+        }
+        const deviceList = await db.retrieveUserDeviceList([id]);
         return res.send(msgpack.encode(deviceList));
     });
 

package/dist/__tests__/Database.spec.d.ts

@@ -1 +0,0 @@
-export {};