@decentnetwork/beagle-service 0.0.0-stage → 0.2.0

package/dist/index.js

@@ -0,0 +1,191 @@
+// @decentnetwork/beagle-service — the web backend for a site on Beagle identity.
+//
+// Three parts; a site may use the first alone:
+//
+//   auth   Sign in with Beagle on the server: nonce, exact origin, the two
+//          assertions, a launch replay store that survives a restart, sessions.
+//   users  one record per userid: visitor, registered or blocked; the site's
+//          own registration fields; the visitor limit; the blacklist.
+//   agent  the site's own Carrier identity: who may become a friend, requests
+//          held on disk, a persistent outbox, messages with the sender's record.
+//
+// Nothing here makes one site special. Every key is an equal identity; a site
+// that uses this package is one Carrier service provider among any number.
+import { join } from "node:path";
+import { checkOrigins, createAuth, loadSecret } from "./auth.js";
+import { msgOf } from "./util.js";
+import { createRoutes } from "./routes.js";
+import { jsonFileStateStore, jsonFileUserStore } from "./stores.js";
+import { createUsers } from "./users.js";
+export async function createBeagleService(config) {
+    if (!config || typeof config !== "object")
+        throw new TypeError("beagle-service: createBeagleService needs a config object");
+    const dataDir = config.dataDir ?? "./data";
+    const now = config.now ?? Date.now;
+    const log = config.log ?? ((m) => console.warn(m));
+    // A malformed origin is found here, when the service is created, before a
+    // file is made or a request is answered.
+    checkOrigins(config.origins);
+    const store = config.store ?? jsonFileUserStore(join(dataDir, "users.json"));
+    const state = config.state ?? jsonFileStateStore(join(dataDir, "state.json"), now);
+    const secret = loadSecret(config.sessionSecret, join(dataDir, "session-secret"));
+    const users = createUsers({
+        store,
+        register: config.register,
+        visitor: config.visitor,
+        importProfile: config.importProfile,
+        now,
+        log,
+        fetch: config.fetch,
+    });
+    const auth = createAuth({ origins: config.origins, state, users, secret, serverId: config.serverId, session: config.session, now, log });
+    let agent;
+    const routes = createRoutes({
+        auth,
+        users,
+        session: config.session,
+        publicUser: config.publicUser,
+        onSignIn: config.onSignIn,
+        rateLimit: config.rateLimit,
+        agent: () => agent,
+        log,
+        now,
+    });
+    if (config.agent) {
+        // Only when asked for: a site that needs sign-in alone starts no peer.
+        const { createAgent } = await import("./agent.js");
+        agent = await createAgent({ config: config.agent, users, dir: join(dataDir, "agent"), now, log, fetch: config.fetch });
+    }
+    return {
+        users,
+        auth,
+        agent,
+        handle: routes.handle,
+        handlers: routes.handlers,
+        nodeHandler: routes.nodeHandler,
+        expressHandler: routes.expressHandler,
+        nextApiHandler: routes.nextApiHandler,
+        fetchHandler: routes.fetchHandler,
+        route: routes.route,
+        close: async () => {
+            await agent?.stop();
+        },
+    };
+}
+// ── one service per process ─────────────────────────────────────────────────
+const SHARED = Symbol.for("@decentnetwork/beagle-service/shared");
+const UNAVAILABLE = {
+    status: 500,
+    headers: { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" },
+    body: { ok: false, error: "server-error", message: "Sign-in is not available on this server." },
+};
+/** The service for this process: created on first use, and the same one for
+ *  every caller with the same `key`, however many times the calling module is
+ *  evaluated (Next.js bundles pages and API routes apart) and whichever of
+ *  `import` and `require` loaded this package.
+ *
+ *  Returns at once and never throws. A configuration that is wrong shows in
+ *  `ready()`, which rejects; the routes then answer 500 and `authenticate`
+ *  answers `ok: false`. A failed start is not kept: it is tried again, at most
+ *  every `retryMs`.
+ *
+ *  `config` may be a function, so that reading the environment happens at
+ *  first use and not when the module is loaded. The first call for a key
+ *  decides the configuration. */
+export function sharedBeagleService(config, opts = {}) {
+    const registry = (globalThis[SHARED] ??= new Map());
+    const key = opts.key ?? "default";
+    const held = registry.get(key);
+    if (held)
+        return held;
+    const retryMs = opts.retryMs ?? 5000;
+    let starting = null;
+    let failure = null;
+    let complainedAt = 0;
+    function ready() {
+        if (starting)
+            return starting;
+        if (failure && Date.now() - failure.at < retryMs)
+            return Promise.reject(failure.error);
+        const attempt = Promise.resolve().then(() => createBeagleService(typeof config === "function" ? config() : config));
+        starting = attempt;
+        attempt.then(() => (failure = null), (error) => {
+            // Not kept: the next call after `retryMs` starts again.
+            if (starting === attempt)
+                starting = null;
+            failure = { error, at: Date.now() };
+            if (Date.now() - complainedAt > 60_000) {
+                complainedAt = Date.now();
+                console.error(`beagle-service: the service could not start: ${msgOf(error)}`);
+            }
+        });
+        return attempt;
+    }
+    const down = (res) => {
+        if (res.headersSent)
+            return;
+        res.statusCode = UNAVAILABLE.status;
+        for (const [k, v] of Object.entries(UNAVAILABLE.headers))
+            res.setHeader(k, v);
+        res.end(JSON.stringify(UNAVAILABLE.body));
+    };
+    const node = (pick) => async (req, res, next) => {
+        let svc;
+        try {
+            svc = await ready();
+        }
+        catch {
+            return down(res);
+        }
+        return pick(svc)(req, res, next);
+    };
+    const shared = {
+        ready,
+        async authenticate(req, o) {
+            try {
+                return await (await ready()).auth.authenticate(req, o);
+            }
+            catch {
+                return { ok: false, status: 500, error: "server-error" };
+            }
+        },
+        async handle(req) {
+            try {
+                return await (await ready()).handle(req);
+            }
+            catch {
+                return UNAVAILABLE;
+            }
+        },
+        route: (name) => node((svc) => svc.route(name)),
+        nodeHandler: (o) => node((svc) => svc.nodeHandler(o)),
+        expressHandler: (o) => node((svc) => svc.expressHandler(o)),
+        nextApiHandler: (o) => node((svc) => svc.nextApiHandler(o)),
+        fetchHandler: (o) => async (request) => {
+            try {
+                return await (await ready()).fetchHandler(o)(request);
+            }
+            catch {
+                return new Response(JSON.stringify(UNAVAILABLE.body), { status: UNAVAILABLE.status, headers: UNAVAILABLE.headers });
+            }
+        },
+        async close() {
+            if (registry.get(key) === shared)
+                registry.delete(key);
+            const svc = await starting?.catch(() => null);
+            starting = null;
+            await svc?.close();
+        },
+    };
+    registry.set(key, shared);
+    return shared;
+}
+export { createAuth, normalizeOrigins, NONCE_TTL_MS, LAUNCH_MAX_AGE_MS } from "./auth.js";
+export { createUsers, DEFAULT_PROFILE_GATEWAY } from "./users.js";
+export { createAgent, loadServers, DEFAULT_SERVERS_URL } from "./agent.js";
+export { Outbox, splitText } from "./outbox.js";
+export { jsonFileUserStore, memoryUserStore, jsonFileStateStore, memoryStateStore, applyUserChange } from "./stores.js";
+export { defaultPublicUser } from "./routes.js";
+// `export *` rather than `export type *`: types.ts holds types only, and this
+// form is read by TypeScript versions before 5.0 too.
+export * from "./types.js";

package/dist/outbox.d.ts

@@ -0,0 +1,63 @@
+import type { DeadLetter, DeadLetterReason, DeliveryEvent, OutboxItem } from "./types.js";
+import { type Logger } from "./util.js";
+/** Cut `text` into parts that each fit one message (about 700 bytes of
+ *  UTF-8), at a line break or a space where one is near. The parts joined
+ *  together are the text, unchanged. */
+export declare function splitText(text: string): string[];
+export interface OutboxTransport {
+    /** "friend": the friendship is settled. "pending": a request is out, or one
+     *  of theirs is held. "none": there is no way a message could reach them. */
+    relation(userid: string): "friend" | "pending" | "none";
+    /** A live session that carried something from the friend a moment ago. */
+    proven(userid: string): boolean;
+    /** Live send with a short confirmation window; throws when nothing took it. */
+    sendLive(userid: string, text: string, deliveryId: string): Promise<"acked" | "accepted" | "offline">;
+    /** Post to an Express node whatever the session looks like; throws when it cannot. */
+    sendExpress(userid: string, text: string, deliveryId: string): Promise<void>;
+}
+export interface OutboxOptions {
+    file: string;
+    deadFile: string;
+    transport: OutboxTransport;
+    now?: () => number;
+    log?: Logger;
+    baseDelayMs?: number;
+    maxDelayMs?: number;
+    maxAgeMs?: number;
+    onDelivered?: (e: DeliveryEvent) => void;
+    onDead?: (d: DeadLetter) => void;
+}
+/**
+ * Messages on their way out, kept in one JSON file.
+ *
+ * - A message is written to disk before `enqueue` returns and removed only
+ *   after a road took it, so a restart loses nothing.
+ * - Its `deliveryId` is fixed when it is queued. A retry, a second road or a
+ *   send repeated after a crash carries the same id, and a friend that reads
+ *   the delivery envelope shows it once.
+ * - One queue per recipient: a recipient's messages go in order, and an
+ *   unreachable recipient never holds up another.
+ * - The road (the rule the agentnet daemon uses): a session that has just
+ *   proven itself gets the message live, with a short wait for the
+ *   confirmation; otherwise, or when no confirmation comes, it is posted to
+ *   an Express node at once. Only when neither road takes it does it wait,
+ *   with a delay that doubles up to `maxDelayMs`.
+ * - What can never arrive is moved to the dead-letter file with the reason.
+ */
+export declare class Outbox {
+    #private;
+    constructor(o: OutboxOptions);
+    list(): OutboxItem[];
+    deadLetters(limit?: number): DeadLetter[];
+    /** Put a text on disk for `to`. Synchronous: when it returns, the message survives a crash. */
+    enqueue(to: string, text: string): OutboxItem[];
+    /** Record a message that cannot be sent at all, without queueing it. */
+    refuse(to: string, text: string, why: DeadLetterReason): OutboxItem[];
+    /** Dead-letter everything queued for `to`. */
+    dropFor(to: string, why: DeadLetterReason): number;
+    /** Something changed for `to` (accepted, connected): try now, not after the delay. */
+    kick(to: string): void;
+    stop(): void;
+    /** Try the head of every recipient's queue that is due. */
+    pump(): Promise<void>;
+}

package/dist/outbox.js

@@ -0,0 +1,314 @@
+import { appendFileSync, mkdirSync, readFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { randomBytes } from "node:crypto";
+import { atomicWriteJson, msgOf, readJsonFile } from "./util.js";
+// ── splitting ───────────────────────────────────────────────────────────────
+//
+// A Carrier friend message that stays one packet carries at most 1024 bytes of
+// text. A friend that reads the delivery envelope gets
+// "\x1eDNPACK1:" + base64url(JSON {t:"msg", id, text}), so the JSON may be at
+// most floor((1024 - 9) * 3 / 4) = 761 bytes; an envelope-less friend gets the
+// text itself. Each part is cut to fit the tighter of the two, so it travels
+// as one packet on either road and to either kind of friend.
+const ENVELOPE_JSON_MAX = 761;
+const ID_RESERVE = "x".repeat(48);
+const fits = (text) => Buffer.byteLength(JSON.stringify({ t: "msg", id: ID_RESERVE, text })) <= ENVELOPE_JSON_MAX;
+/** Cut `text` into parts that each fit one message (about 700 bytes of
+ *  UTF-8), at a line break or a space where one is near. The parts joined
+ *  together are the text, unchanged. */
+export function splitText(text) {
+    if (fits(text))
+        return [text];
+    const cps = Array.from(text); // code points: never cut inside a surrogate pair
+    const parts = [];
+    let start = 0;
+    while (start < cps.length) {
+        let lo = 1;
+        let hi = Math.min(cps.length - start, ENVELOPE_JSON_MAX);
+        while (lo < hi) {
+            const mid = (lo + hi + 1) >> 1;
+            if (fits(cps.slice(start, start + mid).join("")))
+                lo = mid;
+            else
+                hi = mid - 1;
+        }
+        let n = lo;
+        if (start + n < cps.length) {
+            // Prefer to end the part after a line break, else after a space, when
+            // one lies in its last third.
+            const floor = Math.floor(n * (2 / 3));
+            let cut = -1;
+            for (let i = n - 1; i >= floor; i--) {
+                if (cps[start + i] === "\n") {
+                    cut = i + 1;
+                    break;
+                }
+            }
+            if (cut < 0) {
+                for (let i = n - 1; i >= floor; i--) {
+                    if (cps[start + i] === " ") {
+                        cut = i + 1;
+                        break;
+                    }
+                }
+            }
+            if (cut > 0)
+                n = cut;
+        }
+        parts.push(cps.slice(start, start + n).join(""));
+        start += n;
+    }
+    return parts;
+}
+/**
+ * Messages on their way out, kept in one JSON file.
+ *
+ * - A message is written to disk before `enqueue` returns and removed only
+ *   after a road took it, so a restart loses nothing.
+ * - Its `deliveryId` is fixed when it is queued. A retry, a second road or a
+ *   send repeated after a crash carries the same id, and a friend that reads
+ *   the delivery envelope shows it once.
+ * - One queue per recipient: a recipient's messages go in order, and an
+ *   unreachable recipient never holds up another.
+ * - The road (the rule the agentnet daemon uses): a session that has just
+ *   proven itself gets the message live, with a short wait for the
+ *   confirmation; otherwise, or when no confirmation comes, it is posted to
+ *   an Express node at once. Only when neither road takes it does it wait,
+ *   with a delay that doubles up to `maxDelayMs`.
+ * - What can never arrive is moved to the dead-letter file with the reason.
+ */
+export class Outbox {
+    #o;
+    #items;
+    #busy = new Set();
+    /** The message being sent to each recipient right now. */
+    #inFlight = new Map();
+    /** Recipients whose queue was dropped while a send was in flight, and why. */
+    #dropAfter = new Map();
+    #stopped = false;
+    #now;
+    #log;
+    #base;
+    #max;
+    #maxAge;
+    constructor(o) {
+        this.#o = o;
+        this.#now = o.now ?? Date.now;
+        this.#log = o.log ?? (() => { });
+        this.#base = o.baseDelayMs ?? 5_000;
+        this.#max = o.maxDelayMs ?? 10 * 60_000;
+        this.#maxAge = o.maxAgeMs ?? 72 * 3_600_000;
+        const loaded = readJsonFile(o.file, []);
+        this.#items = Array.isArray(loaded) ? loaded.filter((i) => i && typeof i.to === "string" && typeof i.text === "string" && typeof i.deliveryId === "string") : [];
+        // Whatever delay was running when the process ended is over.
+        for (const i of this.#items)
+            i.nextAt = 0;
+    }
+    list() {
+        return this.#items.map((i) => ({ ...i }));
+    }
+    deadLetters(limit = 500) {
+        let raw = "";
+        try {
+            raw = readFileSync(this.#o.deadFile, "utf8");
+        }
+        catch {
+            return [];
+        }
+        const out = [];
+        for (const line of raw.split("\n")) {
+            if (!line)
+                continue;
+            try {
+                out.push(JSON.parse(line));
+            }
+            catch {
+                /* a torn last line */
+            }
+        }
+        return out.slice(-limit);
+    }
+    #make(to, text) {
+        const id = `${this.#now().toString(36)}${randomBytes(9).toString("base64url")}`;
+        const texts = splitText(text);
+        const createdAt = new Date(this.#now()).toISOString();
+        return texts.map((t, part) => ({
+            id,
+            part,
+            parts: texts.length,
+            to,
+            text: t,
+            deliveryId: `bs-${id}-${part}`,
+            createdAt,
+            attempts: 0,
+            nextAt: 0,
+        }));
+    }
+    /** Put a text on disk for `to`. Synchronous: when it returns, the message survives a crash. */
+    enqueue(to, text) {
+        const items = this.#make(to, text);
+        this.#items.push(...items);
+        this.#persist();
+        return items.map((i) => ({ ...i }));
+    }
+    /** Record a message that cannot be sent at all, without queueing it. */
+    refuse(to, text, why) {
+        const items = this.#make(to, text);
+        for (const i of items)
+            this.#deadLetter(i, why);
+        return items;
+    }
+    /** Dead-letter everything queued for `to`. */
+    dropFor(to, why) {
+        // A message already on the wire is not called dead: it may well arrive.
+        // Its attempt finishes; only if it fails is it dead-lettered too.
+        const flying = this.#inFlight.get(to);
+        if (flying)
+            this.#dropAfter.set(to, why);
+        const mine = this.#items.filter((i) => i.to === to && i !== flying);
+        if (!mine.length)
+            return 0;
+        this.#items = this.#items.filter((i) => i.to !== to || i === flying);
+        this.#persist();
+        for (const i of mine)
+            this.#deadLetter(i, why);
+        return mine.length;
+    }
+    /** Something changed for `to` (accepted, connected): try now, not after the delay. */
+    kick(to) {
+        for (const i of this.#items)
+            if (i.to === to)
+                i.nextAt = 0;
+        void this.pump();
+    }
+    stop() {
+        this.#stopped = true;
+    }
+    /** Try the head of every recipient's queue that is due. */
+    async pump() {
+        if (this.#stopped)
+            return;
+        const t = this.#now();
+        const due = new Set();
+        const seen = new Set();
+        for (const i of this.#items) {
+            if (seen.has(i.to))
+                continue;
+            seen.add(i.to);
+            if (!this.#busy.has(i.to) && i.nextAt <= t)
+                due.add(i.to);
+        }
+        await Promise.all([...due].map((to) => this.#drain(to)));
+    }
+    async #drain(to) {
+        this.#busy.add(to);
+        try {
+            for (;;) {
+                if (this.#stopped)
+                    return;
+                const head = this.#items.find((i) => i.to === to);
+                if (!head || head.nextAt > this.#now())
+                    return;
+                if (!(await this.#deliver(head)))
+                    return;
+            }
+        }
+        catch (e) {
+            this.#log(`beagle-service: outbox for ${to.slice(0, 8)} failed: ${msgOf(e)}`);
+        }
+        finally {
+            this.#busy.delete(to);
+        }
+    }
+    /** true when the item left the queue (delivered or dead-lettered). */
+    async #deliver(item) {
+        const t = this.#o.transport;
+        const relation = t.relation(item.to);
+        if (relation === "none")
+            return this.#settleDead(item, "not-a-friend");
+        if (this.#now() - Date.parse(item.createdAt) > this.#maxAge) {
+            return this.#settleDead(item, relation === "pending" ? "not-a-friend" : "expired", item.lastError);
+        }
+        if (relation === "pending")
+            return this.#wait(item, "the friend request is not accepted yet");
+        let result;
+        this.#inFlight.set(item.to, item);
+        try {
+            result = await this.#road(item);
+        }
+        catch (e) {
+            const err = msgOf(e);
+            const dropped = this.#dropAfter.get(item.to);
+            if (dropped)
+                return this.#settleDead(item, dropped, err);
+            // The far side dropped us between the check and the send.
+            if (/not a friend/i.test(err) && t.relation(item.to) === "none")
+                return this.#settleDead(item, "not-a-friend", err);
+            return this.#wait(item, err);
+        }
+        finally {
+            this.#inFlight.delete(item.to);
+            this.#dropAfter.delete(item.to);
+        }
+        this.#remove(item);
+        this.#o.onDelivered?.({ id: item.id, deliveryId: item.deliveryId, to: item.to, ...result });
+        return true;
+    }
+    async #road(item) {
+        const t = this.#o.transport;
+        const { to, text, deliveryId } = item;
+        const live = async () => {
+            const how = await t.sendLive(to, text, deliveryId);
+            return { via: how === "offline" ? "offline" : "online", confirmed: how === "acked" };
+        };
+        const express = async () => {
+            await t.sendExpress(to, text, deliveryId);
+            return { via: "offline", confirmed: false };
+        };
+        const [first, second, a, b] = t.proven(to) ? [live, express, "live", "express"] : [express, live, "express", "live"];
+        try {
+            return await first();
+        }
+        catch (e1) {
+            try {
+                return await second();
+            }
+            catch (e2) {
+                throw new Error(`${a}: ${msgOf(e1)}; ${b}: ${msgOf(e2)}`);
+            }
+        }
+    }
+    #wait(item, error) {
+        item.attempts += 1;
+        item.lastError = error;
+        item.nextAt = this.#now() + Math.min(this.#max, this.#base * 2 ** Math.min(item.attempts - 1, 20));
+        this.#persist();
+        return false;
+    }
+    #settleDead(item, why, error) {
+        this.#remove(item);
+        this.#deadLetter(item, why, error);
+        return true;
+    }
+    #remove(item) {
+        const i = this.#items.indexOf(item);
+        if (i >= 0)
+            this.#items.splice(i, 1);
+        this.#persist();
+    }
+    #deadLetter(item, why, error) {
+        const letter = { at: new Date(this.#now()).toISOString(), why, ...(error ? { error } : {}), item: { ...item } };
+        try {
+            mkdirSync(dirname(this.#o.deadFile), { recursive: true });
+            appendFileSync(this.#o.deadFile, JSON.stringify(letter) + "\n", { mode: 0o600 });
+        }
+        catch (e) {
+            this.#log(`beagle-service: could not write the dead-letter file: ${msgOf(e)}`);
+        }
+        this.#log(`beagle-service: dead-letter (${why}) for ${item.to.slice(0, 8)}${error ? `: ${error}` : ""}`);
+        this.#o.onDead?.(letter);
+    }
+    #persist() {
+        atomicWriteJson(this.#o.file, this.#items);
+    }
+}

package/dist/routes.d.ts

@@ -0,0 +1,29 @@
+import type { AdapterOptions, Agent, Auth, Handlers, NodeHandler, RateLimitConfig, RouteHandler, RouteName, ServiceRequest, SessionConfig, SignInHook, UserRecord, Users } from "./types.js";
+import { type Logger } from "./util.js";
+/** The library's own fields only: a site's extra fields (a plan, notes) do
+ *  not reach the page unless the site says so with `publicUser`. */
+export declare function defaultPublicUser(user: UserRecord): Record<string, unknown>;
+export interface RoutesOptions {
+    auth: Auth;
+    users: Users;
+    session?: SessionConfig;
+    publicUser?: (user: UserRecord) => Record<string, unknown>;
+    onSignIn?: SignInHook;
+    rateLimit?: RateLimitConfig | false;
+    agent: () => Agent | undefined;
+    log: Logger;
+    now?: () => number;
+}
+/** Counts sign-in and launch attempts in fixed windows, per client and for
+ *  all clients together. Returns the seconds to wait when a request is over
+ *  the limit, else 0. In memory: each server process counts for itself. */
+export declare function createRateLimit(config: RateLimitConfig | false | undefined, now: () => number): (req: ServiceRequest) => number;
+export declare function createRoutes(opts: RoutesOptions): {
+    handle: RouteHandler;
+    handlers: Handlers;
+    nodeHandler: (o?: AdapterOptions) => NodeHandler;
+    expressHandler: (o?: AdapterOptions) => NodeHandler;
+    nextApiHandler: (o?: AdapterOptions) => NodeHandler;
+    fetchHandler: (o?: AdapterOptions) => (request: Request) => Promise<Response>;
+    route: (name: RouteName) => NodeHandler;
+};