@clubedge/create-clubedge-app 0.6.0 → 0.7.0

package/template/packages/cache-redis/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clubedge/cache-redis",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Redis rate limiter and cache for @clubedge/cache.",
   "private": true,
   "license": "Apache-2.0",

package/template/packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clubedge/core",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "private": true,
   "license": "Apache-2.0",
   "type": "module",

package/template/packages/db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clubedge/db",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "private": true,
   "license": "Apache-2.0",
   "type": "module",

package/template/packages/local-db/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@clubedge/local-db",
+  "version": "0.7.0",
+  "description": "PostgreSQL for development without Docker: PGlite served over the PostgreSQL protocol.",
+  "private": true,
+  "license": "Apache-2.0",
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": "./src/server.ts"
+  },
+  "scripts": {
+    "dev": "tsx scripts/serve.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@electric-sql/pglite": "0.5.8"
+  },
+  "devDependencies": {
+    "postgres": "^3.4.7",
+    "tsx": "^4.20.5",
+    "typescript": "^5.9.2"
+  }
+}

package/template/packages/local-db/scripts/serve.ts

@@ -0,0 +1,66 @@
+// Runs PostgreSQL in this process with PGlite and serves it over the PostgreSQL protocol, so the
+// app, migrations, and Drizzle Kit connect with an ordinary DATABASE_URL. For development only:
+// PGlite has a single session, so clients take turns (see src/server.ts).
+import { spawn } from "node:child_process";
+import { mkdirSync } from "node:fs";
+import { connect } from "node:net";
+import { resolve } from "node:path";
+import { PGlite } from "@electric-sql/pglite";
+import { serveLocalPostgres } from "../src/server";
+
+const host = "127.0.0.1";
+const port = Number(process.env.LOCAL_DB_PORT ?? 5433);
+const root = resolve(import.meta.dirname, "../../..");
+const dataDir = resolve(root, ".data/pglite");
+const url = `postgresql://postgres:postgres@${host}:${port}/postgres`;
+
+/** Resolves true when something already accepts connections on the port. */
+function isPortInUse(): Promise<boolean> {
+  return new Promise((done) => {
+    const socket = connect({ host, port });
+    socket.once("connect", () => {
+      socket.end();
+      done(true);
+    });
+    socket.once("error", () => done(false));
+  });
+}
+
+/** Applies every package's migrations, like `pnpm db:migrate`, against this server. */
+function migrate(): Promise<boolean> {
+  return new Promise((done) => {
+    const child = spawn("pnpm", ["run", "db:migrate"], {
+      cwd: root,
+      env: { ...process.env, DATABASE_URL: url },
+      shell: process.platform === "win32",
+      stdio: "inherit",
+    });
+    child.once("error", () => done(false));
+    child.once("exit", (code) => done(code === 0));
+  });
+}
+
+// Check the port before opening the data directory: two processes must never open it at once.
+if (await isPortInUse()) {
+  console.info(`Port ${port} is in use; the local database is probably already running.`);
+  process.exit(0);
+}
+
+mkdirSync(dataDir, { recursive: true });
+const db = await PGlite.create(dataDir);
+const server = await serveLocalPostgres(db, { host, port });
+console.info(`Local PostgreSQL (PGlite) on ${url}, data in .data/pglite`);
+
+let stopping = false;
+async function stop() {
+  if (stopping) return;
+  stopping = true;
+  // Close PGlite cleanly so its data directory stays consistent.
+  await server.close();
+  await db.close();
+  process.exit(0);
+}
+for (const signal of ["SIGINT", "SIGTERM", "SIGBREAK"] as const) process.on(signal, stop);
+
+if (await migrate()) console.info("Local database ready.");
+else console.error("Migrations failed; the database is still running.");

package/template/packages/local-db/src/server.test.ts

@@ -0,0 +1,119 @@
+import { PGlite } from "@electric-sql/pglite";
+import postgres from "postgres";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { serveLocalPostgres, type LocalPostgresServer } from "./server";
+
+describe("serveLocalPostgres", () => {
+  let db: PGlite;
+  let server: LocalPostgresServer;
+  let url: string;
+
+  beforeAll(async () => {
+    db = await PGlite.create();
+    server = await serveLocalPostgres(db, { port: 0 });
+    url = `postgresql://postgres:postgres@127.0.0.1:${server.port}/postgres`;
+  });
+
+  afterAll(async () => {
+    await server.close();
+    await db.close();
+  });
+
+  it("answers queries from several clients at once", async () => {
+    // Prepared statements off, as packages/db configures postgres.js.
+    const clients = [1, 2, 3].map(() => postgres(url, { max: 4, prepare: false }));
+    try {
+      // Statements with and without parameters, so one client's statement can never stand in
+      // for another's.
+      const results = await Promise.all(
+        Array.from({ length: 60 }, (_, i) =>
+          i % 2
+            ? clients[i % 3]!`select ${i}::int as n, ${`value ${i}`}::text as label`
+            : clients[i % 3]!`select 0 as n, 'constant' as label`,
+        ),
+      );
+      results.forEach(([row], i) =>
+        expect(row).toEqual(i % 2 ? { n: i, label: `value ${i}` } : { n: 0, label: "constant" }),
+      );
+    } finally {
+      await Promise.all(clients.map((client) => client.end()));
+    }
+  });
+
+  it("keeps other clients out of an open transaction", async () => {
+    const a = postgres(url, { max: 4, prepare: false });
+    const b = postgres(url, { max: 4, prepare: false });
+    try {
+      await a`create table entries (source text not null)`;
+      const work: Promise<unknown>[] = [];
+      for (let i = 0; i < 12; i++) {
+        work.push(
+          a
+            .begin(async (sql) => {
+              await sql`insert into entries values ('transaction')`;
+              await sql`select pg_sleep(0.005)`;
+              if (i % 3 === 0) throw new Error("roll back");
+            })
+            .catch(() => undefined),
+        );
+        work.push(b`insert into entries values ('outside')`);
+      }
+      await Promise.all(work);
+
+      const rows = await a<{ source: string; count: number }[]>`
+        select source, count(*)::int as count from entries group by source order by source`;
+      // Rolled-back transactions take only their own inserts with them.
+      expect(rows).toEqual([
+        { source: "outside", count: 12 },
+        { source: "transaction", count: 8 },
+      ]);
+    } finally {
+      await a.end();
+      await b.end();
+    }
+  });
+
+  it("rolls back a transaction left open by a client that disconnects", async () => {
+    const a = postgres(url, { max: 1, prepare: false });
+    await a`create table drafts (id int)`;
+    const reserved = await a.reserve();
+    await reserved`begin`;
+    await reserved`insert into drafts values (1)`;
+    await a.end({ timeout: 0 });
+
+    const b = postgres(url, { max: 1, prepare: false });
+    try {
+      expect(await b`select count(*)::int as count from drafts`).toEqual([{ count: 0 }]);
+    } finally {
+      await b.end();
+    }
+  });
+});
+
+describe("LocalPostgresServer.close", () => {
+  it("finishes the rollback of open transactions before it resolves", async () => {
+    const db = await PGlite.create();
+    const server = await serveLocalPostgres(db, { port: 0 });
+    const client = postgres(`postgresql://postgres:postgres@127.0.0.1:${server.port}/postgres`, {
+      max: 1,
+      prepare: false,
+    });
+    try {
+      const reserved = await client.reserve();
+      await reserved`begin`;
+      await reserved`create table pending (id int)`;
+      expect(db.isInTransaction()).toBe(true);
+
+      await server.close();
+
+      // The caller can close PGlite right away: nothing is left running on it.
+      expect(db.isInTransaction()).toBe(false);
+      expect((await db.query("select to_regclass('pending') as name")).rows).toEqual([
+        { name: null },
+      ]);
+    } finally {
+      await client.end({ timeout: 0 });
+      await db.close();
+    }
+  });
+});

package/template/packages/local-db/src/server.ts

@@ -0,0 +1,153 @@
+import { createServer, type Server, type Socket } from "node:net";
+import type { PGlite } from "@electric-sql/pglite";
+
+// Request codes sent instead of a startup message (PostgreSQL protocol, "Message Formats").
+const SSL_REQUEST = 80877103;
+const GSSENC_REQUEST = 80877104;
+const CANCEL_REQUEST = 80877102;
+// Messages that send a request off: Sync, simple Query, Flush, Terminate, CopyDone, CopyFail.
+const REQUEST_END = new Set(["S", "Q", "H", "X", "c", "f"].map((type) => type.charCodeAt(0)));
+const FLUSH = "H".charCodeAt(0);
+
+export interface LocalPostgresServer {
+  port: number;
+  close(): Promise<void>;
+}
+
+/**
+ * Serves a PGlite database over the PostgreSQL protocol so ordinary clients (postgres.js, Drizzle
+ * Kit, psql) can share it. PGlite has a single session, so clients take turns: a client keeps the
+ * session from its first message until its Sync or simple Query, including across Flush (clients
+ * such as postgres.js describe a statement, then bind it), and for as long as its transaction is
+ * open. Other clients wait meanwhile.
+ */
+export async function serveLocalPostgres(
+  db: PGlite,
+  { host = "127.0.0.1", port = 5433 }: { host?: string; port?: number } = {},
+): Promise<LocalPostgresServer> {
+  await db.waitReady;
+
+  // The connection that may use the session, and the connections waiting for it.
+  let owner: Socket | null = null;
+  const waiting: Array<() => void> = [];
+
+  async function acquire(socket: Socket) {
+    while (owner && owner !== socket) await new Promise<void>((wake) => waiting.push(wake));
+    owner = socket;
+  }
+
+  function release(socket: Socket) {
+    if (owner !== socket) return;
+    owner = null;
+    for (const wake of waiting.splice(0)) wake();
+  }
+
+  async function run(socket: Socket, messages: Uint8Array[]) {
+    // Requests still queued when their client disconnected are dropped.
+    if (socket.destroyed) return;
+    await acquire(socket);
+    try {
+      for (const message of messages) {
+        await db.execProtocolRawStream(message, {
+          onRawData: (data) => {
+            if (socket.writable) socket.write(data);
+          },
+        });
+      }
+    } finally {
+      // After a Flush the client continues the same request, so it keeps the session.
+      const flushed = messages.at(-1)?.[0] === FLUSH;
+      if (!flushed && !db.isInTransaction()) release(socket);
+    }
+  }
+
+  async function disconnect(socket: Socket) {
+    if (owner !== socket) return;
+    if (db.isInTransaction()) await db.exec("ROLLBACK");
+    release(socket);
+  }
+
+  const sockets = new Set<Socket>();
+  // Settles once a connection's queued work, including its disconnect cleanup, has finished.
+  const connections = new Set<Promise<void>>();
+  const server: Server = createServer((socket) => {
+    sockets.add(socket);
+    socket.setNoDelay(true);
+    let buffer = Buffer.alloc(0);
+    let started = false;
+    let request: Uint8Array[] = [];
+    // Requests from one connection run in order.
+    let queue = Promise.resolve();
+
+    function enqueue(messages: Uint8Array[]) {
+      queue = queue.then(() => run(socket, messages)).catch(() => void socket.destroy());
+    }
+
+    socket.on("data", (chunk) => {
+      buffer = Buffer.concat([buffer, chunk]);
+      while (true) {
+        if (!started) {
+          // Startup-phase messages have no type byte: a length, then a protocol or request code.
+          if (buffer.length < 8) return;
+          const length = buffer.readInt32BE(0);
+          if (buffer.length < length) return;
+          const code = buffer.readInt32BE(4);
+          const message = buffer.subarray(0, length);
+          buffer = buffer.subarray(length);
+          if (code === SSL_REQUEST || code === GSSENC_REQUEST) {
+            socket.write("N"); // Encryption is not supported; the client continues in plain text.
+          } else if (code === CANCEL_REQUEST) {
+            socket.end(); // Cancellation is not supported.
+            return;
+          } else {
+            started = true;
+            enqueue([new Uint8Array(message)]);
+          }
+          continue;
+        }
+        if (buffer.length < 5) return;
+        const length = 1 + buffer.readInt32BE(1);
+        if (buffer.length < length) return;
+        const type = buffer[0]!;
+        request.push(new Uint8Array(buffer.subarray(0, length)));
+        buffer = buffer.subarray(length);
+        if (REQUEST_END.has(type)) {
+          enqueue(request);
+          request = [];
+        }
+        if (type === "X".charCodeAt(0)) {
+          queue = queue.then(() => void socket.end());
+          return;
+        }
+      }
+    });
+
+    const finished = new Promise<void>((resolve) => {
+      socket.on("close", () => {
+        sockets.delete(socket);
+        queue = queue.then(() => disconnect(socket)).catch(() => undefined);
+        void queue.then(resolve);
+      });
+    });
+    connections.add(finished);
+    void finished.then(() => connections.delete(finished));
+    socket.on("error", () => socket.destroy());
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(port, host, () => resolve());
+  });
+  const address = server.address();
+
+  return {
+    port: typeof address === "object" && address ? address.port : port,
+    async close() {
+      const closed = new Promise<void>((resolve) => server.close(() => resolve()));
+      for (const socket of sockets) socket.destroy();
+      await closed;
+      // Let queued work and rollbacks finish, so the caller can then close PGlite safely.
+      await Promise.all(connections);
+    },
+  };
+}

package/template/packages/local-db/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "include": ["src/**/*.ts", "scripts/**/*.ts"]
+}

package/template/packages/storage/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@clubedge/storage",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "private": true,
   "license": "Apache-2.0",
   "type": "module",
   "sideEffects": false,
   "exports": {
-    ".": "./src/index.ts"
+    ".": "./src/index.ts",
+    "./file-system": "./src/file-system.ts"
   },
   "scripts": {
     "typecheck": "tsc --noEmit"

package/template/packages/storage/src/file-system.test.ts

@@ -0,0 +1,110 @@
+import { mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { createFileSystemStorage } from "./file-system";
+
+describe("createFileSystemStorage", () => {
+  let directory: string;
+
+  beforeEach(async () => {
+    directory = await mkdtemp(join(tmpdir(), "clubedge-storage-"));
+  });
+
+  afterEach(async () => {
+    await rm(directory, { recursive: true, force: true });
+  });
+
+  it("uploads, reads back, and deletes objects in nested folders", async () => {
+    const storage = createFileSystemStorage({ directory });
+    await storage.upload({
+      key: "avatars/user-1.txt",
+      body: new TextEncoder().encode("hello"),
+      contentType: "text/plain",
+    });
+
+    expect(await readFile(join(directory, "avatars", "user-1.txt"), "utf8")).toBe("hello");
+    expect(await storage.exists("avatars/user-1.txt")).toBe(true);
+    expect(await new Response(await storage.download("avatars/user-1.txt")).text()).toBe("hello");
+
+    await storage.delete("avatars/user-1.txt");
+    expect(await storage.exists("avatars/user-1.txt")).toBe(false);
+  });
+
+  it("accepts blobs and streams as bodies", async () => {
+    const storage = createFileSystemStorage({ directory });
+    await storage.upload({
+      key: "blob.txt",
+      body: new Blob(["from a blob"]),
+      contentType: "text/plain",
+    });
+    await storage.upload({
+      key: "stream.txt",
+      body: new Blob(["from a stream"]).stream(),
+      contentType: "text/plain",
+    });
+
+    expect(await readFile(join(directory, "blob.txt"), "utf8")).toBe("from a blob");
+    expect(await readFile(join(directory, "stream.txt"), "utf8")).toBe("from a stream");
+  });
+
+  it("rejects keys that would leave the folder", async () => {
+    const storage = createFileSystemStorage({ directory });
+    const body = new Uint8Array();
+    for (const key of ["../outside.txt", "a/../../outside.txt", "", join(directory, "..", "x")]) {
+      await expect(storage.upload({ key, body, contentType: "text/plain" })).rejects.toThrow(
+        "Invalid storage key",
+      );
+    }
+  });
+
+  it("leaves no file behind when a streamed upload fails", async () => {
+    const storage = createFileSystemStorage({ directory });
+    const failing = new ReadableStream<Uint8Array>({
+      start(controller) {
+        controller.enqueue(new TextEncoder().encode("partial"));
+        controller.error(new Error("connection lost"));
+      },
+    });
+
+    await expect(
+      storage.upload({ key: "uploads/broken.txt", body: failing, contentType: "text/plain" }),
+    ).rejects.toThrow("connection lost");
+    expect(await readdir(join(directory, "uploads"))).toEqual([]);
+  });
+
+  it("rejects keys that leave the folder through a symbolic link", async () => {
+    const outside = await mkdtemp(join(tmpdir(), "clubedge-outside-"));
+    try {
+      await writeFile(join(outside, "secret.txt"), "secret");
+      await mkdir(directory, { recursive: true });
+      // A junction on Windows needs no extra privileges; other systems ignore the type.
+      await symlink(outside, join(directory, "linked"), "junction");
+      const storage = createFileSystemStorage({ directory });
+      const body = new TextEncoder().encode("x");
+
+      await expect(
+        storage.upload({ key: "linked/new.txt", body, contentType: "text/plain" }),
+      ).rejects.toThrow("Invalid storage key");
+      await expect(storage.download("linked/secret.txt")).rejects.toThrow("Invalid storage key");
+      await expect(storage.delete("linked/secret.txt")).rejects.toThrow("Invalid storage key");
+      expect(await readFile(join(outside, "secret.txt"), "utf8")).toBe("secret");
+      expect(await readdir(outside)).toEqual(["secret.txt"]);
+    } finally {
+      await rm(outside, { recursive: true, force: true });
+    }
+  });
+
+  it("treats objects as missing before the folder exists", async () => {
+    const storage = createFileSystemStorage({ directory: join(directory, "not-created") });
+    expect(await storage.exists("a.txt")).toBe(false);
+    await expect(storage.delete("a.txt")).resolves.toBeUndefined();
+    await expect(storage.download("a.txt")).rejects.toThrow("Object not found");
+  });
+
+  it("fails to download a missing object and deletes missing objects quietly", async () => {
+    const storage = createFileSystemStorage({ directory });
+    await expect(storage.download("missing.txt")).rejects.toThrow();
+    await expect(storage.delete("missing.txt")).resolves.toBeUndefined();
+  });
+});

package/template/packages/storage/src/file-system.ts

@@ -0,0 +1,124 @@
+import { randomUUID } from "node:crypto";
+import { createReadStream, createWriteStream } from "node:fs";
+import { access, lstat, mkdir, realpath, rename, rm } from "node:fs/promises";
+import { dirname, isAbsolute, relative, resolve } from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import type { ReadableStream as NodeReadableStream } from "node:stream/web";
+import type { StorageProvider, UploadInput } from "./types";
+
+export interface FileSystemStorageConfig {
+  /** Folder that holds the objects; created on first upload. */
+  directory: string;
+}
+
+function bodyStream(body: UploadInput["body"]): Readable {
+  if (body instanceof Uint8Array) return Readable.from([body]);
+  if (body instanceof ArrayBuffer) return Readable.from([new Uint8Array(body)]);
+  const stream = body instanceof Blob ? body.stream() : body;
+  return Readable.fromWeb(stream as NodeReadableStream<Uint8Array>);
+}
+
+function isInside(root: string, path: string) {
+  const inside = relative(root, path);
+  return inside !== "" && !inside.startsWith("..") && !isAbsolute(inside);
+}
+
+function isMissing(error: unknown) {
+  return (error as NodeJS.ErrnoException).code === "ENOENT";
+}
+
+/**
+ * Stores objects as files in a local folder, for development without an object storage service.
+ * Keys map to paths inside the folder. Keys that would leave it are rejected, including through
+ * symbolic links inside the folder.
+ */
+export function createFileSystemStorage({ directory }: FileSystemStorageConfig): StorageProvider {
+  const root = resolve(directory);
+
+  function lexicalPath(key: string) {
+    const path = resolve(root, key);
+    if (!key || !isInside(root, path)) throw new Error(`Invalid storage key: ${key}`);
+    return path;
+  }
+
+  /** Throws unless the deepest existing folder on the way to `path` is inside the root. */
+  async function assertNoLinkEscape(path: string) {
+    const realRoot = await realpath(root);
+    let folder = dirname(path);
+    while (true) {
+      try {
+        const realFolder = await realpath(folder);
+        if (realFolder !== realRoot && !isInside(realRoot, realFolder)) {
+          throw new Error(`Invalid storage key: ${relative(root, path)}`);
+        }
+        break;
+      } catch (error) {
+        if (!isMissing(error)) throw error;
+        folder = dirname(folder);
+      }
+    }
+    try {
+      if ((await lstat(path)).isSymbolicLink()) {
+        throw new Error(`Invalid storage key: ${relative(root, path)}`);
+      }
+    } catch (error) {
+      if (!isMissing(error)) throw error;
+    }
+  }
+
+  /** The file for `key`, checked against the root. Resolves null when the root does not exist. */
+  async function existingPath(key: string) {
+    const path = lexicalPath(key);
+    try {
+      await assertNoLinkEscape(path);
+    } catch (error) {
+      if (isMissing(error)) return null;
+      throw error;
+    }
+    return path;
+  }
+
+  return {
+    async upload({ key, body }) {
+      const path = lexicalPath(key);
+      await mkdir(root, { recursive: true });
+      await assertNoLinkEscape(path);
+      await mkdir(dirname(path), { recursive: true });
+      await assertNoLinkEscape(path);
+      // Stream into a temporary file, then rename it, so a failed upload leaves nothing behind.
+      const partial = `${path}.${randomUUID()}.partial`;
+      try {
+        await pipeline(bodyStream(body), createWriteStream(partial, { flags: "wx" }));
+        await rename(partial, path);
+      } catch (error) {
+        await rm(partial, { force: true });
+        throw error;
+      }
+      return { key };
+    },
+    async download(key) {
+      const path = await existingPath(key);
+      if (!path) throw new Error(`Object not found: ${key}`);
+      await access(path);
+      return Readable.toWeb(createReadStream(path)) as ReadableStream<Uint8Array>;
+    },
+    async delete(key) {
+      const path = await existingPath(key);
+      if (path) await rm(path, { force: true });
+    },
+    async exists(key) {
+      const path = await existingPath(key);
+      if (!path) return false;
+      try {
+        await access(path);
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    async getSignedUrl() {
+      throw new Error("Local folder storage cannot create URLs; serve the files through your app.");
+    },
+  };
+}

package/template/packages/storage-s3/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clubedge/storage-s3",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "S3-compatible adapter for @clubedge/storage.",
   "private": true,
   "license": "Apache-2.0",

package/template/packages/storage-supabase/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clubedge/storage-supabase",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Supabase Storage adapter for @clubedge/storage.",
   "private": true,
   "license": "Apache-2.0",

package/template/packages/ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clubedge/ui",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "private": true,
   "license": "Apache-2.0",
   "type": "module",