@twasik4/pocket-service 1.0.0

package/README.md

@@ -0,0 +1,241 @@
+# Pocket Service
+
+Pocket service is a wrapper that provides you a variety of options for a easy, boiler plate free, typescript service.
+
+Pocket service comes with the following features built in:
+
+- Express API with a few built in endpoints
+- Full type support via a builder pattern
+- An optional mongo module
+- An optional clickhouse module
+- An optional redis module
+- An optional redis worker module for consuming redis streams
+- A way to add your own modules/startup hooks that run during the build phase
+
+## Creating a service
+
+Creating a service is done with the service class. It will streamline the creation process and reduce boilerplate.
+
+The following is the most basic service you can create.
+
+```ts
+const service = new Service().build();
+```
+
+This basic service will include an express server with a few [built in endpoints](#built-in-endpoints) as well as a built in winston instance for logging.
+
+### Built-in Endpoints
+
+The following are the base routes available:
+
+| Method | Route   | Description                                |
+| ------ | ------- | ------------------------------------------ |
+| GET    | /       | OK Check                                   |
+| GET    | /health | Used for health information on the service |
+
+### Modules
+
+1. withMongo()
+
+A full MongoDB instance with a custom collection abstraction that offers full type support and auto completion.
+
+2. withRedis()
+
+A full Redis instance, this has built in service registration.
+
+3. withClickhouse()
+
+A WIP, this will be similar to the mongo abstraction and offer strong typing for queries.
+
+4. withWorkers()
+
+This adds built in workers to consume Redis streams off the main loop.
+
+The following additional routes will get added to express:
+
+| Method | Route                          | Description                                         |
+| ------ | ------------------------------ | --------------------------------------------------- |
+| GET    | /workers                       | Used for worker health information                  |
+| GET    | /workers/:stream               | Gets all messages from this services stream         |
+| POST   | /workers/:stream/:id/ack       | Force acknowledge a message in this services stream |
+| GET    | /workers/:stream/dlq           | Gets all messages from this services DLQ stream     |
+| POST   | /workers/:stream/dlq/:id/retry | Force retry a message in this services DLQ stream   |
+
+5. withModules(customModules: CustomModule[])
+
+Use this to register custom modules to the service.
+
+### Configurations
+
+1. withName()
+
+Sets the name of the service, used primarily with service registration via Redis. Defaults to a random byte string.
+
+```ts
+const service = new Service().withName("Service Name").build();
+```
+
+2. withPort()
+
+Set the port of the express server. Defaults to 3100.
+
+```ts
+const service = new Service().withPort(8111).build();
+```
+
+Setting the port to the above would make the express instance listen on that port. So your root endpoint would be http://localhost:8111/ and display the following
+
+```ts
+{
+  "isSuccess": true,
+  "message": "Service Name service is running",
+  "uptime": {
+    "days": 0,
+    "hours": 0,
+    "minutes": 0,
+    "seconds": 4
+  }
+}
+```
+
+3. withUrl()
+
+Sets the base url of the service. Useful if you use service registration via Redis.
+
+```ts
+const service = new Service().withUrl("https://my-domain.com").build();
+```
+
+4. withExpressOptions()
+
+Manually configure various express options like CORS, custom middleware, etc.
+
+```ts
+const service = new Service()
+  .withExpressOptions({
+    asJson: true,
+    corsWhitelist: ["http://localhost:5173"],
+    credentials: true,
+  })
+  .build();
+```
+
+The above code snippet would parse all incoming bodies with the express JSON parser as well as enforce CORS, whitelisting localhost on 5173 and pass credentials.
+
+5. addRouter()
+
+Registers a custom router with express. This offers ways to set additional metadata and offers the ability to add validation via Zod schemas.
+
+An example auth router could look like the following.
+
+```ts
+const { router, routes, addRoute } = createMetaRouter();
+
+addRoute(
+  {
+    fullPath: "/login",
+    method: "POST",
+    requireAuth: false,
+    bodyValidator: z.object({
+      email: z.email(),
+      password: z.string(),
+    }),
+    meta: {
+      description: "Login endpoint",
+    },
+  },
+  async (req, res) => {
+    // contents here
+    res
+      .status(200)
+      .json({ isSuccess: true, message: "Successfully logged in." });
+  },
+);
+
+addRoute(
+  {
+    fullPath: "/register",
+    method: "POST",
+    requireAuth: false,
+    bodyValidator: z.object({
+      email: z.email(),
+      password: z.string().min(6),
+    }),
+    meta: {
+      description: "Register endpoint",
+    },
+  },
+  async (req, res) => {
+    // contents here
+    res.status(200).json({ isSuccess: true, message: "Registered" });
+  },
+);
+
+addRoute(
+  {
+    fullPath: "/logout",
+    method: "POST",
+    requireAuth: true,
+    bodyValidator: z.object({
+      refreshToken: z.string(),
+    }),
+    meta: {
+      description: "Logout endpoint",
+    },
+  },
+  async (req, res) => {
+    // contents here
+    res.status(200).json({ isSuccess: true, message: "Logged out" });
+  },
+);
+
+addRoute(
+  {
+    fullPath: "/me",
+    method: "GET",
+    requireAuth: true,
+    meta: {
+      description: "Get current authenticated user info",
+    },
+  },
+  async (req, res) => {
+    // contents here
+    res.status(200).json({ isSuccess: true, message: "Me" });
+  },
+);
+
+addRoute(
+  {
+    fullPath: "/refresh",
+    method: "POST",
+    requireAuth: false,
+    meta: {
+      description: "Register endpoint",
+    },
+  },
+  async (req, res) => {
+    // contents here
+    res.json({ isSuccess: true, message: "Token refreshed" });
+  },
+);
+
+export { router as AuthRouter, routes as AuthRoutes };
+```
+
+### Methods
+
+1. shutdown()
+
+This shuts down the service, gracefully stopping all modules and the service itself.
+
+2. build()
+
+The build phase, this is where all modules get initialized and loaded as well as what starts up the express server.
+
+3. heartbeat()
+
+A method to start an interval for service registration with Redis. This gets automatically called during the build phase if the Redis module has been loaded.
+
+4. register()
+
+A method to immediately register the service with Redis.

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@twasik4/pocket-service",
+  "version": "1.0.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "type": "module",
+  "peerDependencies": {
+    "@clickhouse/client": "^1.12.1",
+    "cors": "^2.8.5",
+    "express": "^5.1.0",
+    "mongodb": "^6.20.0",
+    "redis": "^5.8.2",
+    "winston": "^3.17.0",
+    "winston-transport": "^4.9.0",
+    "zod": "^4.1.12"
+  },
+  "devDependencies": {
+    "@clickhouse/client": "^1.12.1",
+    "@types/cors": "^2.8.19",
+    "@types/express": "^5.0.3",
+    "@types/node": "^24.5.2",
+    "@types/winston": "^2.4.4",
+    "cors": "^2.8.5",
+    "express": "^5.1.0",
+    "mongodb": "^6.20.0",
+    "mongodb-memory-server": "^11.0.1",
+    "redis": "^5.8.2",
+    "vitest": "^3.2.4",
+    "winston": "^3.17.0",
+    "winston-transport": "^4.9.0",
+    "zod": "^4.1.12"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "test": "vitest",
+    "test:watch": "vitest --watch"
+  }
+}

package/src/index.ts

@@ -0,0 +1 @@
+export * from "./workers";

package/src/worker-thread.ts

@@ -0,0 +1,159 @@
+import { createClient } from "redis";
+import { parentPort, workerData } from "worker_threads";
+import { RedisStreamEventHandler } from "./workers/stream-handler";
+
+(async () => {
+  const { name, redisUrl, handlers, index, subscriptions } = workerData as {
+    name: string;
+    redisUrl: string;
+    handlers: Record<string, string>;
+    index: number;
+    subscriptions: string[];
+  };
+  const redis = createClient({ url: redisUrl });
+  await redis.connect();
+
+  parentPort?.postMessage({
+    level: "info",
+    message: `Connected to Redis at ${redisUrl}`,
+  });
+
+  const loadedHandlers = new Map();
+
+  for (const [_, filePath] of Object.entries(handlers)) {
+    const mod = await import(filePath);
+    if (mod.default?.type && mod.default?.execute) {
+      loadedHandlers.set(mod.default.type, mod.default);
+    }
+  }
+
+  parentPort?.postMessage({
+    level: "create",
+    loadedHandlers: Object.keys(loadedHandlers),
+  });
+
+  const stream = `${name}:events`;
+  const group = `${name}-group`;
+  const consumer = `${name}-worker-${index}`;
+  const dlqStream = `dlq:${stream}`;
+
+  for (const streamName of subscriptions) {
+    // TODO: infer from handlers
+    try {
+      await redis.xGroupCreate(streamName, group, "0", {
+        MKSTREAM: true,
+      });
+    } catch (error) {
+      if ((error as Error).message.includes("BUSYGROUP")) {
+        parentPort?.postMessage({
+          level: "warn",
+          message: `Group ${group} already exists for stream ${streamName}, skipping creation`,
+        });
+      } else {
+        parentPort?.postMessage({
+          level: "error",
+          message: `Error creating group: ${(error as Error).message}`,
+        });
+      }
+    }
+  }
+
+  while (true) {
+    const keys = subscriptions.map((s) => ({ key: s, id: ">" })); // TODO: infer from handlers
+    const res = await redis.xReadGroup(group, consumer, keys, {
+      COUNT: 10,
+      BLOCK: 5000,
+    });
+
+    if (!res) {
+      continue;
+    }
+
+    if (Array.isArray(res)) {
+      for (const { name: streamName, messages } of res as {
+        name: string;
+        messages: {
+          id: string;
+          message: { eventType: string };
+        }[];
+      }[]) {
+        for (const { id, message } of messages) {
+          const eventType = message.eventType;
+          const handler = loadedHandlers.get(
+            eventType,
+          ) as RedisStreamEventHandler;
+
+          if (!handler) {
+            parentPort?.postMessage({
+              level: "error",
+              message: `No handler found for event type: ${eventType}`,
+            });
+            await redis.xAck(stream, group, id);
+            continue;
+          }
+
+          const timeoutMs = 3000;
+          const maxRetries = 3;
+          let attempts = 0;
+
+          while (attempts < maxRetries) {
+            attempts++;
+            try {
+              const result = await Promise.race([
+                await handler.execute(message, {
+                  emit: async (ev) => {
+                    redis.xAdd(
+                      `${ev.type.split(":")[0]}:events`,
+                      "*",
+                      ev as Record<string, any>,
+                    );
+                  },
+                  log: {
+                    info: (msg) =>
+                      parentPort?.postMessage({ level: "info", message: msg }),
+                    warn: (msg) =>
+                      parentPort?.postMessage({ level: "warn", message: msg }),
+                    error: (msg) =>
+                      parentPort?.postMessage({ level: "error", message: msg }),
+                  },
+                  service: streamName,
+                }),
+                new Promise((resolve) =>
+                  setTimeout(() => resolve("timeout"), timeoutMs),
+                ),
+              ]);
+
+              if (result === "timeout") {
+                throw new Error(
+                  `Message processing timed out after ${timeoutMs}ms`,
+                );
+              }
+
+              await redis.xAck(group, group, id);
+              parentPort?.postMessage({
+                level: "info",
+                message: `Message ${id} processed successfully after ${attempts} attempts`,
+              });
+              return;
+            } catch (error) {
+              parentPort?.postMessage({
+                level: "error",
+                message: `Error processing message ${id}: ${
+                  error instanceof Error ? error.message : error
+                }`,
+              });
+              if (attempts === maxRetries) {
+                await redis.xAdd(dlqStream, "*", {
+                  originalId: id,
+                  message: JSON.stringify(message),
+                  error: error instanceof Error ? error.message : String(error),
+                  timestamp: new Date().toISOString(),
+                });
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+})();

package/src/workers/db/clickhouse/index.ts

@@ -0,0 +1,76 @@
+import { createClient } from "@clickhouse/client";
+
+export interface ClickhouseTable<TSchema extends Record<string, any>> {
+  name: string;
+  createSQL: string;
+  schema: () => TSchema;
+}
+
+export interface ClickhouseConfig<
+  TTables extends Record<string, ClickhouseTable<any>>
+> {
+  url: string;
+  username?: string;
+  password?: string;
+  tables: TTables;
+}
+
+export function createTypedClickhouse<
+  TTables extends Record<string, ClickhouseTable<any>>
+>(config: ClickhouseConfig<TTables>) {
+  const client = createClient({
+    host: config.url,
+    username: config.username,
+    password: config.password,
+  });
+
+  async function ensureTables() {
+    for (const { createSQL } of Object.values(config.tables || {})) {
+      await client.command({ query: createSQL });
+    }
+  }
+
+  const api = Object.fromEntries(
+    Object.entries(config.tables || {}).map(([key, table]) => {
+      type Row = ReturnType<typeof table.schema>;
+      type Filter = Partial<Row>;
+
+      const select = async (where?: Filter): Promise<Row[]> => {
+        let query = `SELECT * FROM ${table.name}`;
+        if (where && Object.keys(where).length > 0) {
+          const conditions = Object.keys(where)
+            .map(([k, value]) => {
+              if (typeof value === "string") return `${k} = '${value}'`;
+              return `${k} = ${value}`;
+            })
+            .join(" AND ");
+          query += ` WHERE ${conditions}`;
+        }
+        const res = await client.query({
+          query,
+        });
+        const rows = await res.json<{ data: any[] }>();
+        return rows.data as Row[];
+      };
+
+      const insert = async (rows: Row[]) => {
+        await client.insert({
+          table: table.name,
+          values: rows,
+          format: "JSONEachRow",
+        });
+      };
+
+      return [key, { select, insert }];
+    })
+  ) as {
+    [K in keyof TTables]: {
+      select(
+        where?: Partial<ReturnType<TTables[K]["schema"]>>
+      ): Promise<ReturnType<TTables[K]["schema"]>[]>;
+      insert(rows: ReturnType<TTables[K]["schema"]>[]): Promise<void>;
+    };
+  };
+
+  return { client, ensureTables, ...api };
+}