@takaro/db 0.0.0-next.09a7ca1

package/src/migrations/sql/20251115071948-remove-deleted-domain-state.ts

@@ -0,0 +1,46 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  // First, migrate any existing DELETED domains to DISABLED state
+  await knex.raw(`UPDATE domains SET state = 'DISABLED' WHERE state = 'DELETED'`);
+
+  // PostgreSQL requires a multi-step process to update enum constraints
+  // 1. Add a temporary column with the new enum values (without DELETED)
+  // 2. Copy data from old column to new column
+  // 3. Drop old column
+  // 4. Rename new column to old column name
+
+  await knex.schema.alterTable('domains', (table) => {
+    table.enum('state_new', ['ACTIVE', 'DISABLED', 'MAINTENANCE']).notNullable().defaultTo('ACTIVE');
+  });
+
+  await knex.raw(`UPDATE domains SET state_new = state::text::varchar`);
+
+  await knex.schema.alterTable('domains', (table) => {
+    table.dropColumn('state');
+  });
+
+  await knex.schema.alterTable('domains', (table) => {
+    table.renameColumn('state_new', 'state');
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  // Drop the enum type if it exists to prevent "type already exists" errors
+  await knex.raw('DROP TYPE IF EXISTS state_new CASCADE');
+
+  // Revert to include DELETED in enum values
+  await knex.schema.alterTable('domains', (table) => {
+    table.enum('state_new', ['ACTIVE', 'DISABLED', 'MAINTENANCE', 'DELETED']).notNullable().defaultTo('ACTIVE');
+  });
+
+  await knex.raw(`UPDATE domains SET state_new = state::text::varchar`);
+
+  await knex.schema.alterTable('domains', (table) => {
+    table.dropColumn('state');
+  });
+
+  await knex.schema.alterTable('domains', (table) => {
+    table.renameColumn('state_new', 'state');
+  });
+}

package/src/migrations/sql/20251220000000-inventory-diff-storage.ts

@@ -0,0 +1,164 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  // Create playerInventoryBaseline table - full inventory snapshots every hour
+  await knex.raw(`
+    CREATE TABLE "playerInventoryBaseline" (
+      id UUID DEFAULT gen_random_uuid() NOT NULL,
+      "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
+      "updatedAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
+      domain VARCHAR(255) NOT NULL,
+      "playerId" UUID NOT NULL,
+      "baselineId" UUID NOT NULL,
+      "itemId" UUID NOT NULL,
+      quantity INTEGER NOT NULL,
+      quality VARCHAR(255),
+      CONSTRAINT "playerInventoryBaseline_pkey" PRIMARY KEY ("createdAt", "baselineId", "itemId"),
+      CONSTRAINT "playerInventoryBaseline_domain_foreign" FOREIGN KEY (domain) REFERENCES domains(id) ON DELETE CASCADE,
+      CONSTRAINT "playerInventoryBaseline_playerId_foreign" FOREIGN KEY ("playerId") REFERENCES "playerOnGameServer"(id) ON DELETE CASCADE,
+      CONSTRAINT "playerInventoryBaseline_itemId_foreign" FOREIGN KEY ("itemId") REFERENCES items(id) ON DELETE CASCADE,
+      CONSTRAINT "playerInventoryBaseline_quantity_check" CHECK (quantity >= 0)
+    ) PARTITION BY RANGE ("createdAt")
+  `);
+
+  // Create playerInventoryDiff table - individual item changes between baselines
+  await knex.raw(`
+    CREATE TABLE "playerInventoryDiff" (
+      id UUID DEFAULT gen_random_uuid() NOT NULL,
+      "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
+      "updatedAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
+      domain VARCHAR(255) NOT NULL,
+      "playerId" UUID NOT NULL,
+      "itemId" UUID NOT NULL,
+      "changeType" VARCHAR(10) NOT NULL,
+      "previousQuantity" INTEGER,
+      "newQuantity" INTEGER,
+      "previousQuality" VARCHAR(255),
+      "newQuality" VARCHAR(255),
+      CONSTRAINT "playerInventoryDiff_pkey" PRIMARY KEY ("createdAt", "playerId", "itemId"),
+      CONSTRAINT "playerInventoryDiff_domain_foreign" FOREIGN KEY (domain) REFERENCES domains(id) ON DELETE CASCADE,
+      CONSTRAINT "playerInventoryDiff_playerId_foreign" FOREIGN KEY ("playerId") REFERENCES "playerOnGameServer"(id) ON DELETE CASCADE,
+      CONSTRAINT "playerInventoryDiff_itemId_foreign" FOREIGN KEY ("itemId") REFERENCES items(id) ON DELETE CASCADE,
+      CONSTRAINT "playerInventoryDiff_changeType_check" CHECK ("changeType" IN ('added', 'removed', 'changed'))
+    ) PARTITION BY RANGE ("createdAt")
+  `);
+
+  // Create partition function for playerInventoryBaseline
+  await knex.raw(`
+    CREATE OR REPLACE FUNCTION ensure_player_inventory_baseline_partition(date_param VARCHAR DEFAULT NULL)
+    RETURNS VOID AS $$
+    DECLARE
+      current_day_start DATE;
+      next_day_start DATE;
+      partition_name TEXT;
+      partition_exists BOOLEAN;
+      target_date DATE;
+    BEGIN
+      IF date_param IS NOT NULL THEN
+        target_date := DATE(date_param::TIMESTAMP);
+      ELSE
+        target_date := CURRENT_DATE;
+      END IF;
+
+      current_day_start := DATE_TRUNC('day', target_date);
+      next_day_start := current_day_start + INTERVAL '1 day';
+
+      partition_name := 'playerInventoryBaseline_' || TO_CHAR(current_day_start, 'YYYY_MM_DD');
+
+      SELECT EXISTS (
+        SELECT 1 FROM pg_class c
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE c.relname = partition_name
+          AND n.nspname = 'public'
+      ) INTO partition_exists;
+
+      IF NOT partition_exists THEN
+        EXECUTE format(
+          'CREATE TABLE %I PARTITION OF "playerInventoryBaseline"
+           FOR VALUES FROM (%L) TO (%L)',
+          partition_name,
+          current_day_start,
+          next_day_start
+        );
+
+        RAISE NOTICE 'Created partition: % for date: %',
+          partition_name, current_day_start;
+      END IF;
+    END;
+    $$ LANGUAGE plpgsql;
+  `);
+
+  // Create partition function for playerInventoryDiff
+  await knex.raw(`
+    CREATE OR REPLACE FUNCTION ensure_player_inventory_diff_partition(date_param VARCHAR DEFAULT NULL)
+    RETURNS VOID AS $$
+    DECLARE
+      current_day_start DATE;
+      next_day_start DATE;
+      partition_name TEXT;
+      partition_exists BOOLEAN;
+      target_date DATE;
+    BEGIN
+      IF date_param IS NOT NULL THEN
+        target_date := DATE(date_param::TIMESTAMP);
+      ELSE
+        target_date := CURRENT_DATE;
+      END IF;
+
+      current_day_start := DATE_TRUNC('day', target_date);
+      next_day_start := current_day_start + INTERVAL '1 day';
+
+      partition_name := 'playerInventoryDiff_' || TO_CHAR(current_day_start, 'YYYY_MM_DD');
+
+      SELECT EXISTS (
+        SELECT 1 FROM pg_class c
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE c.relname = partition_name
+          AND n.nspname = 'public'
+      ) INTO partition_exists;
+
+      IF NOT partition_exists THEN
+        EXECUTE format(
+          'CREATE TABLE %I PARTITION OF "playerInventoryDiff"
+           FOR VALUES FROM (%L) TO (%L)',
+          partition_name,
+          current_day_start,
+          next_day_start
+        );
+
+        RAISE NOTICE 'Created partition: % for date: %',
+          partition_name, current_day_start;
+      END IF;
+    END;
+    $$ LANGUAGE plpgsql;
+  `);
+
+  // Create indexes for playerInventoryBaseline
+  await knex.raw(
+    `CREATE INDEX "playerInventoryBaseline_domain_createdAt_idx" ON "playerInventoryBaseline" (domain, "createdAt")`,
+  );
+  await knex.raw(`CREATE INDEX "playerInventoryBaseline_playerId_idx" ON "playerInventoryBaseline" ("playerId")`);
+  await knex.raw(
+    `CREATE INDEX "playerInventoryBaseline_playerId_createdAt_idx" ON "playerInventoryBaseline" ("playerId", "createdAt" DESC)`,
+  );
+  await knex.raw(`CREATE INDEX "playerInventoryBaseline_baselineId_idx" ON "playerInventoryBaseline" ("baselineId")`);
+
+  // Create indexes for playerInventoryDiff
+  await knex.raw(
+    `CREATE INDEX "playerInventoryDiff_domain_createdAt_idx" ON "playerInventoryDiff" (domain, "createdAt")`,
+  );
+  await knex.raw(`CREATE INDEX "playerInventoryDiff_playerId_idx" ON "playerInventoryDiff" ("playerId")`);
+  await knex.raw(
+    `CREATE INDEX "playerInventoryDiff_playerId_createdAt_idx" ON "playerInventoryDiff" ("playerId", "createdAt" DESC)`,
+  );
+  await knex.raw(
+    `CREATE INDEX "playerInventoryDiff_itemId_createdAt_idx" ON "playerInventoryDiff" ("itemId", "createdAt" DESC)`,
+  );
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.raw('DROP FUNCTION IF EXISTS ensure_player_inventory_baseline_partition(VARCHAR)');
+  await knex.raw('DROP FUNCTION IF EXISTS ensure_player_inventory_diff_partition(VARCHAR)');
+  await knex.raw('DROP TABLE IF EXISTS "playerInventoryBaseline" CASCADE');
+  await knex.raw('DROP TABLE IF EXISTS "playerInventoryDiff" CASCADE');
+}

package/src/migrations/util/alterEnum.ts

@@ -0,0 +1,9 @@
+export function formatAlterTableEnumSql(tableName: string, columnName: string, enums: string[]) {
+  const constraintName = `${tableName}_${columnName}_check`;
+  return [
+    `ALTER TABLE "${tableName}" DROP CONSTRAINT IF EXISTS "${constraintName}";`,
+    `ALTER TABLE "${tableName}" ADD CONSTRAINT "${constraintName}" CHECK ("${columnName}" = ANY (ARRAY['${enums.join(
+      "'::text, '",
+    )}'::text]));`,
+  ].join('\n');
+}

package/src/queryBuilder.ts

@@ -0,0 +1,263 @@
+import { IsEnum, IsNumber, IsOptional, IsString } from 'class-validator';
+import { errors } from '@takaro/util';
+import {
+  QueryBuilder as ObjectionQueryBuilder,
+  Model as ObjectionModel,
+  Page,
+  AnyQueryBuilder,
+  Expression,
+  PrimitiveValue,
+} from 'objection';
+import { getKnex, isDbAvailable } from './knex.js';
+
+export class ITakaroQuery<T> {
+  @IsOptional()
+  filters?: {
+    [key: string]: unknown[] | string[] | boolean | null | undefined;
+  };
+
+  @IsOptional()
+  search?: {
+    [key: string]: unknown[] | string[] | boolean | null | undefined;
+  };
+
+  @IsOptional()
+  greaterThan?: {
+    [key in keyof T]?: unknown;
+  };
+
+  @IsOptional()
+  lessThan?: {
+    [key in keyof T]?: unknown;
+  };
+
+  @IsOptional()
+  @IsNumber()
+  page?: number;
+
+  @IsOptional()
+  @IsNumber()
+  limit?: number;
+
+  @IsOptional()
+  @IsString()
+  sortBy?: string;
+
+  @IsOptional()
+  @IsString()
+  @IsEnum(['asc', 'desc'])
+  sortDirection?: SortDirection;
+
+  @IsOptional()
+  @IsString({ each: true })
+  extend?: string[];
+}
+
+export enum SortDirection {
+  asc = 'asc',
+  desc = 'desc',
+}
+
+const modelColumns = new Map<string, string[]>();
+
+export async function populateModelColumns() {
+  if (await isDbAvailable()) {
+    const knex = await getKnex();
+    // Find all tables
+    const tables = await knex.raw(
+      "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'",
+    );
+    // For each table, store the columns
+    for (const table of tables.rows) {
+      const tableName = table.table_name;
+      const columns = await knex(tableName).columnInfo();
+      modelColumns.set(tableName, Object.keys(columns));
+    }
+  }
+}
+
+await populateModelColumns();
+
+export class QueryBuilder<Model extends ObjectionModel, OutputDTO> {
+  private query: ITakaroQuery<OutputDTO>;
+  constructor(rawQuery: ITakaroQuery<OutputDTO> = new ITakaroQuery()) {
+    this.query = JSON.parse(JSON.stringify(rawQuery));
+    this.validateArrayFilters();
+  }
+
+  private validateArrayFilters() {
+    // Validate filters
+    if (this.query.filters) {
+      for (const [key, value] of Object.entries(this.query.filters)) {
+        if (value !== null && value !== undefined && !Array.isArray(value) && typeof value !== 'boolean') {
+          throw new errors.BadRequestError(
+            `Filter values must be arrays or boolean. Found invalid value for '${key}': '${value}'. Example: { ${key}: ["${value}"] }`,
+          );
+        }
+      }
+    }
+
+    // Validate search
+    if (this.query.search) {
+      for (const [key, value] of Object.entries(this.query.search)) {
+        if (value !== null && value !== undefined && !Array.isArray(value) && typeof value !== 'boolean') {
+          throw new errors.BadRequestError(
+            `Search values must be arrays or boolean. Found invalid value for '${key}': '${value}'. Example: { ${key}: ["${value}"] }`,
+          );
+        }
+      }
+    }
+  }
+
+  /**
+   * Our custom query builder can only handle columns that exist in the table
+   * However, we might want to do complex/relational logic in higher layers
+   * To make sure we don't error out here, we filter out any columns that don't exist in the current table
+   */
+  private cleanQueryObject(tableName: string) {
+    const columns = modelColumns.get(tableName) ?? [];
+
+    for (const key in this.query.filters) {
+      if (Object.prototype.hasOwnProperty.call(this.query.filters, key)) {
+        if (!columns.includes(key)) {
+          if (this.query.filters) {
+            // Preserve boolean values, only clear arrays
+            if (typeof this.query.filters[key] !== 'boolean') {
+              this.query.filters[key] = [];
+            }
+          }
+        }
+      }
+    }
+
+    for (const key in this.query.search) {
+      if (Object.prototype.hasOwnProperty.call(this.query.search, key)) {
+        if (!columns.includes(key)) {
+          if (this.query.search) {
+            // Preserve boolean values, only clear arrays
+            if (typeof this.query.search[key] !== 'boolean') {
+              this.query.search[key] = [];
+            }
+          }
+        }
+      }
+    }
+
+    return this.query;
+  }
+
+  build(queryBuilder: ObjectionQueryBuilder<Model, Model[]>): ObjectionQueryBuilder<Model, Page<Model>> {
+    const tableName = queryBuilder.modelClass().tableName;
+    this.cleanQueryObject(tableName);
+
+    const pagination = this.pagination();
+    const sorting = this.sorting();
+
+    let qry = queryBuilder.page(pagination.page, pagination.limit).orderBy(sorting.sortBy, sorting.sortDirection);
+
+    qry = this.filters(tableName, qry);
+    qry = this.greaterThan(tableName, qry);
+    qry = this.lessThan(tableName, qry);
+
+    if (this.query.search) {
+      qry.where((builder) => {
+        for (const search in this.query.search) {
+          if (Object.prototype.hasOwnProperty.call(this.query.search, search)) {
+            const searchVal = this.query.search[search];
+            if (Array.isArray(searchVal)) {
+              searchVal.forEach((val) => {
+                if (val) {
+                  builder.orWhere(`${tableName}.${search}`, 'ilike', `%${val}%`);
+                }
+              });
+            }
+          }
+        }
+      });
+    }
+
+    for (const extend of this.query.extend ?? []) {
+      qry.withGraphFetched(extend);
+    }
+
+    return qry;
+  }
+
+  private filters(
+    tableName: string,
+    query: ObjectionQueryBuilder<Model, Page<Model>>,
+  ): ObjectionQueryBuilder<Model, Page<Model>> {
+    for (const filter in this.query.filters) {
+      if (Object.prototype.hasOwnProperty.call(this.query.filters, filter)) {
+        const searchVal = this.query.filters[filter];
+
+        if (typeof searchVal === 'boolean') {
+          query.where(`${tableName}.${filter}`, searchVal);
+        } else if (searchVal && Array.isArray(searchVal)) {
+          if (searchVal.includes('null')) {
+            query.whereNull(`${tableName}.${filter}`);
+            continue;
+          }
+
+          if (searchVal.length) {
+            query.whereIn(`${tableName}.${filter}`, searchVal.filter(Boolean) as unknown as AnyQueryBuilder);
+          }
+        }
+      }
+    }
+
+    return query;
+  }
+
+  private greaterThan(
+    tableName: string,
+    query: ObjectionQueryBuilder<Model, Page<Model>>,
+  ): ObjectionQueryBuilder<Model, Page<Model>> {
+    for (const filter in this.query.greaterThan) {
+      if (Object.prototype.hasOwnProperty.call(this.query.greaterThan, filter)) {
+        const searchVal = this.query.greaterThan[filter];
+        if (searchVal) {
+          query.where(`${tableName}.${filter}`, '>=', searchVal as unknown as Expression<PrimitiveValue>);
+        }
+      }
+    }
+
+    return query;
+  }
+
+  private lessThan(
+    tableName: string,
+    query: ObjectionQueryBuilder<Model, Page<Model>>,
+  ): ObjectionQueryBuilder<Model, Page<Model>> {
+    for (const filter in this.query.lessThan) {
+      if (Object.prototype.hasOwnProperty.call(this.query.lessThan, filter)) {
+        const searchVal = this.query.lessThan[filter];
+        if (searchVal) {
+          query.where(`${tableName}.${filter}`, '<=', searchVal as unknown as Expression<PrimitiveValue>);
+        }
+      }
+    }
+
+    return query;
+  }
+
+  private sorting(): { sortBy: string; sortDirection: SortDirection } {
+    if (!this.query.sortBy) {
+      return {
+        sortBy: 'id',
+        sortDirection: SortDirection.asc,
+      };
+    }
+    return {
+      sortBy: this.query.sortBy,
+      sortDirection: this.query.sortDirection ?? SortDirection.asc,
+    };
+  }
+
+  private pagination() {
+    return {
+      page: this.query.page ?? 0,
+      limit: this.query.limit ?? 100,
+    };
+  }
+}

package/src/redis.ts

@@ -0,0 +1,59 @@
+import { createClient, RedisClientOptions } from 'redis';
+import { config } from './config.js';
+import { logger, health } from '@takaro/util';
+
+export type RedisClient = ReturnType<typeof createClient>;
+
+class RedisClass {
+  private log = logger('redis');
+
+  private clients = new Map<string, RedisClient>();
+
+  /**
+   * Get a Redis client from the cache, or create a new one.
+   * The client will already be connected.
+   * @param name Name for the client, used to cache the client.
+   * @param extra Any extra Redis options to pass to the client.
+   * @returns
+   */
+  async getClient(name: string, extra: RedisClientOptions = {}): Promise<RedisClient> {
+    const normalizedName = name.toLowerCase().replaceAll(/[^a-z0-9]/g, '_');
+    const cachedClient = this.clients.get(normalizedName);
+
+    if (cachedClient) return cachedClient;
+
+    this.log.debug(`Creating new Redis client for ${normalizedName}`);
+
+    const client = createClient({
+      name: normalizedName,
+      username: config.get('redis.username'),
+      password: config.get('redis.password'),
+      socket: {
+        host: config.get('redis.host'),
+        port: config.get('redis.port'),
+      },
+      ...extra,
+    });
+
+    await client.connect();
+    this.clients.set(normalizedName, client);
+
+    health.registerHook(normalizedName, async () => {
+      await client.ping();
+    });
+
+    return client;
+  }
+
+  /**
+   * Disconnect all clients and clear the cache.
+   */
+  async destroy() {
+    for (const [name, client] of this.clients.entries()) {
+      this.log.debug(`Disconnecting Redis client ${name}`);
+      await client.disconnect();
+    }
+  }
+}
+
+export const Redis = new RedisClass();

package/tsconfig.build.json

@@ -0,0 +1,14 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "./src",
+    "outDir": "dist"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["src/**/*.test.ts"],
+  "references": [
+    { "path": "../lib-config/tsconfig.build.json" },
+    { "path": "../lib-util/tsconfig.build.json" }
+  ]
+}

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": ["src/**/*"]
+}

package/typedoc.json

@@ -0,0 +1,3 @@
+{
+  "entryPoints": ["src/main.ts"]
+}