@takaro/db 0.0.0-next.0da151e

package/src/TakaroModel.ts

@@ -0,0 +1,33 @@
+import Objection, { Model } from 'objection';
+
+export class NOT_DOMAIN_SCOPED_TakaroModel extends Model {
+  id: string;
+  createdAt: string;
+  updatedAt: string;
+
+  static get idColumn() {
+    return 'id';
+  }
+
+  $beforeInsert() {
+    this.createdAt = new Date().toISOString();
+  }
+
+  $beforeUpdate() {
+    this.updatedAt = new Date().toISOString();
+  }
+}
+
+export class TakaroModel extends NOT_DOMAIN_SCOPED_TakaroModel {
+  domain: string;
+
+  static get modifiers() {
+    return {
+      domainScoped(query: Objection.QueryBuilder<Model>, domainId: string) {
+        const tableName = query.modelClass().tableName;
+        if (!domainId) throw new Error('Domain ID is required for domainScoped modifier');
+        query.where(`${tableName}.domain`, domainId);
+      },
+    };
+  }
+}

package/src/__tests__/encryption.integration.test.ts

@@ -0,0 +1,57 @@
+import { compareHashed, decrypt, encrypt, hash } from '../encryption.js';
+import { expect } from '@takaro/test';
+import { getKnex } from '../knex.js';
+import { describe, before, it } from 'node:test';
+
+describe('Database encryption', () => {
+  before(async () => {
+    const knex = await getKnex();
+    await knex.raw('CREATE EXTENSION IF NOT EXISTS pgcrypto;');
+  });
+
+  describe('Encryption', () => {
+    it('Can encrypt a simple string', async () => {
+      const encrypted = await encrypt('test');
+      expect(encrypted).to.not.eq('test');
+    });
+
+    it('Can decrypt a simple string', async () => {
+      const encrypted = await encrypt('test');
+      const decrypted = await decrypt(encrypted);
+      expect(decrypted).to.eq('test');
+    });
+
+    it('Can handle complex JSON structures', async () => {
+      const encrypted = await encrypt(JSON.stringify({ foo: 'bar' }));
+      const decrypted = await decrypt(encrypted);
+      expect(decrypted).to.eq(JSON.stringify({ foo: 'bar' }));
+    });
+
+    it('Can handle very long values', async () => {
+      const longValue = 'a'.repeat(10000);
+      const encrypted = await encrypt(longValue);
+      const decrypted = await decrypt(encrypted);
+      expect(decrypted).to.eq(longValue);
+    });
+  });
+
+  describe('Hashing', async () => {
+    it('Can hash a simple string', async () => {
+      const hashed = await hash('test');
+      expect(hashed).to.not.eq('test');
+      expect(hashed).to.match(/^\$2a\$06\$/);
+    });
+
+    it('Can compare a simple string', async () => {
+      const hashed = await hash('test');
+      const result = await compareHashed('test', hashed);
+      expect(result).to.eq(true);
+    });
+
+    it('Returns false when comparing different strings', async () => {
+      const hashed = await hash('test');
+      const result = await compareHashed('test2', hashed);
+      expect(result).to.eq(false);
+    });
+  });
+});

package/src/__tests__/queryBuilder.integration.test.ts

@@ -0,0 +1,208 @@
+import { disconnectKnex, getKnex } from '../knex.js';
+import { populateModelColumns, QueryBuilder, SortDirection } from '../queryBuilder.js';
+import { expect } from '@takaro/test';
+import { TakaroModel } from '../TakaroModel.js';
+import { Model } from 'objection';
+import { sleep } from '@takaro/util';
+import { describe, beforeEach, afterEach, it } from 'node:test';
+
+const TEST_TABLE_USERS_NAME = 'test_users';
+const TEST_TABLE_POSTS_NAME = 'test_posts';
+
+class TestPostModel extends TakaroModel {
+  static tableName = TEST_TABLE_POSTS_NAME;
+  title!: string;
+  userId!: string;
+
+  static relationMappings = {
+    author: {
+      relation: Model.HasOneRelation,
+      modelClass: () => TestUserModel,
+      join: {
+        from: `${TEST_TABLE_POSTS_NAME}.userId`,
+        to: `${TEST_TABLE_USERS_NAME}.id`,
+      },
+    },
+  };
+}
+
+class TestUserModel extends TakaroModel {
+  static tableName = TEST_TABLE_USERS_NAME;
+  name!: string;
+
+  static relationMappings = {
+    posts: {
+      relation: Model.HasManyRelation,
+      modelClass: TestPostModel,
+      join: {
+        from: `${TEST_TABLE_USERS_NAME}.id`,
+        to: `${TEST_TABLE_POSTS_NAME}.userId`,
+      },
+    },
+  };
+}
+
+describe('QueryBuilder', () => {
+  beforeEach(async () => {
+    const knex = await getKnex();
+    await knex.schema.dropTableIfExists(TEST_TABLE_USERS_NAME);
+    await knex.schema.dropTableIfExists(TEST_TABLE_POSTS_NAME);
+    await knex.schema.createTable(TEST_TABLE_USERS_NAME, (table) => {
+      table.timestamps(true, true, true);
+      table.uuid('id').primary().defaultTo(knex.raw('gen_random_uuid ()'));
+      table.string('name');
+    });
+    await knex.schema.createTable(TEST_TABLE_POSTS_NAME, (table) => {
+      table.timestamps(true, true, true);
+      table.uuid('id').primary().defaultTo(knex.raw('gen_random_uuid ()'));
+      table.string('title');
+      table.uuid('userId');
+    });
+
+    TestUserModel.knex(knex);
+    TestPostModel.knex(knex);
+
+    await populateModelColumns();
+  });
+
+  afterEach(async () => {
+    const knex = await getKnex();
+    await knex.schema.dropTableIfExists(TEST_TABLE_USERS_NAME);
+    await disconnectKnex();
+  });
+
+  it('Can create basic filters', async () => {
+    await TestUserModel.query().insert({ name: 'test1' });
+    await TestUserModel.query().insert({ name: 'test2' });
+    await TestUserModel.query().insert({ name: 'test3' });
+
+    const res = await new QueryBuilder<TestUserModel, TestUserModel>({
+      filters: { name: ['test2'] },
+    }).build(TestUserModel.query());
+
+    expect(res.results).to.have.lengthOf(1);
+    expect(res.results[0].name).to.equal('test2');
+  });
+
+  it('Can filter based on multiple values', async () => {
+    await TestUserModel.query().insert({ name: 'test1' });
+    await TestUserModel.query().insert({ name: 'test2' });
+    await TestUserModel.query().insert({ name: 'test3' });
+
+    const res = await new QueryBuilder<TestUserModel, TestUserModel>({
+      filters: { name: ['test2', 'test3'] },
+      sortBy: 'name',
+    }).build(TestUserModel.query());
+
+    expect(res.results).to.have.lengthOf(2);
+    expect(res.results[0].name).to.equal('test2');
+    expect(res.results[1].name).to.equal('test3');
+  });
+
+  it('Can do paging', async () => {
+    const totalTestRecords = 100;
+
+    for (let i = 0; i < totalTestRecords; i++) {
+      const number = i.toString().padStart(3, '0');
+      await TestUserModel.query().insert({ name: `test${number}` });
+    }
+
+    const res = await new QueryBuilder<TestUserModel, TestUserModel>({
+      page: 2,
+      limit: 10,
+      sortBy: 'name',
+      sortDirection: SortDirection.asc,
+    }).build(TestUserModel.query());
+
+    expect(res.results).to.have.lengthOf(10);
+    expect(res.results[0].name).to.equal('test020');
+    expect(res.total).to.equal(totalTestRecords);
+  });
+
+  it('Can do sorting', async () => {
+    await TestUserModel.query().insert({ name: 'test1' });
+    await TestUserModel.query().insert({ name: 'test2' });
+    await TestUserModel.query().insert({ name: 'test3' });
+
+    const res = await new QueryBuilder<TestUserModel, TestUserModel>({
+      sortBy: 'name',
+      sortDirection: SortDirection.desc,
+    }).build(TestUserModel.query());
+
+    expect(res.results).to.have.lengthOf(3);
+    expect(res.results[0].name).to.equal('test3');
+  });
+
+  it('Can do extending', async () => {
+    const user = await TestUserModel.query().insert({ name: 'test1' });
+    await TestPostModel.query().insert({ title: 'test1', userId: user.id });
+
+    const res = await new QueryBuilder<
+      TestPostModel & { author?: TestUserModel },
+      TestPostModel & { author?: TestUserModel }
+    >({ extend: ['author'] }).build(TestPostModel.query());
+
+    expect(res.results).to.have.lengthOf(1);
+    expect(res.results[0].author?.name).to.equal('test1');
+  });
+
+  it('Can do partial matching of strings', async () => {
+    await TestUserModel.query().insert({ name: 'test1' });
+    await TestUserModel.query().insert({ name: 'test2' });
+    await TestUserModel.query().insert({ name: 'test3' });
+
+    const res = await new QueryBuilder<TestUserModel, TestUserModel>({
+      search: {
+        name: ['test'],
+      },
+    }).build(TestUserModel.query());
+
+    expect(res.results).to.have.lengthOf(3);
+
+    const res2 = await new QueryBuilder<TestUserModel, TestUserModel>({
+      search: {
+        name: ['1'],
+      },
+    }).build(TestUserModel.query());
+
+    expect(res2.results).to.have.lengthOf(1);
+  });
+
+  it('Can search for multiple values, ORs them', async () => {
+    await TestUserModel.query().insert({ name: 'test1' });
+    await TestUserModel.query().insert({ name: 'test2' });
+    await TestUserModel.query().insert({ name: 'test3' });
+
+    const res = await new QueryBuilder<TestUserModel, TestUserModel>({
+      search: {
+        name: ['st1', 'st2'],
+      },
+    }).build(TestUserModel.query());
+
+    expect(res.results).to.have.lengthOf(2);
+  });
+
+  it('Can search between a date range', async () => {
+    const start = new Date();
+    await TestUserModel.query().insert({ name: 'test1' });
+    await TestUserModel.query().insert({ name: 'test2' });
+    const end = new Date();
+
+    // Quick hack to make the test pass
+    // Relying on times like this with an external DB is troublesome...
+    await sleep(250);
+
+    await TestUserModel.query().insert({ name: 'test3' });
+
+    const res = await new QueryBuilder<TestUserModel, TestUserModel>({
+      greaterThan: {
+        createdAt: start.toISOString(),
+      },
+      lessThan: {
+        createdAt: end.toISOString(),
+      },
+    }).build(TestUserModel.query());
+
+    expect(res.results).to.have.lengthOf(2);
+  });
+});

package/src/config.ts

@@ -0,0 +1,122 @@
+import { Config, IBaseConfig, baseConfigConvict } from '@takaro/config';
+
+export interface IDbConfig extends IBaseConfig {
+  postgres: {
+    connectionString: string;
+    host: string;
+    port: number;
+    user: string;
+    password: string;
+    database: string;
+    ssl: boolean;
+    ca: string;
+  };
+  redis: {
+    host: string;
+    port: number;
+    username: string;
+    password: string;
+  };
+  systemSchema: string;
+  baseDomainSchema: string;
+  encryptionKey: string;
+}
+
+export const configSchema = {
+  postgres: {
+    connectionString: {
+      doc: 'The Postgres connection string to use',
+      format: String,
+      default: '',
+      env: 'POSTGRES_CONNECTION_STRING',
+    },
+    host: {
+      doc: 'The Postgres host to connect to',
+      format: String,
+      default: 'localhost',
+      env: 'POSTGRES_HOST',
+    },
+    port: {
+      doc: 'The Postgres port to connect to',
+      format: Number,
+      default: 5432,
+      env: 'POSTGRES_PORT',
+    },
+    user: {
+      doc: 'The Postgres user to connect as',
+      format: String,
+      default: 'postgres',
+      env: 'POSTGRES_USER',
+    },
+    password: {
+      doc: 'The Postgres password to use',
+      format: String,
+      default: 'postgres',
+      env: 'POSTGRES_PASSWORD',
+    },
+    database: {
+      doc: 'The Postgres database to use',
+      format: String,
+      default: 'postgres',
+      env: 'POSTGRES_DB',
+    },
+    ssl: {
+      doc: 'Whether to use SSL for the connection',
+      format: Boolean,
+      default: false,
+      env: 'POSTGRES_SSL',
+    },
+    ca: {
+      doc: 'The CA certificate to use for SSL connections',
+      format: String,
+      default: '',
+      env: 'POSTGRES_CA',
+    },
+  },
+  redis: {
+    host: {
+      doc: 'The host of the redis server',
+      format: String,
+      default: 'localhost',
+      env: 'REDIS_HOST',
+    },
+    port: {
+      doc: 'The port of the redis server',
+      format: Number,
+      default: 6379,
+      env: 'REDIS_PORT',
+    },
+    username: {
+      doc: 'The username of the redis server',
+      format: String,
+      default: '',
+      env: 'REDIS_USERNAME',
+    },
+    password: {
+      doc: 'The password of the redis server',
+      format: String,
+      default: '',
+      env: 'REDIS_PASSWORD',
+    },
+  },
+  systemSchema: {
+    doc: 'The Postgres schema to use for system-related actions (like domain management)',
+    format: String,
+    default: 'takaro',
+    env: 'POSTGRES_SYSTEM_SCHEMA',
+  },
+  baseDomainSchema: {
+    doc: 'String used as base for creating name of domain-scoped schemas',
+    format: String,
+    default: 'domain_',
+    env: 'POSTGRES_BASE_DOMAIN_SCHEMA',
+  },
+  encryptionKey: {
+    doc: 'Encryption key used for encrypting sensitive data',
+    format: String,
+    default: null,
+    env: 'POSTGRES_ENCRYPTION_KEY',
+  },
+};
+
+export const config = new Config<IDbConfig>([configSchema, baseConfigConvict]);

package/src/encryption.ts

@@ -0,0 +1,26 @@
+import { config } from './config.js';
+import { getKnex } from './knex.js';
+
+export async function encrypt(value: string): Promise<string> {
+  const knex = await getKnex();
+  const res = await knex.raw('SELECT PGP_SYM_ENCRYPT(?,?) AS value', [value, config.get('encryptionKey')]);
+  return res.rows[0].value;
+}
+
+export async function decrypt(value: string): Promise<string> {
+  const knex = await getKnex();
+  const res = await knex.raw('SELECT PGP_SYM_DECRYPT(?,?) AS value', [value, config.get('encryptionKey')]);
+  return res.rows[0].value;
+}
+
+export async function hash(value: string): Promise<string> {
+  const knex = await getKnex();
+  const res = await knex.raw("SELECT crypt(?, gen_salt('bf')) as value", [value]);
+  return res.rows[0].value;
+}
+
+export async function compareHashed(value: string, hash: string): Promise<boolean> {
+  const knex = await getKnex();
+  const res = await knex.raw('SELECT crypt(?, ?) = ? as result', [value, hash, hash]);
+  return res.rows[0].result;
+}

package/src/errorTypeGuards.ts

@@ -0,0 +1,11 @@
+import { errors } from '@takaro/util';
+
+export function parseCheckViolationError(error: unknown, constraintName: string): errors.BadRequestError | null {
+  if (!(error instanceof Error)) {
+    return null;
+  }
+  if (error.name === 'CheckViolationError') {
+    return new errors.BadRequestError(`Check constraint "${constraintName}" failed`);
+  }
+  return null;
+}

package/src/knex.ts

@@ -0,0 +1,59 @@
+import knexPkg from 'knex';
+import { config } from './config.js';
+import { logger, health } from '@takaro/util';
+import { TakaroModel } from './TakaroModel.js';
+
+const log = logger('sql');
+
+const { knex: createKnex } = knexPkg;
+
+type KnexClient = knexPkg.Knex<TakaroModel, unknown[]>;
+
+export function getKnexOptions(extra: Record<string, unknown> = {}) {
+  const opts: knexPkg.Knex.Config = {
+    client: 'pg',
+    connection: {
+      connectionString: config.get('postgres.connectionString'),
+      host: config.get('postgres.host'),
+      port: config.get('postgres.port'),
+      user: config.get('postgres.user'),
+      password: config.get('postgres.password'),
+      database: config.get('postgres.database'),
+      ssl: config.get('postgres.ssl') ? { ca: config.get('postgres.ca') } : false,
+    },
+    ...extra,
+  };
+  return opts;
+}
+
+let cachedKnex: KnexClient | null = null;
+
+export async function getKnex(): Promise<KnexClient> {
+  if (cachedKnex) return cachedKnex;
+
+  log.debug('Missed knex cache, creating new client');
+  const knex = createKnex(getKnexOptions());
+  cachedKnex = knex;
+
+  health.registerHook('db', isDbAvailable);
+
+  return cachedKnex;
+}
+
+export async function isDbAvailable(): Promise<boolean> {
+  try {
+    const knex = await getKnex();
+    await knex.raw('SELECT 1');
+    return true;
+  } catch (error) {
+    log.error(error);
+    return false;
+  }
+}
+
+export async function disconnectKnex(): Promise<void> {
+  if (!cachedKnex) return;
+  await cachedKnex.destroy();
+  cachedKnex = null;
+  log.info('Disconnected knex');
+}

package/src/main.ts

@@ -0,0 +1,15 @@
+export { TakaroModel, NOT_DOMAIN_SCOPED_TakaroModel } from './TakaroModel.js';
+
+export { ITakaroQuery, QueryBuilder, SortDirection } from './queryBuilder.js';
+
+export { getKnex, disconnectKnex } from './knex.js';
+
+export { migrate, migrateUndo } from './migrations/index.js';
+
+export * from './encryption.js';
+
+export { configSchema, IDbConfig } from './config.js';
+
+export { Redis, RedisClient } from './redis.js';
+
+export * from './errorTypeGuards.js';

package/src/migrations/index.ts

@@ -0,0 +1,70 @@
+import { getKnex } from '../knex.js';
+import { readdir } from 'fs/promises';
+import { Knex } from 'knex';
+import path from 'node:path';
+import { logger } from '@takaro/util';
+import * as url from 'url';
+const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
+
+const log = logger('db:migrations');
+
+interface IMigration {
+  name: string;
+  up: (knex: Knex) => Promise<void>;
+  down: (knex: Knex) => Promise<void>;
+}
+
+class TakaroMigrationSource {
+  // Hack to get around ts compiler/monorepo/dynamic import weirdness
+  // See: https://github.com/TypeStrong/ts-node/discussions/1290
+  private dynamicImport = new Function('specifier', 'return import(specifier)');
+
+  async getMigrations() {
+    const folderPath = path.join(__dirname, 'sql');
+    const files = await readdir(folderPath);
+    const migrations = files
+      .filter((file) => file.endsWith('.js'))
+      .map((file) => {
+        return this.dynamicImport(`${folderPath}/${file}`).then((migration: IMigration) => {
+          return {
+            name: file,
+            up: migration.up,
+            down: migration.down,
+          };
+        });
+      });
+    return Promise.all(migrations);
+  }
+
+  getMigrationName(migration: IMigration) {
+    return migration.name;
+  }
+
+  async getMigration(migration: IMigration) {
+    return migration;
+  }
+}
+
+export async function migrate() {
+  const knex = await getKnex();
+  knex.on('query', (queryData) => {
+    log.debug(queryData.sql);
+  });
+  await knex.migrate.latest({
+    migrationSource: new TakaroMigrationSource(),
+  });
+  await knex.destroy();
+  log.info('Migrations complete');
+}
+
+export async function migrateUndo() {
+  const knex = await getKnex();
+  knex.on('query', (queryData) => {
+    log.debug(queryData.sql);
+  });
+  await knex.migrate.down({
+    migrationSource: new TakaroMigrationSource(),
+  });
+  await knex.destroy();
+  log.info('Migrations rollback complete');
+}