@tasenor/common-node 1.9.16

package/src/database/DB.ts

@@ -0,0 +1,396 @@
+import { Crypto, DatabaseName, error, Hostname, ID, isDatabaseName, log, Secret, Url } from '@dataplug/tasenor-common'
+import { randomString, vault } from '..'
+import knex, { Knex } from 'knex'
+import { types } from 'pg'
+import { builtins } from 'pg-types'
+import { create as opaque } from 'ts-opaque'
+
+// How many times to retry migration before giving up.
+const MIGRATION_RETRY = 20
+
+// Fix Knex messing around with the date values and timezones.
+const parseDate = (val) => val
+types.setTypeParser(builtins.TIMESTAMPTZ, parseDate)
+types.setTypeParser(builtins.TIMESTAMP, parseDate)
+types.setTypeParser(builtins.DATE, parseDate)
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type KnexDatabase = Knex<any, any[]>
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type KnexConfig = Record<string, any>
+export type KnexConnectionInfo = {
+  host: string,
+  port: string | number,
+  database: string,
+  user: string,
+  password: string
+}
+
+/**
+ * Check if the database with the given name exists.
+ * @param master The master database.
+ * @param name Name of the DB to check.
+ * @returns True if it is found.
+ */
+const exists = async (master: KnexDatabase, name: DatabaseName): Promise<boolean> => {
+  return !!(await master('databases').select('*').where({ name }).first())
+}
+
+/**
+ * Construct a Knex configuration for the given database.
+ * @param master
+ * @param name
+ * @param hostOverride
+ * @returns
+ */
+const getConfig = async (master: KnexDatabase, name: DatabaseName, hostOverride: null | Hostname = null): Promise<KnexConfig> => {
+  const userDb = await master('databases').select('*').where({ name }).first()
+  const password = await Crypto.decrypt(vault.get('SECRET') as Secret, userDb.password)
+  if (!password) {
+    throw new Error('Failed to get password.')
+  }
+  return {
+    client: 'postgresql',
+    connection: {
+      host: hostOverride === null ? userDb.host : hostOverride,
+      port: userDb.port,
+      database: userDb.name,
+      user: userDb.user,
+      password
+    },
+    pool: {
+      min: 1,
+      max: 3
+    }
+  }
+}
+
+const cache = {}
+/**
+ * Get the cached connection to the given customer database.
+ * @param master Master database connection.
+ * @param name Name of the database.
+ * @param hostOverride If given, use this host instead of the host in database.
+ * @returns Knex connection.
+ */
+const get = async (master: KnexDatabase, name: DatabaseName, hostOverride: null | Hostname = null): Promise<KnexDatabase> => {
+  if (!(await exists(master, name))) {
+    delete cache[name]
+    throw new Error(`Database '${name}' does not exist.`)
+  }
+
+  const cacheName = hostOverride ? `${name}:${hostOverride}` : name
+
+  if (!cache[cacheName]) {
+    const knexConfig = await getConfig(master, name, hostOverride)
+    cache[cacheName] = knex(knexConfig)
+  }
+  return cache[cacheName]
+}
+
+/**
+ * Disconnect and drop cached connection.
+ * @param name Name of the database.
+ */
+const disconnect = async (name: string): Promise<void> => {
+  for (const conn of Object.entries(cache)) {
+    if (conn[0].split(':')[0] === name) {
+      log(`Disconnecting ${conn[0]}`)
+      await (conn[1] as Knex).destroy()
+      delete cache[conn[0]]
+    }
+  }
+}
+
+/**
+ * Get the configuration for the root connection.
+ * @returns Configuration object.
+ */
+const getRootConfig = (): KnexConfig => {
+  const url = new URL(vault.get('DATABASE_URL'))
+  const knexConfig = {
+    client: 'postgresql',
+    connection: {
+      host: url.hostname,
+      port: parseInt(url.port) || 5432,
+      database: 'postgres',
+      user: vault.get('DATABASE_ROOT_USER'),
+      password: vault.get('DATABASE_ROOT_PASSWORD')
+    },
+    pool: {
+      min: 1,
+      max: 1
+    }
+  }
+  return knexConfig
+}
+
+/**
+ * Convert database URL to knex config.
+ * @param knexUrl
+ * @returns
+ */
+const getKnexConfig = (knexUrl: Url): KnexConfig => {
+  const url = new URL(knexUrl)
+  return {
+    client: 'postgresql',
+    connection: {
+      host: url.hostname,
+      port: parseInt(url.port) || 5432,
+      database: url.pathname.replace(/^\//, ''),
+      user: url.username,
+      password: url.password
+    },
+    pool: {
+      min: 1,
+      max: 5
+    }
+  }
+}
+
+/**
+ * Get the cached connetion to the root access.
+ * @returns Knex instance.
+ */
+let rootCache
+const getRoot = (): KnexDatabase => {
+  if (rootCache) return rootCache
+
+  rootCache = knex(getRootConfig())
+  return rootCache
+}
+
+const getMasterConfig = (): KnexConfig => {
+  return getKnexConfig(vault.get('DATABASE_URL') as Url)
+}
+
+/**
+ * Get the master database.
+ * @returns A Knex instance.
+ */
+let masterCache
+const getMaster = (): KnexDatabase => {
+  if (masterCache) return masterCache
+
+  masterCache = knex(getMasterConfig())
+  return masterCache
+}
+
+/**
+ * Test if the name is valid database name.
+ * @param name
+ * @returns True if valid.
+ */
+const isValidName = (name: string): boolean => {
+  return isDatabaseName(name)
+}
+
+/**
+ * Create new customer database.
+ * @param db Knex for bookkeeper master database.
+ * @param name Name of the new database.
+ * @param host A host name of the DB host.
+ * @param port A port used by the DB host.
+ * @param migrations Path of the migrations file.
+ * @returns ID
+ */
+const create = async (masterDb: KnexDatabase, name: DatabaseName, host: Hostname, port: number, migrations: string | null = null, hostOverride: null | Hostname = null): Promise<ID> => {
+  if (await exists(masterDb, name)) {
+    throw new Error(`Database '${name}' exist.`)
+  }
+  if (!isValidName(name)) {
+    throw new Error(`Invalid database name '${name}'.`)
+  }
+  const rootDb = getRoot()
+  const user = 'user' + randomString(20)
+  const password = randomString(64)
+  const entry = {
+    name,
+    host,
+    port,
+    user,
+    password: await Crypto.encrypt(vault.get('SECRET') as Secret, password),
+    config: {}
+  }
+  await rootDb.raw(`CREATE USER "${user}" WITH PASSWORD '${password}'`)
+  await rootDb.raw(`CREATE DATABASE "${name}"`)
+  await rootDb.raw(`GRANT ALL PRIVILEGES ON DATABASE "${name}" TO "${user}"`)
+  const id = (await masterDb('databases').insert(entry).returning('id'))[0].id
+
+  if (migrations) {
+    await migrate(masterDb, name, migrations, hostOverride)
+  }
+
+  return id
+}
+
+/**
+ * Apply migrations to the named customer database.
+ * @param masterDb
+ * @param name
+ * @param migrations
+ * @param hostOverride
+ */
+const migrate = async (masterDb: KnexDatabase, name: DatabaseName, migrations: string, hostOverride: null | Hostname = null): Promise<void> => {
+  log(`Migrating database '${name}'.`)
+  const conf = await getConfig(masterDb, name, hostOverride)
+  conf.migrations = { directory: migrations }
+  const db = knex(conf)
+  await db.migrate.latest()
+  // Do not leave hanging connections.
+  await db.destroy()
+}
+
+/**
+ * Apply migrations to the master database.
+ */
+const migrateMaster = async (migrations: string): Promise<void> => {
+  function delay(time) {
+    return new Promise(resolve => setTimeout(resolve, time))
+  }
+
+  log('Migrating master database.')
+
+  const conf = await getMasterConfig()
+  conf.migrations = { directory: migrations }
+  const db = knex(conf)
+  for (let i = 0; i < MIGRATION_RETRY; i++) {
+    try {
+      await db.migrate.latest()
+      log('Migrating master database successful.')
+      break
+    } catch (err) {
+      if (i < MIGRATION_RETRY - 1) {
+        error('Migration failed. Retrying...')
+        await delay(1000)
+      }
+    }
+  }
+  // Do not leave hanging connections.
+  await db.destroy()
+}
+
+/**
+ * Roll bakc last migrations from the named customer database.
+ * @param masterDb
+ * @param name
+ * @param migrations
+ * @param hostOverride
+ */
+const rollback = async (masterDb: KnexDatabase, name: DatabaseName, migrations: string, hostOverride: null | Hostname = null): Promise<void> => {
+  const conf = await getConfig(masterDb, name, hostOverride)
+  conf.migrations = { directory: migrations }
+  const db = knex(conf)
+  await db.migrate.rollback()
+  // Do not leave hanging connections.
+  await db.destroy()
+}
+
+/**
+ * Destroy customer database.
+ * @param masterDb
+ * @param name
+ * @param hostOverride
+ */
+const destroy = async (masterDb: KnexDatabase, name: DatabaseName, hostOverride: null | Hostname = null): Promise<string|null> => {
+  const db = await masterDb('databases').where({ name }).first()
+  if (!db) {
+    return 'Database not found.'
+  }
+  const dbToDelete = await get(masterDb, name, hostOverride)
+  await dbToDelete.raw(`DROP OWNED BY ${db.user}`)
+
+  disconnect(name)
+
+  await masterDb('database_users').where({ database_id: db.id }).delete()
+  await masterDb('databases').where({ id: db.id }).delete()
+
+  const rootDb = getRoot()
+  await rootDb.raw(`DROP OWNED BY ${db.user}`)
+  await rootDb.raw(`DROP USER ${db.user}`)
+  await rootDb.raw(`DROP DATABASE ${db.name} WITH (FORCE)`)
+
+  return null
+}
+
+/**
+ * Generate a name for a database that does not yet exist.
+ * @param masterDb - Master database.
+ * @param init - Initial part of the name.
+ * @returns Suitable name.
+ */
+async function findName(masterDb: KnexDatabase, init: string): Promise<string> {
+  let n = 1
+  let name = init
+  while (await exists(masterDb, opaque(name))) {
+    n++
+    name = `${init}${n}`
+  }
+  return name
+}
+
+/**
+ * Terminate all DB connections.
+ */
+async function disconnectAll() {
+  if (masterCache) {
+    log('Disconnecting master DB.')
+    await masterCache.destroy()
+  }
+  if (rootCache) {
+    log('Disconnecting root DB.')
+    await rootCache.destroy()
+  }
+  for (const conn of Object.entries(cache)) {
+    log(`Disconnecting ${conn[0]}`)
+    await (conn[1] as Knex).destroy()
+  }
+}
+
+/**
+ * Get a list of all customer database connection infos.
+ */
+async function customerDbs(hostOverride: null | Hostname = null): Promise<KnexConnectionInfo[]> {
+  const all: KnexConnectionInfo[] = []
+  await vault.initialize()
+  const master = await getMaster()
+  const dbs = await master('databases').select('name', 'host', 'port', 'user', 'password')
+  for (const db of dbs) {
+    const conf = await getConfig(master, db.name, hostOverride)
+    const { host, port, database, user, password } = conf.connection as KnexConnectionInfo
+    all.push({ host, port, database, user, password })
+  }
+  return all
+}
+
+/**
+ * Resolve database host from environment DATABASE_URL if set.
+ */
+function envHost(): string | null {
+  if (!process.env.DATABASE_URL) {
+    return null
+  }
+  const url = new URL(process.env.DATABASE_URL)
+  return url.hostname
+}
+
+export const DB = {
+  create,
+  customerDbs,
+  destroy,
+  disconnectAll,
+  envHost,
+  exists,
+  findName,
+  get,
+  getConfig,
+  getKnexConfig,
+  getMaster,
+  getMasterConfig,
+  getRoot,
+  getRootConfig,
+  isValidName,
+  migrate,
+  migrateMaster,
+  rollback
+}

package/src/database/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Database connectivity and utilities.
+ *
+ * @module tasenor-common-node/src/database
+ */
+export * from './BookkeeperImporter'
+export * from './DB'

package/src/doccer.ts

@@ -0,0 +1,29 @@
+/**
+ *
+ * # Tasenor Common for Node
+ *
+ * Type definitions and functionality implementation for Node.
+ *
+ * An interactive stateful process is a backend system which handles a process with a
+ * states, but the transition from one state to another could need user input from
+ * the client - e.g. web browser.
+ *
+ * ## Sub-modules
+ *
+ * * [cli](tasenor_common_node_src_cli.html)
+ * * [database](tasenor_common_node_src_database.html)
+ * * [export](tasenor_common_node_src_export.html)
+ * * [import](tasenor_common_node_src_import.html)
+ * * [net](tasenor_common_node_src_net.html)
+ * * [plugins](tasenor_common_node_src_plugins.html)
+ * * [reports](tasenor_common_node_src_reports.html)
+ * * [system](tasenor_common_node_src_system.html)
+ * * [common](tasenor_common_node_src_common.html)
+ * * [error](tasenor_common_node_src_error.html)
+ * * [import](tasenor_common_node_src_import.html)
+ * * [process](tasenor_common_node_src_process.html)
+ * * [server](tasenor_common_node_src_server.html)
+ *
+ * @module tasenor-common-node
+ */
+export * from './index'

package/src/error.ts

@@ -0,0 +1,32 @@
+/**
+ * Error types for processing and an exception used to ask more information from UI.
+ *
+ * @module tasenor-common-node/src/error
+ */
+
+import { TasenorElement } from '@dataplug/tasenor-common'
+
+export class ProcessingError extends Error {}
+export class InvalidFile extends ProcessingError {}
+export class InvalidArgument extends ProcessingError {}
+export class BadState extends ProcessingError {}
+export class NotImplemented extends ProcessingError {}
+export class NotFound extends ProcessingError {}
+export class DatabaseError extends ProcessingError {}
+export class SystemError extends ProcessingError {}
+
+/**
+ * Special exception to halt processing in order to require more configuration information from UI.
+ */
+export class AskUI extends Error {
+  public element: TasenorElement
+
+  constructor(element: TasenorElement) {
+    super('Need more information from UI.')
+    this.element = element
+  }
+}
+
+export function isAskUI(obj: unknown): obj is AskUI {
+  return (obj instanceof Error) && 'element' in obj
+}

package/src/export/Exporter.ts

@@ -0,0 +1,136 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+import path from 'path'
+import fs from 'fs'
+import { KnexDatabase } from '..'
+import { log, ParsedTsvFileData, BookkeeperConfig, DirectoryPath, TsvFilePath, JsonFilePath, Json, Value, TarFilePath } from '@dataplug/tasenor-common'
+import { create } from 'ts-opaque'
+import { system } from '../system'
+import dayjs from 'dayjs'
+
+/**
+ * Common functionality for exporters.
+ */
+export class Exporter {
+
+  /**
+   * Version number of the file format produced by this exporter.
+   */
+  VERSION = 2
+
+  /**
+   * Read all accounts from the database and generate TSV-data.
+   * @param db Knex connection to use.
+   * @returns
+   */
+  async getAccounts(db: KnexDatabase): Promise<ParsedTsvFileData> {
+    throw new Error(`Exporter ${this.constructor.name} does not implement getAccounts().`)
+  }
+
+  /**
+   * Read all periods from the database and generate TSV-data.
+   * @param db Knex connection to use.
+   * @returns
+   */
+  async getPeriods(db: KnexDatabase): Promise<ParsedTsvFileData> {
+    throw new Error(`Exporter ${this.constructor.name} does not implement getPeriods().`)
+  }
+
+  /**
+   * Read all entries and documents from the database and generate TSV-data.
+   * @param db Knex connection to use.
+   * @returns
+   */
+  async getEntries(db: KnexDatabase): Promise<ParsedTsvFileData> {
+    throw new Error(`Exporter ${this.constructor.name} does not implement getEntries().`)
+  }
+
+  /**
+   * Read configuration information from database and construct compiled configuration.
+   * @param db Knex connection to use.
+   * @returns
+   */
+  async getConfig(db: KnexDatabase): Promise<BookkeeperConfig> {
+    throw new Error(`Exporter ${this.constructor.name} does not implement getConfig().`)
+  }
+
+  /**
+   * Read all tags from the database and generate TSV-data.
+   * @param db Knex connection to use.
+   * @param out Directory to write image files.
+   * @returns
+   */
+  async getTags(db: KnexDatabase, out: DirectoryPath): Promise<ParsedTsvFileData> {
+    throw new Error(`Exporter ${this.constructor.name} does not implement getTags().`)
+  }
+
+  /**
+   * Write prepared data to TSV file.
+   * @param path Output file path.
+   * @param lines Data content.
+   */
+  writeTsv(path: TsvFilePath, lines: ParsedTsvFileData): void {
+    log(`Writing ${path}`)
+    fs.writeFileSync(path, lines.map(l => l.join('\t')).join('\n') + '\n')
+  }
+
+  /**
+   * Write prepared data to JSON file.
+   * @param path Output file path.
+   * @param lines Data content.
+   */
+  writeJson(path: JsonFilePath, data: Json): void {
+    log(`Writing ${path}`)
+    fs.writeFileSync(path, JSON.stringify(data, null, 4) + '\n')
+  }
+
+  /**
+   * Save complete backup of the Sqlite database to the given directory.
+   * @param db Database connection.
+   * @param out Directory to store all files.
+   * @returns Configuration constructed from the database.
+   */
+  async dump(db: KnexDatabase, out: DirectoryPath): Promise<BookkeeperConfig> {
+
+    const accountDir = path.join(out, 'accounts')
+    if (!fs.existsSync(accountDir)) {
+      fs.mkdirSync(accountDir)
+    }
+    log(`Saving file format version ${this.VERSION}.`)
+    this.writeJson(create(path.join(out, 'VERSION')), this.VERSION)
+    const conf = await this.getConfig(db)
+    this.writeJson(create(path.join(out, 'settings.json')), conf as unknown as Value)
+    const accounts = await this.getAccounts(db)
+    this.writeTsv(create(path.join(accountDir, 'fi-EUR.tsv')), accounts)
+    const periods = await this.getPeriods(db)
+    this.writeTsv(create(path.join(out, 'periods.tsv')), periods)
+    const entries = await this.getEntries(db)
+    this.writeTsv(create(path.join(out, 'entries.tsv')), entries)
+    const tags = await this.getTags(db, out)
+    this.writeTsv(create(path.join(out, 'tags.tsv')), tags)
+
+    return conf
+  }
+
+  /**
+   * Construct a tar-package for the given configuration from the source directory.
+   * @param conf Configuration found from the database.
+   * @param out Directory containing files extracted as a backup.
+   * @param destPath Destionation file name if given.
+   * @returns Path to the tar-package.
+   */
+  async makeTar(conf: BookkeeperConfig, out: DirectoryPath, destPath: DirectoryPath | undefined): Promise<TarFilePath> {
+    const name = conf.companyName || 'unknown'
+    const tar = `${name.replace(/[^-a-zA-Z0-9]/, '_')}-${dayjs().format('YYYY-MM-DD')}.tasenor`
+    const tarPath = `${out}/../${tar}`
+    const dest = process.cwd()
+    if (!destPath) {
+      destPath = create<DirectoryPath>(path.join(dest, tar))
+    }
+    if (path.dirname(destPath) === '.') {
+      destPath = create<DirectoryPath>(path.join(dest, path.basename(destPath)))
+    }
+    await system(`cd "${out}" && tar cjf "${tarPath}" . && mv "${tarPath}" "${destPath}" && rm -fr ${out}`)
+    log(`Package ready ${destPath}`)
+    return destPath as unknown as TarFilePath
+  }
+}