@tasenor/common-node 1.9.16

package/src/process/ProcessStep.ts

@@ -0,0 +1,100 @@
+import { Process } from './Process'
+import { DatabaseError } from '../error'
+import { KnexDatabase } from '../database'
+import { ID, Directions, ImportAction, ImportState } from '@dataplug/tasenor-common'
+
+/**
+ * A basic information of the processing step.
+ */
+export interface ProcessStepData {
+  processId?: ID
+  number: number
+  state: ImportState
+  handler: string
+  action?: ImportAction
+  directions?: Directions
+  started?: Date
+  finished?: Date
+}
+
+/**
+ * Data of the one step in the process including possible directions and action taken to the next step, if any.
+ */
+export class ProcessStep {
+
+  process: Process
+
+  id: ID
+  processId: ID
+  number: number
+  state: ImportState
+  handler: string
+  started: Date | undefined
+  finished: Date | undefined
+  directions?: Directions
+  action?: ImportAction | undefined
+
+  constructor(obj: ProcessStepData) {
+    this.processId = obj.processId || null
+    this.number = obj.number
+    this.state = obj.state
+    this.handler = obj.handler
+    this.directions = obj.directions ? new Directions(obj.directions) : undefined
+    this.action = obj.action
+    this.started = obj.started
+    this.finished = obj.finished
+  }
+
+  toString(): string {
+    return `ProcessStep ${this.number} of Process #${this.processId}`
+  }
+
+  /**
+   * Get a reference to the database.
+   */
+  get db(): KnexDatabase {
+    return this.process.db
+  }
+
+  /**
+   * Save the process info to the database.
+   */
+  async save(): Promise<ID> {
+    if (this.id) {
+      await this.db('process_steps').update(this.toJSON()).where({ id: this.id })
+      return this.id
+    } else {
+      this.started = new Date()
+      this.id = (await this.db('process_steps').insert(this.toJSON()).returning('id'))[0].id
+      if (this.id) return this.id
+      throw new DatabaseError(`Saving process ${JSON.stringify(this.toJSON)} failed.`)
+    }
+  }
+
+  /**
+   * Get the loaded process information as JSON object.
+   * @returns
+   */
+  toJSON(): ProcessStepData {
+    return {
+      processId: this.processId,
+      number: this.number,
+      state: this.state,
+      directions: this.directions,
+      handler: this.handler,
+      action: this.action,
+      started: this.started,
+      finished: this.finished
+    }
+  }
+
+  /**
+   * Set directions and update database.
+   * @param db
+   * @param directions
+   */
+  async setDirections(db: KnexDatabase, directions: Directions): Promise<void> {
+    this.directions = directions
+    await db('process_steps').update({ directions: directions.toJSON() }).where({ id: this.id })
+  }
+}

package/src/process/ProcessingSystem.ts

@@ -0,0 +1,202 @@
+import { Process } from './Process'
+import { ProcessFile, ProcessFileData } from './ProcessFile'
+import { ProcessStep } from './ProcessStep'
+import { ProcessHandler, ProcessHandlerMap } from './ProcessHandler'
+import { ProcessConnector } from './ProcessConnector'
+import { ProcessName, ProcessConfig, ID } from '@dataplug/tasenor-common'
+import { InvalidArgument } from '../error'
+import { KnexDatabase } from '../database'
+
+/**
+ * An instance of the full processing system.
+ */
+export class ProcessingSystem {
+
+  db: KnexDatabase
+  handlers: ProcessHandlerMap = {}
+  connector: ProcessConnector
+  logger: {
+    info: (...msg) => void
+    error: (...msg) => void
+  }
+
+  /**
+   * Initialize the system and set the database instance for storing process data.
+   * @param db
+   */
+  constructor(db: KnexDatabase, connector: ProcessConnector) {
+    this.db = db
+    this.logger = {
+      info: (...msg) => console.log(new Date(), ...msg),
+      error: (...msg) => console.error(new Date(), ...msg)
+    }
+    this.connector = connector
+  }
+
+  /**
+   * Get the translation from the connector.
+   * @param language
+   * @param text
+   * @returns
+   */
+  async getTranslation(text: string, language: string): Promise<string> {
+    return this.connector.getTranslation(text, language)
+  }
+
+  /**
+   * Register new handler class for processing.
+   * @param handler
+   */
+  register(handler: ProcessHandler): void {
+    if (!handler) {
+      throw new InvalidArgument('A handler was undefined.')
+    }
+    if (!handler.name) {
+      throw new InvalidArgument('A handler without name cannot be registered.')
+    }
+    if (handler.name in this.handlers) {
+      throw new InvalidArgument(`The handler '${handler.name}' is already defined.`)
+    }
+    if (handler.name.length > 32) {
+      throw new InvalidArgument(`The handler name '${handler.name}' is too long.`)
+    }
+    handler.system = this
+    this.handlers[handler.name] = handler
+  }
+
+  /**
+   * Initialize new process and save it to the database.
+   * @param type
+   * @param name
+   * @param file
+   * @returns New process that is already in crashed state, if no handler
+   */
+  async createProcess(name: ProcessName, files: ProcessFileData[], config: ProcessConfig): Promise<Process> {
+    // Set up the process.
+    const process = new Process(this, name, config)
+    await process.save()
+
+    // Check if we have files.
+    if (files.length < 1) {
+      await process.crashed(new InvalidArgument('No files given to create a process.'))
+      return process
+    }
+
+    // Save the first file and attach it to the process.
+    const file = files[0]
+    const processFile = new ProcessFile(file)
+    process.addFile(processFile)
+    await processFile.save(this.db)
+
+    // Find the handler.
+    let selectedHandler: ProcessHandler | null = null
+    for (const handler of Object.values(this.handlers)) {
+      try {
+        if (handler.canHandle(processFile)) {
+          selectedHandler = handler
+          break
+        }
+      } catch (err) {
+        await process.crashed(err)
+        return process
+      }
+    }
+    if (!selectedHandler) {
+      await process.crashed(new InvalidArgument(`No handler found for the file ${file.name} of type ${file.type}.`))
+      return process
+    }
+
+    // Check if the handler accepts the rest of the files.
+    for (let i = 1; i < files.length; i++) {
+      const processFile = new ProcessFile(files[i])
+      // await processFile.save(this.db)
+      if (!selectedHandler.canAppend(processFile)) {
+        await process.crashed(new InvalidArgument(`The file ${files[i].name} of type ${files[i].type} cannot be appended to handler.`))
+        return process
+      }
+      process.addFile(processFile)
+      await processFile.save(this.db)
+    }
+
+    // Create initial step using the handler.
+    let state
+    try {
+      state = selectedHandler.startingState(process.files)
+    } catch (err) {
+      await process.crashed(err)
+      return process
+    }
+    const step = new ProcessStep({
+      number: 0,
+      handler: selectedHandler.name,
+      state
+    })
+
+    process.addStep(step)
+    await step.save()
+
+    process.currentStep = 0
+    await process.save()
+    this.logger.info(`Created process ${process}.`)
+
+    // Find directions forward from the initial state.
+    await this.checkFinishAndFindDirections(selectedHandler, step)
+
+    return process
+  }
+
+  /**
+   * Check if we are in the finished state and if not, find the directions forward.
+   */
+  async checkFinishAndFindDirections(handler: ProcessHandler, step: ProcessStep): Promise<void> {
+    let result
+    try {
+      result = handler.checkCompletion(step.state)
+    } catch (err) {
+      return step.process.crashed(err)
+    }
+
+    if (result === undefined) {
+      let directions
+      try {
+        directions = await handler.getDirections(step.state, step.process.config)
+      } catch (err) {
+        return step.process.crashed(err)
+      }
+      await step.setDirections(this.db, directions)
+    } else {
+      // Process is finished.
+      step.directions = undefined
+      step.action = undefined
+      step.finished = new Date()
+      await step.save()
+      step.process.complete = true
+      step.process.successful = result
+      await step.process.save()
+    }
+    await step.process.updateStatus()
+  }
+
+  /**
+   * Get the named handler or throw an error if not registered.
+   * @param name
+   * @returns
+   */
+  getHandler(name: string): ProcessHandler {
+    if (!(name in this.handlers)) {
+      throw new InvalidArgument(`There is no handler for '${name}'.`)
+    }
+    return this.handlers[name]
+  }
+
+  /**
+   * Load the process data from the disk.
+   * @param id
+   * @returns
+   */
+  async loadProcess(id: ID): Promise<Process> {
+    const process = new Process(this, null)
+    await process.load(id)
+    return process
+  }
+}

package/src/process/index.ts

@@ -0,0 +1,11 @@
+/**
+ * Implementation for process handling.
+ *
+ * @module tasenor-common-node/src/process
+ */
+export * from './Process'
+export * from './ProcessConnector'
+export * from './ProcessFile'
+export * from './ProcessingSystem'
+export * from './ProcessStep'
+export * from './ProcessHandler'

package/src/reports/conversions.ts

@@ -0,0 +1,52 @@
+import { ReportQueryParams } from '@dataplug/tasenor-common'
+import json2csv from 'json2csv'
+import { sprintf } from 'sprintf-js'
+
+/**
+ * Convert report to CSV format.
+ * @param {Object} report
+ * @param {Object} options
+ * @param {String} options.lang Localize number using this language.
+ */
+export function data2csv(report, options: ReportQueryParams) {
+  const csv: Record<string, string>[] = []
+
+  const render = {
+    id: (column, entry) => entry.id,
+    name: (column, entry) => `${entry.isAccount ? entry.number + ' ' : ''}${entry.name}`,
+    text: (column, entry) => entry[column.name],
+    numeric: (column, entry) => (entry.amounts &&
+      !entry.hideTotal &&
+      entry.amounts[column.name] !== '' &&
+      !isNaN(entry.amounts[column.name]) &&
+      entry.amounts[column.name] !== undefined)
+      ? (entry.amounts[column.name] === null ? '—' : sprintf('%.2f', entry.amounts[column.name] / 100))
+      : ''
+  }
+
+  const { data, columns } = report
+  let line: Record<string, string> = {}
+  if (!options.dropTitle) {
+    columns.forEach((column) => (line[column.name] = column.title))
+    csv.push(line)
+  }
+
+  data.forEach((entry) => {
+    if (entry.paragraphBreak) {
+      return
+    }
+    line = {}
+    columns.forEach((column) => {
+      if (entry.pageBreak || entry.paragraphBreak) {
+        line[column.name] = ''
+      } else {
+        line[column.name] = render[column.type](column, entry)
+      }
+    })
+    csv.push(line)
+  })
+
+  const fields = columns.map((c) => c.name)
+
+  return json2csv.parse(csv, { fields, header: false })
+}

package/src/reports/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Utilities for report building and handling.
+ *
+ * @module tasenor-common-node/src/reports
+ */
+export * from './conversions'

package/src/server/ISPDemoServer.ts

@@ -0,0 +1,122 @@
+import path from 'path'
+import express from 'express'
+import { Server } from 'http'
+import fs from 'fs'
+import Knex from 'knex'
+import cors from 'cors'
+import { router } from './router'
+import { ID } from '@dataplug/tasenor-common'
+import { ProcessHandler, ProcessConnector, defaultConnector, ProcessingSystem } from '../process'
+import { KnexDatabase } from '../database'
+
+/**
+ * Simple demo server for one or more handler.
+ *
+ * Usage:
+ * ```
+ *  const handler1 = new MyCustomHandler('Custom 1')
+ *  const handler2 = new MyCustomHandler('Custom 2')
+ *  const server = new ISPDemoServer(PORT, DATABASE_URL, [handler1, handler2])
+ *  server.start()
+ * ```
+ */
+export class ISPDemoServer {
+  private app = express()
+  private server: Server
+  private port: number
+  private db: KnexDatabase
+  private handlers: ProcessHandler[]
+  private connector: ProcessConnector
+  private configDefaults: Record<string, unknown>
+
+  /**
+   * Prepare settings.
+   *
+   * @param port
+   * @param databaseUrl
+   * @param handlers
+   * @param connector
+   */
+  constructor(port: number, databaseUrl: string, handlers: ProcessHandler[], connector: ProcessConnector|null = null, configDefaults: Record<string, unknown> = {}) {
+    this.port = port
+    this.configDefaults = configDefaults
+    let migrationsPath = path.normalize(path.join(__dirname, '/migrations/01_init.js'))
+    if (!fs.existsSync(migrationsPath)) {
+      migrationsPath = path.normalize(path.join(__dirname, '../../dist/migrations/01_init.js'))
+    }
+    if (!fs.existsSync(migrationsPath)) {
+      migrationsPath = path.normalize(path.join(__dirname, '../../../dist/migrations/01_init.js'))
+    }
+    if (!fs.existsSync(migrationsPath)) {
+      console.log(__dirname)
+      throw new Error(`Cannot find migrations file '${migrationsPath}'.`)
+    }
+    this.db = Knex({
+      client: 'pg',
+      connection: databaseUrl,
+      migrations: {
+        directory: path.dirname(migrationsPath)
+      }
+    })
+    this.handlers = handlers
+    if (connector) {
+      this.connector = connector
+    } else {
+      this.connector = defaultConnector
+    }
+  }
+
+  /**
+   * Launch the demo server.
+   *
+   * @param reset If set, reset the database on boot.
+   */
+  public start = async (reset = false): Promise<void> => {
+
+    if (reset) {
+      await this.db.migrate.rollback()
+    }
+    await this.db.migrate.latest()
+
+    const systemCreator = () => {
+      const system = new ProcessingSystem(this.db, this.connector)
+      this.handlers.forEach(handler => system.register(handler))
+      return system
+    }
+
+    this.app.use((req, res, next) => { res.locals.server = this; next() })
+    this.app.use((req, res, next) => { console.log(new Date(), req.method, req.url); next() })
+    this.app.use(cors())
+    this.app.use(express.json({ limit: '1024MB' }))
+    this.app.use('/api/isp', router(this.db, systemCreator))
+
+    this.server = this.app.listen(this.port, () => {
+      console.log(new Date(), `Server started on port ${this.port}.`)
+      this.connector.initialize(this)
+    })
+
+    this.server.on('error', (msg) => {
+      console.error(new Date(), msg)
+    })
+  }
+
+  /**
+   * Exit the server. If an error is given, raise also that error.
+   * @param err
+   */
+  public stop = async (err: Error | undefined = undefined): Promise<void> => {
+    console.log(new Date(), 'Stopping the server.')
+    await this.server.close(() => {
+      if (err) {
+        throw err
+      } else {
+        process.exit()
+      }
+    })
+  }
+
+  async lastProcessID(): Promise<ID> {
+    const ids = await this.db('processes').max('id').first()
+    return ids ? ids.max : null
+  }
+}

package/src/server/api.ts

@@ -0,0 +1,37 @@
+import { ProcessModelData, ID, ProcessModelDetailedData, ProcessStepModelData } from '@dataplug/tasenor-common'
+import { KnexDatabase } from '../database'
+
+export type ProcessApi = {
+  process: {
+    getAll: () => Promise<ProcessModelData[]>,
+    get: (id: ID) => Promise<ProcessModelDetailedData>
+    getStep: (id: ID, step: number) => Promise<ProcessStepModelData>
+  }
+}
+
+/**
+ * Data query API for processes.
+ * @param db
+ * @returns
+ */
+export default function(db: KnexDatabase): ProcessApi {
+  return {
+    process: {
+      getAll: async (): Promise<ProcessModelData[]> => {
+        return db('processes').select('*').orderBy('created', 'desc')
+      },
+      get: async (id: ID): Promise<ProcessModelDetailedData> => {
+        const data = await db('processes').select('*').where({ id }).first()
+        if (data) {
+          const steps = await db('process_steps').select('id', 'action', 'directions', 'number', 'started', 'finished').where({ processId: id }).orderBy('number')
+          data.steps = steps || []
+        }
+        return data
+      },
+      getStep: async (id: ID, number: number): Promise<ProcessStepModelData> => {
+        const data = await db('process_steps').select('*').where({ processId: id, number }).first()
+        return data
+      }
+    }
+  }
+}

package/src/server/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Server related helpers.
+ *
+ * @module tasenor-common-node/src/server
+ */
+export * from './router'
+export * from './ISPDemoServer'

package/src/server/router.ts

@@ -0,0 +1,60 @@
+import express, { Request, Router } from 'express'
+import { KnexDatabase } from '../database'
+import { ProcessingSystem } from '../process'
+import apiCreator from './api'
+
+export type ProcessingConfigurator = (req: Request) => ProcessingSystem
+export function router(db: KnexDatabase, configurator: ProcessingConfigurator): Router {
+
+  const router = express.Router()
+  const api = apiCreator(db)
+
+  router.get('/',
+    async (req, res) => {
+      return res.send(await api.process.getAll())
+    }
+  )
+
+  router.get('/:id',
+    async (req, res) => {
+      return res.send(await api.process.get(parseInt(req.params.id)))
+    }
+  )
+
+  router.post('/',
+    async (req, res) => {
+      const system = configurator(req)
+      const { files, config } = req.body
+      const names = files.map(f => f.name)
+      const process = await system.createProcess(
+        `Uploading files ${names.join(', ')}`,
+        files,
+        { ...res.locals.server.configDefaults, ...config }
+      )
+      if (process.canRun()) {
+        await process.run()
+      }
+      return res.send(await api.process.get(process.id))
+    })
+
+  router.post('/:id',
+    async (req, res) => {
+      const system = configurator(req)
+      const { id } = req.params
+      const process = await system.loadProcess(parseInt(id))
+      await process.input(req.body)
+      if (process.canRun()) {
+        await process.run()
+      }
+      res.sendStatus(204)
+    }
+  )
+
+  router.get('/:id/step/:number',
+    async (req, res) => {
+      return res.send(await api.process.getStep(parseInt(req.params.id), parseInt(req.params.number)))
+    }
+  )
+
+  return router
+}

package/src/system.ts

@@ -0,0 +1,96 @@
+/**
+ * Server operating system related functions.
+ *
+ * @module tasenor-common-node/src/system
+ */
+import { exec, spawn } from 'child_process'
+import { log, error, note } from '@dataplug/tasenor-common'
+
+/**
+ * Helper to execute system command as a promise.
+ * @param command A command.
+ * @param quiet If set, do not output.
+ * @returns Srandard output if successfully executed.
+ */
+export async function system(command: string, quiet = false): Promise<string> {
+  if (!quiet) {
+    log(`Running system command: ${command}`)
+  }
+  return new Promise((resolve, reject) => {
+    exec(command, { maxBuffer: 1024 * 1024 * 500 }, (err, stdout, stderr) => {
+      if (err) {
+        if (!quiet) error(err)
+        return reject(err)
+      }
+      if (stderr && !quiet) {
+        note(`${stderr}`)
+      }
+      if (stdout && !quiet) {
+        log(`${stdout}`)
+      }
+      resolve(stdout)
+    })
+  })
+}
+
+/**
+ * A system call showing real time output of stdout.
+ * @param command
+ * @param quiet
+ * @returns
+ */
+export async function systemPiped(command: string, quiet = false, ignoreError = false): Promise<string | null> {
+  if (!quiet) {
+    log(`Running system command: ${command}`)
+  }
+  return new Promise((resolve, reject) => {
+    let out = ''
+    const proc = spawn(command, { shell: true })
+    proc.stdout.on('data', (data) => {
+      out += data
+      if (!quiet) process.stdout.write(data)
+    })
+
+    proc.stderr.on('data', (data) => {
+      if (!quiet) process.stderr.write(data)
+    })
+
+    proc.on('close', (code) => {
+      if (code) {
+        if (ignoreError) {
+          resolve(null)
+        } else {
+          reject(new Error(`Call '${command}' failed with code ${code}.`))
+        }
+      } else {
+        resolve(out)
+      }
+    })
+  })
+}
+
+/**
+ * Check if the current environment is not development environment.
+ */
+export function isProduction(): boolean {
+  return !isDevelopment()
+}
+
+/**
+ * Check and return the operating environment.
+ * @returns
+ */
+export function nodeEnv() {
+  const env = process.env.NODE_ENV || 'production'
+  if (!['development', 'staging', 'production'].includes(env)) {
+    throw new Error(`Invalid NODE_ENV ${env}.`)
+  }
+  return env
+}
+
+/**
+ * Check if the current environment is development environment.
+ */
+export function isDevelopment(): boolean {
+  return nodeEnv() === 'development'
+}