@tasenor/common-node 1.9.16

package/src/process/Process.ts

@@ -0,0 +1,368 @@
+import clone from 'clone'
+import { BadState, DatabaseError, InvalidArgument, isAskUI } from '../error'
+import { ProcessFile } from './ProcessFile'
+import { ProcessingSystem } from './ProcessingSystem'
+import { ProcessStep } from './ProcessStep'
+import { Directions, ProcessName, ProcessConfig, ProcessStatus, ID, TasenorElement, ImportAction, ImportState } from '@dataplug/tasenor-common'
+import { KnexDatabase } from '../database'
+
+/**
+ * Overall description of the process.
+ */
+export interface ProcessInfo {
+  name: ProcessName
+  config: ProcessConfig
+  complete: boolean
+  successful: boolean | undefined
+  currentStep: number | undefined
+  status: ProcessStatus
+  error?: string
+}
+
+/**
+ * A complete description of the process state and steps taken.
+ */
+export class Process {
+
+  system: ProcessingSystem
+
+  id: ID
+  name: ProcessName
+  config: ProcessConfig
+  complete: boolean
+  successful: boolean | undefined
+  currentStep: number | undefined
+  status: ProcessStatus
+  files: ProcessFile[]
+  steps: ProcessStep[]
+  error: string | undefined
+
+  constructor(system: ProcessingSystem, name: ProcessName | null, config: ProcessConfig = {}) {
+    this.system = system
+
+    this.id = null
+    this.config = config
+    this.name = name || '[no name]'
+    this.complete = false
+    this.successful = undefined
+    this.files = []
+    this.steps = []
+    this.currentStep = undefined
+    this.status = 'INCOMPLETE' as ProcessStatus
+  }
+
+  toString(): string {
+    return `Process #${this.id} ${this.name}`
+  }
+
+  /**
+   * Get the loaded process information as JSON object.
+   * @returns
+   */
+  toJSON(): ProcessInfo {
+    return {
+      name: this.name,
+      config: this.config,
+      complete: this.complete,
+      successful: this.successful,
+      currentStep: this.currentStep,
+      status: this.status,
+      error: this.error
+    }
+  }
+
+  /**
+   * Append a file to this process and link its ID.
+   * @param file
+   */
+  addFile(file: ProcessFile): void {
+    file.processId = this.id
+    this.files.push(file)
+  }
+
+  /**
+   * Append a step to this process and link its ID.
+   * @param step
+   */
+  async addStep(step: ProcessStep): Promise<void> {
+    step.processId = this.id
+    step.process = this
+    this.steps.push(step)
+  }
+
+  /**
+   * Load the current step if not yet loaded and return it.
+   */
+  async getCurrentStep(): Promise<ProcessStep> {
+    if (this.currentStep === null || this.currentStep === undefined) {
+      throw new BadState(`Process #${this.id} ${this.name} has invalid current step.`)
+    }
+    if (this.steps[this.currentStep]) {
+      return this.steps[this.currentStep]
+    }
+    return this.loadStep(this.currentStep)
+  }
+
+  /**
+   * Mark the current state as completed and create new additional step with the new state.
+   * @param state
+   */
+  async proceedToState(action: ImportAction, state: ImportState): Promise<void> {
+    const current = await this.getCurrentStep()
+    const handler = this.system.getHandler(current.handler)
+    current.action = action
+    current.finished = new Date()
+    current.save()
+    const nextStep = new ProcessStep({
+      number: current.number + 1,
+      state,
+      handler: handler.name
+    })
+    this.addStep(nextStep)
+    this.currentStep = (this.currentStep || 0) + 1
+    this.system.logger.info(`Proceeding ${this} to new step ${this.currentStep}.`)
+    this.save()
+    await nextStep.save()
+    await this.system.checkFinishAndFindDirections(handler, nextStep)
+  }
+
+  /**
+   * Get a reference to the database.
+   */
+  get db(): KnexDatabase {
+    return this.system.db
+  }
+
+  /**
+   * Save the process info to the database.
+   */
+  async save(): Promise<ID> {
+    if (this.id) {
+      await this.db('processes').update(this.toJSON()).where({ id: this.id })
+      return this.id
+    } else {
+      this.id = (await this.db('processes').insert(this.toJSON()).returning('id'))[0].id
+      if (this.id) return this.id
+      throw new DatabaseError(`Saving process ${JSON.stringify(this.toJSON)} failed.`)
+    }
+  }
+
+  /**
+   * Load the process data and its files. Note that current step is not yet loaded here, but when using getCurrentStep().
+   * @param id
+   */
+  async load(id: ID): Promise<void> {
+    // Load basic info.
+    const data = await this.db('processes').select('*').where({ id }).first()
+    if (!data) {
+      throw new InvalidArgument(`Cannot find process #${id}`)
+    }
+    Object.assign(this, data)
+    this.id = id
+    // Load files.
+    this.files = (await this.db('process_files').select('*').where({ processId: this.id })).map(fileData => {
+      const file = new ProcessFile(fileData)
+      file.id = fileData.id
+      return file
+    })
+    // Load current step.
+    await this.getCurrentStep()
+  }
+
+  /**
+   * Load the step with the given number from the database.
+   * @param number
+   * @returns
+   */
+  async loadStep(number: number): Promise<ProcessStep> {
+    if (!this.id) {
+      throw new BadState(`Cannot load steps, if the process have no ID ${JSON.stringify(this.toJSON())}.`)
+    }
+    if (this.currentStep === undefined) {
+      throw new BadState(`Cannot load any steps, since process have no current step ${JSON.stringify(this.toJSON())}.`)
+    }
+    const data = await this.db('process_steps').where({ processId: this.id, number }).first()
+    if (!data) {
+      throw new BadState(`Cannot find step ${this.currentStep} for process ${JSON.stringify(this.toJSON())}.`)
+    }
+    this.steps[this.currentStep] = new ProcessStep(data)
+    this.steps[this.currentStep].id = data.id
+    this.steps[this.currentStep].process = this
+    return this.steps[this.currentStep]
+  }
+
+  /**
+   * Check if the process can be run.
+   */
+  canRun(): boolean {
+    return !this.complete && (this.status === 'INCOMPLETE' || this.status === 'WAITING')
+  }
+
+  /**
+   * Execute process as long as it is completed, failed or requires additional input.
+   */
+  async run(): Promise<void> {
+    let step
+    let MAX_RUNS = 100
+    while (true) {
+      MAX_RUNS--
+      if (MAX_RUNS < 0) {
+        this.system.logger.error(`Maximum number of executions reached for the process ${this}.`)
+        break
+      }
+      step = await this.getCurrentStep()
+      if (!step.directions) {
+        this.system.logger.info(`No new directions for the process ${this}.`)
+        break
+      }
+      if (!step.directions.isImmediate()) {
+        this.system.logger.info(`Waiting for more input for the process ${this}.`)
+        await this.updateStatus()
+        break
+      }
+      const handler = this.system.getHandler(step.handler)
+      const state = clone(step.state)
+      const action = clone(step.directions.action)
+      try {
+        if (action) {
+          const nextState = await handler.action(this, action, state, this.files)
+          await this.proceedToState(action, nextState)
+        } else {
+          throw new BadState(`Process step ${step} has no action.`)
+        }
+      } catch (err) {
+        return await this.crashed(err)
+      }
+    }
+  }
+
+  /**
+   * Record the error and mark the process as finished with an error.
+   * @param err
+   */
+  async crashed(err: Error): Promise<void> {
+    if (isAskUI(err)) {
+      // Postpone the action we tried. Instead, create query for UI to add more configuration for later retry.
+      const directions = new Directions({
+        type: 'ui',
+        element: err.element as unknown as TasenorElement
+      })
+      const step = await this.getCurrentStep()
+      step.directions = directions
+      await step.save()
+      await this.updateStatus()
+      return
+    }
+
+    this.system.logger.error(`Processing of ${this} failed:`, err)
+    if (this.currentStep !== undefined && this.currentStep !== null) {
+      const step = await this.loadStep(this.currentStep)
+      step.finished = new Date()
+      await step.save()
+    }
+    this.error = err.stack ? err.stack : `${err.name}: ${err.message}`
+    await this.save()
+    await this.updateStatus()
+  }
+
+  /**
+   * Resolve the status of the process and update it to the database.
+   */
+  async updateStatus(): Promise<void> {
+    let status = 'INCOMPLETE'
+    if (this.error) {
+      status = 'CRASHED'
+    } else {
+      if (this.currentStep === null || this.currentStep === undefined) {
+        throw new BadState(`Cannot check status when there is no current step loaded for ${this}`)
+      }
+      const step = this.steps[this.currentStep]
+      if (step.finished) {
+        if (this.successful === true) status = 'SUCCEEDED'
+        if (this.successful === false) status = 'FAILED'
+      }
+      if (step.directions) {
+        status = step.directions.isImmediate() ? 'INCOMPLETE' : 'WAITING'
+      }
+    }
+    if (this.status !== status) {
+      this.system.logger.info(`Process ${this} is now ${status}`)
+    }
+    this.status = status as ProcessStatus
+    await this.db('processes').update({ status }).where({ id: this.id })
+
+    let directions, state
+    switch (status) {
+      case 'SUCCEEDED':
+        await this.system.connector.success(this.state)
+        break
+      case 'CRASHED':
+        await this.system.connector.fail(this.error)
+        break
+      case 'FAILED':
+        await this.system.connector.fail(this.state)
+        break
+      default:
+        directions = this.currentStep ? this.steps[this.currentStep].directions : null
+        state = this.currentStep ? this.steps[this.currentStep].state : null
+        await this.system.connector.waiting(state, directions)
+    }
+  }
+
+  /**
+   * Get the state of the current step of the process.
+   */
+  get state(): ImportState {
+    if (this.currentStep === null || this.currentStep === undefined) {
+      throw new BadState(`Cannot check state when there is no current step loaded for ${this}`)
+    }
+    const step = this.steps[this.currentStep]
+    return step.state
+  }
+
+  /**
+   * Handle external input coming in.
+   * @param action
+   */
+  async input(action: ImportAction): Promise<void> {
+    this.system.logger.info(`Handling input ${JSON.stringify(action)} on process ${this}.`)
+    const step = await this.getCurrentStep()
+    const handler = this.system.getHandler(step.handler)
+    let nextState
+    try {
+      nextState = await handler.action(this, action, clone(step.state), this.files)
+    } catch (err) {
+      return this.crashed(err)
+    }
+    await this.proceedToState(action, nextState)
+  }
+
+  /**
+   * Roll back the process.
+   */
+  async rollback(): Promise<boolean> {
+    if (this.currentStep === null || this.currentStep === undefined) {
+      throw new BadState('Cannot rollback when there is no current step.')
+    }
+    if (this.currentStep < 1) {
+      throw new BadState('Cannot rollback when there is only initial step in the process.')
+    }
+    const step = await this.getCurrentStep()
+    this.system.logger.info(`Attempt of rolling back '${step}' from '${this}'.`)
+    const handler = this.system.getHandler(step.handler)
+    await handler.rollback(this, this.state)
+    const current = await this.getCurrentStep()
+    current.action = { rollback: true }
+    current.finished = new Date()
+    current.save()
+
+    this.system.logger.info(`Proceeding ${this} to new step ${this.currentStep}.`)
+    this.save()
+
+    this.status = 'ROLLEDBACK'
+    await this.db('processes').update({ status: this.status }).where({ id: this.id })
+    this.system.logger.info(`Roll back of '${this}' successful.`)
+
+    return true
+  }
+}

package/src/process/ProcessConnector.ts

@@ -0,0 +1,45 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+import { ID } from '@dataplug/tasenor-common'
+
+/**
+ * A connector interface for querying information, applying results and running various hooks.
+ */
+export interface ProcessConnector {
+  initialize(server: unknown): Promise<void>
+  getTranslation(text: string, language: string): Promise<string>
+  resultExists(processId: ID, args: unknown): Promise<boolean>
+  applyResult(processId: ID, args: unknown): Promise<Record<string, unknown>>
+  rollback(processId: ID): Promise<boolean>
+  success(state: unknown): Promise<void>
+  waiting(state: unknown, directions): Promise<void>
+  fail(state: unknown): Promise<void>
+}
+
+export const defaultConnector = {
+  async initialize(): Promise<void> {
+    console.log(new Date(), 'Connector initialized.')
+  },
+  async resultExists(processId: ID, args: unknown): Promise<boolean> {
+    return false
+  },
+  async applyResult(): Promise<Record<string, unknown>> {
+    console.log(new Date(), 'Result received.')
+    return {}
+  },
+  async success(): Promise<void> {
+    console.log(new Date(), 'Process completed.')
+  },
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  async waiting(): Promise<void> {
+  },
+  async fail(): Promise<void> {
+    console.error(new Date(), 'Process failed.')
+  },
+  async getTranslation(text: string) {
+    return text
+  },
+  async rollback(processId: ID): Promise<boolean> {
+    return true
+  }
+
+}

package/src/process/ProcessFile.ts

@@ -0,0 +1,169 @@
+import { ID, FileEncoding } from '@dataplug/tasenor-common'
+import chardet from 'chardet'
+import clone from 'clone'
+import { KnexDatabase } from '../database'
+import { DatabaseError, InvalidFile, NotImplemented } from '../error'
+
+/**
+ * A data structure containing input data for the process.
+ */
+export interface ProcessFileData {
+  processId?: ID
+  name: string
+  type?: string
+  encoding: FileEncoding
+  data: string
+}
+
+/**
+ * An instance of input data for processing.
+ */
+export class ProcessFile {
+  id: ID
+  processId: ID
+  name: string
+  type?: string
+  encoding: FileEncoding
+  data: string
+  _decoded?: string
+
+  constructor(obj: ProcessFileData) {
+    this.id = null
+    this.processId = obj.processId || null
+    this.name = obj.name
+    this.type = obj.type
+    this.encoding = obj.encoding
+    this.data = obj.data
+    this._decoded = undefined
+  }
+
+  toString(): string {
+    return `ProcessFile #${this.id} ${this.name}`
+  }
+
+  /**
+   * Get the loaded process information as JSON object.
+   * @returns
+   */
+  toJSON(): ProcessFileData {
+    return {
+      processId: this.processId,
+      name: this.name,
+      type: this.type,
+      encoding: this.encoding,
+      data: this.data
+    }
+  }
+
+  /**
+   * Save the file to the database.
+   */
+  async save(db: KnexDatabase): Promise<ID> {
+    const out = this.toJSON()
+    if (this.encoding === 'json') {
+      out.data = JSON.stringify(out.data)
+    }
+    if (this.id) {
+      await db('process_files').update(out).where({ id: this.id })
+      return this.id
+    } else {
+      this.id = (await db('process_files').insert(out).returning('id'))[0].id
+      if (this.id) return this.id
+      throw new DatabaseError(`Saving process ${JSON.stringify(out)} failed.`)
+    }
+  }
+
+  /**
+   * Check if the first line of the text file matches to the regular expression.
+   * @param re
+   */
+  firstLineMatch(re: RegExp): boolean {
+    const str = this.decode()
+    const n = str.indexOf('\n')
+    const line1 = n < 0 ? str : str.substr(0, n).trim()
+    return re.test(line1)
+  }
+
+  /**
+   * Check if the second line of the text file matches to the regular expression.
+   * @param re
+   */
+  secondLineMatch(re: RegExp): boolean {
+    const lines = this.decode().split('\n')
+    return lines.length > 1 && re.test(lines[1].trim())
+  }
+
+  /**
+   * Check if the third line of the text file matches to the regular expression.
+   * @param re
+   */
+  thirdLineMatch(re: RegExp): boolean {
+    const lines = this.decode().split('\n')
+    return lines.length > 2 && re.test(lines[2].trim())
+  }
+
+  /**
+   * Check if the file begins with the given string.
+   */
+  startsWith(s: string): boolean {
+    let buffer
+    switch (this.encoding) {
+      case 'base64':
+        buffer = Buffer.from(this.data.substring(0, s.length * 2), 'base64')
+        return buffer.toString('ascii').substr(0, s.length) === s
+      default:
+        throw new NotImplemented(`Cannot handle encoding ${this.encoding} in startWith().`)
+    }
+  }
+
+  /**
+   * Find out if the content is binary or text.
+   *
+   * The mime type has to start with `text/`.
+   */
+  isTextFile(): boolean {
+    return this.type?.startsWith('text/') || false
+  }
+
+  /**
+   * Convert chardet encoding to the supported buffer encoding
+   * "ascii" | "utf8" | "utf-8" | "utf16le" | "ucs2" | "ucs-2" | "base64" | "latin1" | "binary" | "hex"
+   */
+  parseEncoding(encoding: string): 'ascii' | 'utf8' | 'utf-8' | 'utf16le' | 'ucs2' | 'ucs-2' | 'base64' | 'latin1' | 'binary' | 'hex' {
+    switch (encoding.toUpperCase()) {
+      case 'UTF-8':
+        return 'utf-8'
+      case 'ISO-8859-1':
+        return 'latin1'
+      case 'UTF-16LE':
+        return 'utf16le'
+      default:
+        throw new InvalidFile(`Not able to map text encoding ${encoding}.`)
+    }
+  }
+
+  /**
+   * Try to recognize the file content and decode if it is a recognizable text format.
+   */
+  decode(): string {
+    if (this._decoded) {
+      return this._decoded
+    }
+    if (this.encoding === 'base64') {
+      const buffer = Buffer.from(this.data, 'base64')
+      const encoding = chardet.detect(buffer)
+      if (!encoding) {
+        throw new InvalidFile(`Cannot determine encoding for '${this}'.`)
+      }
+      this._decoded = buffer.toString(this.parseEncoding(encoding))
+      return this._decoded
+    }
+
+    if (this.encoding === 'utf-8') {
+      this._decoded = clone(this.data)
+      return this._decoded || ''
+    }
+
+    throw new InvalidFile(`An encoding '${this.encoding}' is not yet supported.`)
+  }
+}

package/src/process/ProcessHandler.ts

@@ -0,0 +1,94 @@
+import { ProcessFile } from './ProcessFile'
+import { ProcessingSystem } from './ProcessingSystem'
+import { Process } from './Process'
+import { NotImplemented } from '../error'
+import { Directions, ImportAction, ImportState, ProcessConfig } from '@dataplug/tasenor-common'
+
+/**
+ * A handler taking care of moving between process states.
+ */
+export class ProcessHandler {
+
+  system: ProcessingSystem
+  name: string
+
+  constructor(name: string) {
+    this.name = name
+  }
+
+  /**
+   * Attach this handler to the processing system during the registration.
+   * @param system
+   */
+  connect(system: ProcessingSystem): void {
+    this.system = system
+  }
+
+  /**
+   * Check if we are able to handle the given file.
+   * @param file
+   */
+  canHandle(file: ProcessFile): boolean {
+    throw new NotImplemented(`A handler '${this.name}' cannot check file '${file.name}', since canHandle() is not implemented.`)
+  }
+
+  /**
+   * Check if we are able to append the given file to the process.
+   * @param file
+   */
+  canAppend(file: ProcessFile): boolean {
+    throw new NotImplemented(`A handler '${this.name}' cannot append file '${file.name}', since canAppend() is not implemented.`)
+  }
+
+  /**
+   * Check if the state is either successful `true` or failed `false` or not yet complete `undefined`.
+   * @param state
+   */
+  checkCompletion(state: ImportState): boolean | undefined {
+    throw new NotImplemented(`A handler '${this.name}' cannot check state '${JSON.stringify(state)}', since checkCompletion() is not implemented.`)
+  }
+
+  /**
+   * Execute an action to the state in order to produce new state. Note that state is cloned and can be modified to be new state.
+   * @param action
+   * @param state
+   * @param files
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async action(process: Process, action: ImportAction, state: ImportState, files: ProcessFile[]): Promise<ImportState> {
+    throw new NotImplemented(`A handler '${this.name}' for files ${files.map(f => `'${f}''`).join(', ')} does not implement action()`)
+  }
+
+  /**
+   * Construct intial state from the given data.
+   * @param file
+   */
+  startingState(files: ProcessFile[]): ImportState {
+    throw new NotImplemented(`A handler '${this.name}' for file ${files.map(f => `'${f}''`).join(', ')} does not implement startingState()`)
+  }
+
+  /**
+   * Figure out possible directions from the given state.
+   * @param state
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async getDirections(state: ImportState, config: ProcessConfig): Promise<Directions> {
+    throw new NotImplemented(`A handler '${this.name}' for state '${JSON.stringify(state)}' does not implement getDirections()`)
+  }
+
+  /**
+   * See if it is possible rollback a process.
+   * @param step
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async rollback(process: Process, state: ImportState): Promise<ImportState> {
+    throw new NotImplemented(`A handler '${this.name}' does not implement rollback()`)
+  }
+}
+
+/**
+ * A collection of process handlers.
+ */
+export type ProcessHandlerMap = {
+  [key: string]: ProcessHandler
+}