@tasenor/common-node 1.9.116 → 1.9.118

package/src/commands/report.ts

@@ -2,7 +2,7 @@
 import { sprintf } from 'sprintf-js'
 import { Command } from '.'
 import { ArgumentParser } from 'argparse'
-import { Report, ReportColumnDefinition } from '@tasenor/common'
+import { Report, ReportColumnDefinition, isReportItem } from '@tasenor/common'
 
 class ReportCommand extends Command {
 
@@ -53,7 +53,7 @@ class ReportCommand extends Command {
 
       // Meta data.
       if ('meta' in data) {
-        Object.keys(report.meta as Record<string, unknown>).forEach((meta) => console.log(`${meta}: ${(report.meta as Record<string, unknown>)[meta]}`))
+        Object.keys(report.meta).forEach((meta) => console.log(`${meta}: ${(report.meta)[meta]}`))
         console.log()
       }
 
@@ -71,13 +71,16 @@ class ReportCommand extends Command {
       // Render each report line.
       for (const item of report.data) {
         line = []
-        for (const column of columns) {
-          const text = {
-            id: () => item.id,
-            name: () => item.name,
-            numeric: () => item.amounts && item.amounts[column.name] !== undefined && sprintf('%.2f', (item.amounts[column.name] || 0) / 100)
-          }[column.type]()
-          line.push(text || '')
+        if (isReportItem(item)) {
+          for (const column of columns) {
+            const text = {
+              id: () => item.id,
+              name: () => item.name,
+              currency: () => item.values && item.values[column.name] !== undefined && sprintf('%.2f', (item.values[column.name] as number || 0) / 100),
+              numeric: () => item.values && item.values[column.name] !== undefined && sprintf('%f', (item.values[column.name] || 0))
+            }[column.type]()
+            line.push(text || '')
+          }
         }
         lines.push(line)
       }
@@ -102,6 +105,7 @@ class ReportCommand extends Command {
       }
       return
     }
+
     throw new Error('Default output not implented.')
   }
 

package/src/database/DB.ts

@@ -1,4 +1,4 @@
-import { Crypto, DatabaseName, error, Hostname, ID, isDatabaseName, log, Secret, Url } from '@tasenor/common'
+import { Crypto, DatabaseName, delay, error, Hostname, ID, isDatabaseName, log, Secret, Url } from '@tasenor/common'
 import { randomString, vault } from '..'
 import knex, { Knex } from 'knex'
 import { types } from 'pg'
@@ -245,10 +245,6 @@ const migrate = async (masterDb: KnexDatabase, name: DatabaseName, migrations: s
  * Apply migrations to the master database.
  */
 const migrateMaster = async (migrations: string): Promise<void> => {
-  function delay(time) {
-    return new Promise(resolve => setTimeout(resolve, time))
-  }
-
   log('Migrating master database.')
 
   const conf = await getMasterConfig()

package/src/import/TransactionImportHandler.ts

@@ -188,6 +188,22 @@ export class TransactionImportHandler extends TextFileProcessHandler {
   async segmentationPostProcess(state: ImportStateText<'segmented'>): Promise<ImportStateText<'segmented'>> {
     const shared: Record<SegmentId, Record<string, string>> = {}
 
+    // Remap fields, if defined.
+    if (this.importOptions.fieldRemapping) {
+      Object.keys(this.importOptions.fieldRemapping).forEach(old => {
+        const replace = (this.importOptions.fieldRemapping && this.importOptions.fieldRemapping[old]) || ''
+        for (const fileName of Object.keys(state.files)) {
+          for (let n = 0; n < state.files[fileName].lines.length; n++) {
+            const { columns } = state.files[fileName].lines[n]
+            if (columns[old] !== undefined) {
+              columns[replace] = columns[old]
+              delete columns[old]
+            }
+          }
+        }
+      })
+    }
+
     for (const fileName of Object.keys(state.files)) {
       // Build standard fields.
       const { textField, totalAmountField } = this.importOptions
@@ -765,6 +781,7 @@ export class TransactionImportHandler extends TextFileProcessHandler {
           debug('EXECUTION', res.transactions)
           const hasOld = await this.system.connector.resultExists(process.id, res)
           if (hasOld) {
+            log(`Duplicate transaction found ${JSON.stringify(res.transactions)}.`)
             const allow = await this.UI.getBoolean(process.config, 'allowIdenticalTx', 'Allow creation of identical transactions that has been already created.')
             if (!allow) {
               for (const tx of res.transactions || []) {

package/src/import/TransactionRules.ts

@@ -272,6 +272,7 @@ export class TransactionRules {
    * * `options` - the options of the current rule we are evaluating
    * * `text` - original text of the corresponding line
    * * `lineNumber` - original line number of the corresponding line
+   * * `version` - version number of the source format found
    * If the filter match is found, then questions are provided to UI unless already
    * answered. The reponses to the questions are passed to the any further evaluations.
    */
@@ -326,7 +327,8 @@ export class TransactionRules {
             rule,
             options: rule.options || {},
             text: line.text,
-            lineNumber: line.line
+            lineNumber: line.line,
+            version: this.handler.version
           }
 
           const singleMatch = rule.options && rule.options.singleMatch

package/src/import/TransferAnalyzer.ts

@@ -379,7 +379,7 @@ export class TransferAnalyzer {
       values.giveAsset = myEntry[0].amount < 0 ? myEntry[0].asset : otherEntry[0].asset
     } else if (weHave(['dividend', 'income'], ['currency', 'statement']) || weHave(['tax', 'dividend', 'income'], ['currency', 'statement'])) {
       kind = 'dividend'
-    } else if (weHave(['income'], ['currency', 'statement']) || weHave(['income', 'tax'], ['currency', 'statement'])) {
+    } else if (weHave(['income'], ['currency', 'statement']) || weHave(['income'], ['crypto', 'statement']) || weHave(['income', 'tax'], ['currency', 'statement'])) {
       kind = 'income'
       const statementEntry = shouldHaveOne('income', 'statement')
       values.name = await this.getTranslation(`income-${statementEntry.asset}`)
@@ -1003,7 +1003,7 @@ export class TransferAnalyzer {
           ? transfers.transfers.filter(t => t.reason === 'trade' && t.value && t.value > 0)
           : transfers.transfers.filter(t => t.reason === 'trade' && t.value && t.value < 0))
         if (soldAsset.length !== 1) {
-          throw new BadState(`Did not found unique asset that was sold from ${JSON.stringify(transfers.transfers)}`)
+          throw new BadState(`Did not found unique asset that was traded from ${JSON.stringify(transfers.transfers)}`)
         }
         let reason: AssetTransferReason
         let asset: VATTarget

package/src/plugins/ReportPlugin.ts

@@ -1,9 +1,9 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
 import fs from 'fs'
-import { data2csv } from '..'
+import { KnexDatabase, data2csv } from '..'
 import dayjs from 'dayjs'
 import quarterOfYear from 'dayjs/plugin/quarterOfYear'
-import { ReportOptions, ReportID, ReportFlagName, ReportItem, ReportQueryParams, ReportLine, AccountNumber, ReportColumnDefinition, PeriodModel, ReportFormat, Language } from '@tasenor/common'
+import { ReportOptions, ReportID, ReportFlagName, ReportItem, ReportQueryParams, ReportLine, AccountNumber, ReportColumnDefinition, PeriodModel, ReportFormat, Language, PK, ReportMeta, ReportData, ReportTotals, Report } from '@tasenor/common'
 import { BackendPlugin } from './BackendPlugin'
 
 dayjs.extend(quarterOfYear)
@@ -24,7 +24,7 @@ export class ReportPlugin extends BackendPlugin {
   }
 
   /**
-   * Read in report struture file.
+   * Read in report structure file.
    */
   getReportStructure(id: ReportID, lang: Language) : ReportFormat | undefined {
     const path = this.filePath(`${id}-${lang}.tsv`)
@@ -44,7 +44,7 @@ export class ReportPlugin extends BackendPlugin {
    * Check if the given report is provided by this plugin.
    * @param id
    */
-  hasReport(id) {
+  hasReport(id: ReportID) {
     return this.formats.includes(id)
   }
 
@@ -62,7 +62,7 @@ export class ReportPlugin extends BackendPlugin {
    * Return UI option definitions for the given report.
    * @param id
    */
-  getReportOptions(id): ReportOptions {
+  getReportOptions(id: ReportID): ReportOptions {
     return {}
   }
 
@@ -85,7 +85,7 @@ export class ReportPlugin extends BackendPlugin {
    * @param flags
    * @returns
    */
-  flags2item(flags: ReportFlagName[]) {
+  flags2item(flags: ReportFlagName[]): ReportItem {
     const item: ReportItem = {}
     flags.forEach(flag => {
       if (flag) {
@@ -124,13 +124,13 @@ export class ReportPlugin extends BackendPlugin {
    * @param entries
    * @param options
    */
-  async getColumns(id, entries, options: ReportOptions, settings): Promise<ReportColumnDefinition[]> {
+  async getColumns(id: ReportID, entries: ReportData[], options: ReportOptions, settings: ReportMeta): Promise<ReportColumnDefinition[]> {
     if (!options.periods) {
-      throw new Error(`Need periods to define columns ${JSON.stringify(options)}`)
+      throw new Error(`Need option 'periods' to define columns in ${JSON.stringify(options)}`)
     }
     const columns: ReportColumnDefinition[] = options.periods.map((period) => {
       return {
-        type: 'numeric',
+        type: 'currency',
         name: 'period' + period.id,
         title: this.columnTitle(id, period, options)
       }
@@ -157,20 +157,27 @@ export class ReportPlugin extends BackendPlugin {
    * Force some options, if needed.
    * @returns
    */
-  forceOptions(options) {
+  forceOptions(options: ReportQueryParams): ReportQueryParams {
     return {
       negateAssetAndProfit: false, // A flag to multiply by -1 entries from asset and profit types of accounts.
       addPreviousPeriod: false // A flag to define if the previous period should be displayed for comparison.
     }
   }
 
+  /**
+   * Optional extra SQL limitation for SQL used to gather raw data. See constructSqlQuery().
+   */
+  async extraSQLCondition(): Promise<string | null> {
+    return null
+  }
+
   /**
    * Construct a SQL for the report query.
    * @param db
    * @param options
    * @returns A knex query prepared.
    */
-  async constructSqlQuery(db, options, settings) {
+  async constructSqlQuery(db: KnexDatabase, options: ReportQueryParams, settings: ReportMeta) {
     // Construct value negator.
     let negateSql = '(CASE debit WHEN true THEN 1 ELSE -1 END)'
     if (options.negateAssetAndProfit) {
@@ -180,7 +187,7 @@ export class ReportPlugin extends BackendPlugin {
     // Find periods.
     const periodIds = [options.periodId]
     if (options.addPreviousPeriod) {
-      const recentPeriods = await db.select('*').from('period').where('id', '<=', options.periodId).orderBy('end_date', 'desc').limit(2)
+      const recentPeriods = await db.select('*').from('period').where('id', '<=', options.periodId + '').orderBy('end_date', 'desc').limit(2)
       if (recentPeriods.length > 1) {
         periodIds.push(recentPeriods[1].id)
       }
@@ -196,24 +203,32 @@ export class ReportPlugin extends BackendPlugin {
       'account.type',
       'account.number',
       db.raw(`CAST(ROUND(${negateSql} * entry.amount * 100) AS BIGINT) AS amount`),
-      'entry.description'
+      'entry.description',
+      'entry.data'
     )
       .from('entry')
       .leftJoin('account', 'account.id', 'entry.account_id')
       .leftJoin('document', 'document.id', 'entry.document_id')
-      .whereIn('document.period_id', periodIds)
+      .whereIn('document.period_id', periodIds as PK[])
 
     // Limit by account, if given.
     if (options.accountId) {
       sqlQuery = sqlQuery.andWhere('account.id', '=', options.accountId)
     }
 
+    // Add extras, if any.
+    const extras = await this.extraSQLCondition()
+    if (extras) {
+      sqlQuery = sqlQuery.andWhere(db.raw(extras))
+    }
+
     // Tune ordering.
     sqlQuery = (sqlQuery
       .orderBy('document.date')
       .orderBy('document.number')
       .orderBy('document.id')
-      .orderBy('entry.row_number'))
+      .orderBy('entry.row_number')
+      .orderBy('entry.id'))
 
     return sqlQuery
   }
@@ -239,9 +254,9 @@ export class ReportPlugin extends BackendPlugin {
    * * `needLocalization` if set, value should be localized, i.e. translated via Localization component in ui.
    * * `name` Title of the entry.
    * * `number` Account number if the entry is an account.
-   * * `amounts` An object with entry for each column mapping name of the columnt to the value to display.
+   * * `values` An object with entry for each column mapping name of the columnt to the value to display.
    */
-  async renderReport(db, id, options: ReportQueryParams = {}) {
+  async renderReport(db: KnexDatabase, id: ReportID, options: ReportQueryParams = {}): Promise<Report> {
 
     // Add report forced options.
     Object.assign(options, this.forceOptions(options))
@@ -262,12 +277,12 @@ export class ReportPlugin extends BackendPlugin {
     options.format = this.getReportStructure(id, options.lang || 'en')
 
     // Prepare query.
-    const q = await this.constructSqlQuery(db, options, settings)
-    let entries = await q
+    const q = this.constructSqlQuery(db, options, settings)
+    let entries = await q as ReportData[]
 
     // Process big ints.
     for (const entry of entries) {
-      entry.amount = parseInt(entry.amount)
+      entry.amount = parseInt(entry.amount + '')
     }
 
     // Apply query filtering.
@@ -275,7 +290,7 @@ export class ReportPlugin extends BackendPlugin {
 
     // We have now relevant entries collected. Use plugin features next.
     const columns: ReportColumnDefinition[] = await this.getColumns(id, entries, options as ReportOptions, settings)
-    let data = this.preProcess(id, entries, options, settings, columns)
+    let data = this.preProcess(id, entries, options, settings, columns) as ReportLine[]
     data = this.postProcess(id, data, options, settings, columns)
     const report = {
       format: id,
@@ -302,7 +317,7 @@ export class ReportPlugin extends BackendPlugin {
    * @param options
    * @param settings
    */
-  doFiltering(id, entries, options, settings) {
+  doFiltering(id: ReportID, entries: ReportData[], options: ReportQueryParams, settings: ReportMeta) {
     let filter = (entry) => true
 
     if (options.quarter1) {
@@ -323,7 +338,7 @@ export class ReportPlugin extends BackendPlugin {
    * @param options
    * @param columns
    */
-  preProcess(id, entries, options, settings, columns) {
+  preProcess(id: ReportID, entries: ReportData[], options: ReportQueryParams, settings: ReportMeta, columns: ReportColumnDefinition[]): ReportLine[] {
     throw new Error(`Report plugin ${this.constructor.name} does not implement preProcess().`)
   }
 
@@ -336,12 +351,12 @@ export class ReportPlugin extends BackendPlugin {
    * @param columns Column definitions.
    * @returns
    */
-  postProcess(id, data, options, settings, columns) {
+  postProcess(id: ReportID, data: ReportLine[], options: ReportQueryParams, settings: ReportMeta, columns: ReportColumnDefinition[]): ReportLine[] {
     return data
   }
 
   /**
-   * A helper to combine final report from pre-processed material for reports using text description.
+   * A helper to combine final report from pre-processed material for reports using TSV text description file.
    * @param accountNumbers A set of all account numbers found.
    * @param accountNames A mapping from account numbers to their names.
    * @param columnNames A list of column names.
@@ -349,8 +364,8 @@ export class ReportPlugin extends BackendPlugin {
    * @param totals A mapping from account numbers their total balance.
    * @returns
    */
-  parseAndCombineReport(accountNumbers: AccountNumber[], accountNames, columns, format, totals) {
-    const columnNames = columns.filter((col) => col.type === 'numeric').map((col) => col.name)
+  parseAndCombineReport(accountNumbers: AccountNumber[], accountNames: Record<AccountNumber, string>, columns: ReportColumnDefinition[], format: ReportFormat, totals: ReportTotals): ReportLine[] {
+    const columnNames = columns.filter((col) => col.type === 'currency').map((col) => col.name)
 
     // Parse report and construct format.
     const allAccounts: AccountNumber[] = Array.from(accountNumbers).sort()
@@ -359,9 +374,9 @@ export class ReportPlugin extends BackendPlugin {
       if (/^#/.test(line)) {
         return
       }
-      let [numbers, text, flags] = line.split('\t')
-      numbers = numbers.split(' ')
-      flags = flags ? new Set(flags.trim().split(/\s+/)) : new Set()
+      let [accNumbers, text, accFlags] = line.split('\t')
+      const numbers = accNumbers.split(' ')
+      const flags: Set<ReportFlagName> = accFlags ? new Set(accFlags.trim().split(/\s+/) as ReportFlagName[]) : new Set()
       const tab = text ? text.replace(/^(_*).*/, '$1').length : 0
       text = text ? text.replace(/^_+/, '') : ''
 
@@ -376,10 +391,10 @@ export class ReportPlugin extends BackendPlugin {
       }
 
       // Split the line and reset variables.
-      const amounts: Record<string, number | null> = {}
-      columnNames.forEach((column) => (amounts[column] = null))
+      const values: Record<string, number | null> = {}
+      columnNames.forEach((column) => (values[column] = null))
       let unused = true
-      const item: ReportItem = { tab, ...this.flags2item(flags) }
+      const item: ReportItem = { tab, ...this.flags2item([...flags]) }
 
       // Collect all totals inside any of the account number ranges.
       for (let i = 0; i < numbers.length; i++) {
@@ -391,7 +406,7 @@ export class ReportPlugin extends BackendPlugin {
             if (number >= from && number < to) {
               unused = false
               if (totals[column][number] !== undefined) {
-                amounts[column] += totals[column][number]
+                values[column] = (values[column] || 0) + totals[column][number]
               }
             }
           })
@@ -402,7 +417,7 @@ export class ReportPlugin extends BackendPlugin {
       if (!item.accountDetails) {
         if (item.required || !unused) {
           item.name = text
-          item.amounts = amounts
+          item.values = values
           ret.push(item)
         }
       }
@@ -415,20 +430,20 @@ export class ReportPlugin extends BackendPlugin {
           const to = parts[1]
           allAccounts.forEach((number) => {
             if (number >= from && number < to) {
-              const item = { tab, ...this.flags2item(flags) }
+              const item = { tab, ...this.flags2item([...flags]) }
               item.isAccount = true
               delete item.accountDetails
               item.name = accountNames[number]
               item.number = number
-              item.amounts = {}
+              item.values = {}
               columnNames.forEach((column) => {
-                if (!item.amounts) {
-                  item.amounts = {}
+                if (!item.values) {
+                  item.values = {}
                 }
                 if (totals[column][number] === undefined) {
-                  item.amounts[column] = null
+                  item.values[column] = null
                 } else {
-                  item.amounts[column] = totals[column][number] + 0
+                  item.values[column] = totals[column][number] + 0
                 }
               })
               ret.push(item)
@@ -438,6 +453,7 @@ export class ReportPlugin extends BackendPlugin {
       }
     })
 
-    return ret
+    // TODO: Define stuff so that need no type hint.
+    return ret as ReportLine[]
   }
 }

package/src/process/ProcessFile.ts

@@ -166,4 +166,13 @@ export class ProcessFile {
 
     throw new InvalidFile(`An encoding '${this.encoding}' is not yet supported.`)
   }
+
+  /**
+   * Set the new content for this file (as UTF-8).
+   */
+  set(content: string): void {
+    this.encoding = 'utf-8'
+    this.data = clone(content)
+    this._decoded = undefined
+  }
 }

package/src/process/ProcessHandler.ts

@@ -10,7 +10,10 @@ import { Directions, ImportAction, ImportState, ProcessConfig } from '@tasenor/c
 export class ProcessHandler {
 
   system: ProcessingSystem
+  // Name of the process handler.
   name: string
+  // Source data version (if more than one).
+  version?: number = undefined
 
   constructor(name: string) {
     this.name = name
@@ -25,13 +28,20 @@ export class ProcessHandler {
   }
 
   /**
-   * Check if we are able to handle the given file.
+   * Check if we are able to handle the given file. Can be boolean or version number of data format.
    * @param file
    */
-  canHandle(file: ProcessFile): boolean {
+  canHandle(file: ProcessFile): boolean | number {
     throw new NotImplemented(`A handler '${this.name}' cannot check file '${file.name}', since canHandle() is not implemented.`)
   }
 
+  /**
+   * Initialization hook called after file(s) has been loaded.
+   */
+  async init(files: ProcessFile[]): Promise<ProcessFile[]> {
+    return files
+  }
+
   /**
    * Check if we are able to append the given file to the process.
    * @param file

package/src/process/ProcessingSystem.ts

@@ -92,7 +92,13 @@ export class ProcessingSystem {
     let selectedHandler: ProcessHandler | null = null
     for (const handler of Object.values(this.handlers)) {
       try {
-        if (handler.canHandle(processFile)) {
+        const version = handler.canHandle(processFile)
+        if (version) {
+          if (version === true) {
+            handler.version = 1
+          } else {
+            handler.version = version
+          }
           selectedHandler = handler
           break
         }
@@ -109,7 +115,6 @@ export class ProcessingSystem {
     // Check if the handler accepts the rest of the files.
     for (let i = 1; i < files.length; i++) {
       const processFile = new ProcessFile(files[i])
-      // await processFile.save(this.db)
       if (!selectedHandler.canAppend(processFile)) {
         await process.crashed(new InvalidArgument(`The file ${files[i].name} of type ${files[i].type} cannot be appended to handler.`))
         return process
@@ -117,6 +122,7 @@ export class ProcessingSystem {
       process.addFile(processFile)
       await processFile.save(this.db)
     }
+    process.files = await selectedHandler.init(process.files)
 
     // Create initial step using the handler.
     let state

package/src/reports/conversions.ts

@@ -1,4 +1,4 @@
-import { ReportQueryParams } from '@tasenor/common'
+import { ReportQueryParams, strRound } from '@tasenor/common'
 import json2csv from 'json2csv'
 import { sprintf } from 'sprintf-js'
 
@@ -13,14 +13,23 @@ export function data2csv(report, options: ReportQueryParams) {
 
   const render = {
     id: (column, entry) => entry.id,
-    name: (column, entry) => `${entry.isAccount ? entry.number + ' ' : ''}${entry.name}`,
+    name: (column, entry) => entry.name === undefined ? '' : `${entry.isAccount ? entry.number + ' ' : ''}${entry.name}`,
     text: (column, entry) => entry[column.name],
-    numeric: (column, entry) => (entry.amounts &&
+    // TODO: Here and rendering we could use heuristic string rounding, i.e. get rid of ..3999999 -> ..4
+    //       Need to share function with rendering.
+    numeric: (column, entry) => (entry.values &&
       !entry.hideTotal &&
-      entry.amounts[column.name] !== '' &&
-      !isNaN(entry.amounts[column.name]) &&
-      entry.amounts[column.name] !== undefined)
-      ? (entry.amounts[column.name] === null ? '—' : sprintf('%.2f', entry.amounts[column.name] / 100))
+      entry.values[column.name] !== '' &&
+      !isNaN(entry.values[column.name]) &&
+      entry.values[column.name] !== undefined)
+      ? (entry.values[column.name] === null ? '—' : strRound(entry.values[column.name]))
+      : '',
+    currency: (column, entry) => (entry.values &&
+      !entry.hideTotal &&
+      entry.values[column.name] !== '' &&
+      !isNaN(entry.values[column.name]) &&
+      entry.values[column.name] !== undefined)
+      ? (entry.values[column.name] === null ? '—' : sprintf('%.2f', entry.values[column.name] / 100))
       : ''
   }