@tasenor/common-node 1.9.32 → 1.9.34

package/dist/src/import/TextFileProcessHandler.js

@@ -0,0 +1,354 @@
+import * as csvParse from 'csv-parse';
+import { BadState, NotImplemented } from '../error';
+import { ProcessHandler } from '../process/ProcessHandler';
+import { isImportAction, isImportAnswerAction, isImportConfigureAction, isImportOpAction, Directions, isImportRetryAction } from '@tasenor/common';
+/**
+ * Utility class to provide tools for implementing any text file based process handler.
+ */
+export class TextFileProcessHandler extends ProcessHandler {
+    importOptions = {
+        parser: 'csv',
+        numericFields: [],
+        requiredFields: [],
+        insignificantFields: [],
+        sharedFields: [],
+        textField: null,
+        totalAmountField: null
+    };
+    /**
+     * Split the file to lines and keep line numbers with the lines. Mark state type as initial state.
+     * @param file
+     * @returns
+     */
+    startingState(processFiles) {
+        const files = {};
+        for (const processFile of processFiles) {
+            const original = processFile.decode();
+            let lines;
+            if (this.importOptions.custom) {
+                // Use custom splitter if defined.
+                lines = this.importOptions.custom.splitToLines(original).map((text, idx) => ({
+                    text,
+                    line: idx,
+                    columns: {}
+                }));
+            }
+            else {
+                // Otherwise just split from new lines.
+                lines = original.replace(/\n+$/, '').split('\n').map((text, line) => ({
+                    text,
+                    line,
+                    columns: {}
+                }));
+            }
+            files[processFile.name] = { lines };
+        }
+        return {
+            stage: 'initial',
+            files
+        };
+    }
+    /**
+     * Check the state type is matching to 'complete'.
+     * @param state
+     */
+    checkCompletion(state) {
+        if (state.stage === 'executed') {
+            return true;
+        }
+        return undefined;
+    }
+    /**
+     * A hook to check alternative directions from initial state.
+     * @param state
+     * @returns
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    async needInputForSegmentation(state, config) {
+        return false;
+    }
+    /**
+     * A hook to check alternative directions from segmented state.
+     * @param state
+     * @returns
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    async needInputForClassification(state, config) {
+        return false;
+    }
+    /**
+     * A hook to check alternative directions from classified state.
+     * @param state
+     * @returns
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    async needInputForAnalysis(state, config) {
+        return false;
+    }
+    /**
+     * A hook to check alternative directions from analyzed state.
+     * @param state
+     * @returns
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    async needInputForExecution(state, config) {
+        return false;
+    }
+    /**
+     * Run steps in order 'segmentation', 'classification', 'analysis', 'execution'.
+     * @param state
+     * @returns
+     */
+    async getDirections(state, config) {
+        let input;
+        let directions;
+        switch (state.stage) {
+            case 'initial':
+                input = await this.needInputForSegmentation(state, config);
+                if (input)
+                    return input;
+                directions = new Directions({
+                    type: 'action',
+                    action: { op: 'segmentation' }
+                });
+                break;
+            case 'segmented':
+                input = await this.needInputForClassification(state, config);
+                if (input)
+                    return input;
+                directions = new Directions({
+                    type: 'action',
+                    action: { op: 'classification' }
+                });
+                break;
+            case 'classified':
+                input = await this.needInputForAnalysis(state, config);
+                if (input)
+                    return input;
+                directions = new Directions({
+                    type: 'action',
+                    action: { op: 'analysis' }
+                });
+                break;
+            case 'analyzed':
+                input = await this.needInputForExecution(state, config);
+                if (input)
+                    return input;
+                directions = new Directions({
+                    type: 'action',
+                    action: { op: 'execution' }
+                });
+                break;
+            default:
+                throw new BadState('Cannot find directions from the current state.');
+        }
+        return directions;
+    }
+    /**
+     * Call subclass implementation for each action.
+     * @param action
+     * @param state
+     * @param files
+     */
+    async action(process, action, state, files) {
+        if (!isImportAction(action)) {
+            throw new BadState(`Action is not import action ${JSON.stringify(action)}`);
+        }
+        if (isImportRetryAction(action)) {
+            this.system.logger.info(`Executing 'retry' action on process #${process.id}.`);
+            process.status = 'INCOMPLETE';
+            process.error = undefined;
+            await process.save();
+            return state;
+        }
+        if (isImportOpAction(action)) {
+            this.system.logger.info(`Executing '${action.op}' action on process #${process.id}.`);
+            switch (action.op) {
+                case 'analysis':
+                case 'classification':
+                case 'segmentation':
+                case 'execution':
+                    return this[action.op](process, state, files, process.config);
+                default:
+                    throw new BadState(`Cannot parse action ${JSON.stringify(action)}`);
+            }
+        }
+        if (isImportConfigureAction(action)) {
+            this.system.logger.info(`Updating 'configuration' on process #${process.id}.`);
+            Object.assign(process.config, action.configure);
+            await process.save();
+        }
+        if (isImportAnswerAction(action)) {
+            this.system.logger.info(`Updating 'answers' on process #${process.id}.`);
+            if (!process.config.answers) {
+                process.config.answers = {};
+            }
+            const answers = process.config.answers;
+            for (const segmentId of Object.keys(action.answer)) {
+                answers[segmentId] = answers[segmentId] || {};
+                for (const variable of Object.keys(action.answer[segmentId])) {
+                    answers[segmentId][variable] = action.answer[segmentId][variable];
+                }
+            }
+            await process.save();
+        }
+        return state;
+    }
+    /**
+     * This function must implement gathering of each line together that forms together one import activity.
+     * @param state
+     * @param files
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    async segmentation(process, state, files, config) {
+        throw new NotImplemented(`A class ${this.constructor.name} does not implement segmentation().`);
+    }
+    /**
+     * This function must implement gathering of each line together that forms together one import activity.
+     * @param state
+     * @param files
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    async classification(process, state, files, config) {
+        throw new NotImplemented(`A class ${this.constructor.name} does not implement classification().`);
+    }
+    /**
+     * This function must implement conversion from classified data to the actual executable operations.
+     * @param state
+     * @param files
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    async analysis(process, state, files, config) {
+        throw new NotImplemented(`A class ${this.constructor.name} does not implement analysis().`);
+    }
+    /**
+     * This function must implement applying the result in practice.
+     * @param state
+     * @param files
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    async execution(process, state, files, config) {
+        throw new NotImplemented(`A class ${this.constructor.name} does not implement execution().`);
+    }
+    /**
+     * Parse a single line of CSV.
+     * @param line
+     * @param options
+     * @returns
+     */
+    async parseCsvLine(line, options = {}) {
+        return new Promise((resolve, reject) => {
+            csvParse.parse(line, {
+                delimiter: options.columnSeparator || ',',
+                skip_records_with_error: !!options.skipErrors
+            }, function (err, out) {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve(out[0]);
+                }
+            });
+        });
+    }
+    /**
+     * Go through each file and each line and add CSV interpretation of the content to each line.
+     * @param state
+     * @param options
+     * @returns The original state that has been modified by adding CSV parsed field `columns`.
+     */
+    async parseCSV(state, options = {}) {
+        // Heading names per column.
+        let headings = [];
+        let dropLines = options.cutFromBeginning || 0;
+        // Run loop over all files.
+        let firstLine = true;
+        for (const fileName of Object.keys(state.files)) {
+            // Process each line from CSV.
+            for (let n = 0; n < state.files[fileName].lines.length; n++) {
+                if (dropLines) {
+                    dropLines--;
+                    continue;
+                }
+                const line = { ...state.files[fileName].lines[n] };
+                const text = options.trimLines ? line.text.trim() : line.text;
+                // Collect or define headings on the first line.
+                if (firstLine) {
+                    firstLine = false;
+                    if (options.useFirstLineHeadings) {
+                        headings = await this.parseCsvLine(text, options);
+                        const headCount = {};
+                        for (let i = 0; i < headings.length; i++) {
+                            headCount[headings[i]] = headCount[headings[i]] || 0;
+                            headCount[headings[i]]++;
+                            if (headCount[headings[i]] > 1) {
+                                headings[i] = `${headings[i]}${headCount[headings[i]]}`;
+                            }
+                        }
+                        continue;
+                    }
+                    else {
+                        const size = (await this.parseCsvLine(text, options)).length;
+                        for (let i = 0; i < size; i++) {
+                            headings.push(`${i}`);
+                        }
+                    }
+                }
+                // Map each column to its heading name.
+                const columns = {};
+                const pieces = text.trim() !== '' ? await this.parseCsvLine(text, options) : null;
+                if (pieces) {
+                    pieces.forEach((column, index) => {
+                        if (index < headings.length) {
+                            columns[headings[index]] = column;
+                        }
+                        else {
+                            columns['+'] = columns['+'] || '';
+                            columns['+'] += column + '\n';
+                        }
+                    });
+                    line.columns = columns;
+                    // Add it back with columns.
+                    state.files[fileName].lines[n] = line;
+                }
+            }
+        }
+        const newState = {
+            ...state,
+            stage: 'segmented'
+        };
+        return newState;
+    }
+    /**
+     * Handler for pure custom handler.
+     */
+    async parseCustom(state, options) {
+        for (const fileName of Object.keys(state.files)) {
+            for (const line of state.files[fileName].lines) {
+                line.columns = options.splitToColumns(line.text);
+            }
+        }
+        const newState = {
+            ...state,
+            stage: 'segmented'
+        };
+        return newState;
+    }
+    /**
+     * Split a string to fixed length fields given as name and length mapping.
+     */
+    parseFixedLength(src, offsets, conversions) {
+        const ret = {};
+        const keys = Object.keys(offsets);
+        for (let n = 0, i = 0; i < keys.length; i++) {
+            const column = src.substring(n, n + offsets[keys[i]]);
+            ret[keys[i]] = column.trim();
+            if (conversions[keys[i]]) {
+                ret[keys[i]] = conversions[keys[i]](ret[keys[i]]);
+            }
+            n += offsets[keys[i]];
+        }
+        return ret;
+    }
+}
+//# sourceMappingURL=TextFileProcessHandler.js.map

package/dist/src/import/TextFileProcessHandler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextFileProcessHandler.js","sourceRoot":"","sources":["../../../src/import/TextFileProcessHandler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAEnD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAA;AAC1D,OAAO,EAAgB,cAAc,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,gBAAgB,EAA4B,UAAU,EAAgE,mBAAmB,EAAiD,MAAM,iBAAiB,CAAA;AAGvS;;GAEG;AACH,MAAM,OAAO,sBAAuB,SAAQ,cAAc;IAEjD,aAAa,GAA6B;QAC/C,MAAM,EAAE,KAAK;QACb,aAAa,EAAE,EAAE;QACjB,cAAc,EAAE,EAAE;QAClB,mBAAmB,EAAE,EAAE;QACvB,YAAY,EAAE,EAAE;QAChB,SAAS,EAAE,IAAI;QACf,gBAAgB,EAAE,IAAI;KACvB,CAAA;IAED;;;;OAIG;IACH,aAAa,CAAC,YAA2B;QACvC,MAAM,KAAK,GAA8C,EAAE,CAAA;QAE3D,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;YACtC,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,EAAE,CAAA;YAErC,IAAI,KAAqB,CAAA;YAEzB,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;gBAC7B,kCAAkC;gBAClC,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;oBAC3E,IAAI;oBACJ,IAAI,EAAE,GAAG;oBACT,OAAO,EAAE,EAAE;iBACZ,CAAC,CAAC,CAAA;aACJ;iBAAM;gBACL,uCAAuC;gBACvC,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;oBACpE,IAAI;oBACJ,IAAI;oBACJ,OAAO,EAAE,EAAE;iBACZ,CAAC,CAAC,CAAA;aACJ;YAED,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAA;SACpC;QAED,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,KAAK;SACN,CAAA;IACH,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,KAAkB;QAChC,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE;YAC9B,OAAO,IAAI,CAAA;SACZ;QACD,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;OAIG;IACH,6DAA6D;IAC7D,KAAK,CAAC,wBAAwB,CAAC,KAAkB,EAAE,MAAqB;QACtE,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;;OAIG;IACH,6DAA6D;IAC7D,KAAK,CAAC,0BAA0B,CAAC,KAAkB,EAAE,MAAqB;QACxE,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;;OAIG;IACH,6DAA6D;IAC7D,KAAK,CAAC,oBAAoB,CAAC,KAAkB,EAAE,MAAqB;QAClE,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;;OAIG;IACH,6DAA6D;IAC7D,KAAK,CAAC,qBAAqB,CAAC,KAAkB,EAAE,MAAqB;QACnE,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAAC,KAAkB,EAAE,MAAqB;QAC3D,IAAI,KAAyB,CAAA;QAC7B,IAAI,UAAsB,CAAA;QAC1B,QAAQ,KAAK,CAAC,KAAK,EAAE;YACnB,KAAK,SAAS;gBACZ,KAAK,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;gBAC1D,IAAI,KAAK;oBAAE,OAAO,KAAK,CAAA;gBACvB,UAAU,GAAG,IAAI,UAAU,CAAC;oBAC1B,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,EAAE,EAAE,EAAE,cAAc,EAAE;iBAC/B,CAAC,CAAA;gBACF,MAAK;YACP,KAAK,WAAW;gBACd,KAAK,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;gBAC5D,IAAI,KAAK;oBAAE,OAAO,KAAK,CAAA;gBACvB,UAAU,GAAG,IAAI,UAAU,CAAC;oBAC1B,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,EAAE,EAAE,EAAE,gBAAgB,EAAE;iBACjC,CAAC,CAAA;gBACF,MAAK;YACP,KAAK,YAAY;gBACf,KAAK,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;gBACtD,IAAI,KAAK;oBAAE,OAAO,KAAK,CAAA;gBACvB,UAAU,GAAG,IAAI,UAAU,CAAC;oBAC1B,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE;iBAC3B,CAAC,CAAA;gBACF,MAAK;YACP,KAAK,UAAU;gBACb,KAAK,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;gBACvD,IAAI,KAAK;oBAAE,OAAO,KAAK,CAAA;gBACvB,UAAU,GAAG,IAAI,UAAU,CAAC;oBAC1B,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE;iBAC5B,CAAC,CAAA;gBACF,MAAK;YACP;gBACE,MAAM,IAAI,QAAQ,CAAC,gDAAgD,CAAC,CAAA;SACvE;QACD,OAAO,UAAmC,CAAA;IAC5C,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,CAAC,OAAgB,EAAE,MAAoB,EAAE,KAAkB,EAAE,KAAoB;QAE3F,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE;YAC3B,MAAM,IAAI,QAAQ,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;SAC5E;QAED,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE;YAC/B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,OAAO,CAAC,EAAE,GAAG,CAAC,CAAA;YAC9E,OAAO,CAAC,MAAM,GAAG,YAAY,CAAA;YAC7B,OAAO,CAAC,KAAK,GAAG,SAAS,CAAA;YACzB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAA;YACpB,OAAO,KAAK,CAAA;SACb;QAED,IAAI,gBAAgB,CAAC,MAAM,CAAC,EAAE;YAC5B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,EAAE,wBAAwB,OAAO,CAAC,EAAE,GAAG,CAAC,CAAA;YACrF,QAAQ,MAAM,CAAC,EAAE,EAAE;gBACjB,KAAK,UAAU,CAAC;gBAChB,KAAK,gBAAgB,CAAC;gBACtB,KAAK,cAAc,CAAC;gBACpB,KAAK,WAAW;oBACd,OAAO,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;gBAC/D;oBACE,MAAM,IAAI,QAAQ,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;aACtE;SACF;QAED,IAAI,uBAAuB,CAAC,MAAM,CAAC,EAAE;YACnC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,OAAO,CAAC,EAAE,GAAG,CAAC,CAAA;YAC9E,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;YAC/C,MAAM,OAAO,CAAC,IAAI,EAAE,CAAA;SACrB;QAED,IAAI,oBAAoB,CAAC,MAAM,CAAC,EAAE;YAChC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,OAAO,CAAC,EAAE,GAAG,CAAC,CAAA;YACxE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE;gBAC3B,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAA;aAC5B;YACD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAqD,CAAA;YACpF,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBAClD,OAAO,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;gBAC7C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE;oBAC5D,OAAO,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAA;iBAClE;aACF;YACD,MAAM,OAAO,CAAC,IAAI,EAAE,CAAA;SACrB;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;;OAIG;IACH,6DAA6D;IAC7D,KAAK,CAAC,YAAY,CAAC,OAAgB,EAAE,KAAkB,EAAE,KAAoB,EAAE,MAAqB;QAClG,MAAM,IAAI,cAAc,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,IAAI,qCAAqC,CAAC,CAAA;IACjG,CAAC;IAED;;;;OAIG;IACH,6DAA6D;IAC7D,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,KAAkB,EAAE,KAAoB,EAAE,MAAqB;QACpG,MAAM,IAAI,cAAc,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,IAAI,uCAAuC,CAAC,CAAA;IACnG,CAAC;IAED;;;;OAIG;IACH,6DAA6D;IAC7D,KAAK,CAAC,QAAQ,CAAC,OAAgB,EAAE,KAAkB,EAAE,KAAoB,EAAE,MAAqB;QAC9F,MAAM,IAAI,cAAc,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,IAAI,iCAAiC,CAAC,CAAA;IAC7F,CAAC;IAED;;;;OAIG;IACH,6DAA6D;IAC7D,KAAK,CAAC,SAAS,CAAC,OAAgB,EAAE,KAAkB,EAAE,KAAoB,EAAE,MAAqB;QAC/F,MAAM,IAAI,cAAc,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,IAAI,kCAAkC,CAAC,CAAA;IAC9F,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,IAAY,EAAE,UAA4B,EAAE;QAC7D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE;gBACnB,SAAS,EAAE,OAAO,CAAC,eAAe,IAAI,GAAG;gBACzC,uBAAuB,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU;aAC9C,EAAE,UAAU,GAAG,EAAE,GAAG;gBACnB,IAAI,GAAG,EAAE;oBACP,MAAM,CAAC,GAAG,CAAC,CAAA;iBACZ;qBAAM;oBACL,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;iBAChB;YACH,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAiC,EAAE,UAA4B,EAAE;QAE9E,4BAA4B;QAC5B,IAAI,QAAQ,GAAa,EAAE,CAAA;QAC3B,IAAI,SAAS,GAAG,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAA;QAC7C,2BAA2B;QAC3B,IAAI,SAAS,GAAG,IAAI,CAAA;QACpB,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YAE/C,8BAA8B;YAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC3D,IAAI,SAAS,EAAE;oBACb,SAAS,EAAE,CAAA;oBACX,SAAQ;iBACT;gBACD,MAAM,IAAI,GAAiB,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAA;gBAChE,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAA;gBAE7D,gDAAgD;gBAChD,IAAI,SAAS,EAAE;oBACb,SAAS,GAAG,KAAK,CAAA;oBACjB,IAAI,OAAO,CAAC,oBAAoB,EAAE;wBAChC,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;wBACjD,MAAM,SAAS,GAAG,EAAE,CAAA;wBACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;4BACxC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;4BACpD,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;4BACxB,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE;gCAC9B,QAAQ,CAAC,CAAC,CAAC,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;6BACxD;yBACF;wBACD,SAAQ;qBACT;yBAAM;wBACL,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAA;wBAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE;4BAC7B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;yBACtB;qBACF;iBACF;gBACD,uCAAuC;gBACvC,MAAM,OAAO,GAA2B,EAAE,CAAA;gBAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;gBACjF,IAAI,MAAM,EAAE;oBACV,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;wBAC/B,IAAI,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE;4BAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAA;yBAClC;6BAAM;4BACL,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;4BACjC,OAAO,CAAC,GAAG,CAAC,IAAI,MAAM,GAAG,IAAI,CAAA;yBAC9B;oBACH,CAAC,CAAC,CAAA;oBACF,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;oBAEtB,4BAA4B;oBAC5B,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAA;iBACtC;aACF;SACF;QAED,MAAM,QAAQ,GAAiC;YAC7C,GAAG,KAAmC;YACtC,KAAK,EAAE,WAAW;SACnB,CAAA;QAED,OAAO,QAAQ,CAAA;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,KAAiC,EAAE,OAA4B;QAC/E,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YAC/C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE;gBAC9C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;aACjD;SACF;QAED,MAAM,QAAQ,GAAiC;YAC7C,GAAG,KAAmC;YACtC,KAAK,EAAE,WAAW;SACnB,CAAA;QAED,OAAO,QAAQ,CAAA;IACjB,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,GAAW,EAAE,OAA+B,EAAE,WAA+C;QAC5G,MAAM,GAAG,GAAG,EAAE,CAAA;QACd,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC3C,MAAM,MAAM,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YACrD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;YAC5B,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;gBACxB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;aAClD;YACD,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;SACtB;QACD,OAAO,GAAG,CAAA;IACZ,CAAC;CACF"}

package/dist/src/import/TransactionImportConnector.d.ts

@@ -0,0 +1,38 @@
+import { Asset, AssetType, AssetExchange, AccountNumber, TradeableAsset, StockValueData, Currency, AssetTransfer, BalanceBookkeeping, ProcessConfig, AccountAddress } from '@tasenor/common';
+import { ProcessConnector } from '../process';
+/**
+ * An interface definition for linking generic import processor to the Bookkeeper.
+ */
+export interface TransactionImportConnector extends ProcessConnector {
+    /**
+     * Set the balances at the given time stamp.
+     * @param time
+     * @param balances
+     */
+    initializeBalances(time: Date, balances: BalanceBookkeeping, config: ProcessConfig): Promise<void>;
+    /**
+     * Get the list of accounts that are canditates for holding matching transactions of the given account address.
+     * @param asset
+     */
+    getAccountCanditates(addr: AccountAddress, config: ProcessConfig): Promise<AccountNumber[]>;
+    /**
+     * Get the preferred default account for the given account address.
+     * @param addr
+     */
+    getRate(time: Date, type: AssetType, asset: Asset, currency: Currency, exchange: AssetExchange): Promise<number>;
+    /**
+     * Find out how many assets we have at the given time.
+     * @param time
+     * @param account
+     * @param symbol
+     */
+    getStock(time: Date, account: AccountNumber, symbol: TradeableAsset): Promise<StockValueData>;
+    /**
+     * Find the VAT value for the transfer if any.
+     * @param time
+     * @param transfer
+     * @param currency
+     */
+    getVAT(time: Date, transfer: AssetTransfer, currency: Currency): Promise<null | number>;
+}
+export declare function isTransactionImportConnector(o: unknown): o is TransactionImportConnector;

package/dist/src/import/TransactionImportConnector.js

@@ -0,0 +1,27 @@
+export function isTransactionImportConnector(o) {
+    if (typeof o !== 'object' || o === null) {
+        return false;
+    }
+    let f = o.getRate;
+    if (typeof f !== 'function') {
+        return false;
+    }
+    f = o.getStock;
+    if (typeof f !== 'function') {
+        return false;
+    }
+    f = o.initializeBalances;
+    if (typeof f !== 'function') {
+        return false;
+    }
+    f = o.getAccountCanditates;
+    if (typeof f !== 'function') {
+        return false;
+    }
+    f = o.getVAT;
+    if (typeof f !== 'function') {
+        return false;
+    }
+    return true;
+}
+//# sourceMappingURL=TransactionImportConnector.js.map

package/dist/src/import/TransactionImportConnector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TransactionImportConnector.js","sourceRoot":"","sources":["../../../src/import/TransactionImportConnector.ts"],"names":[],"mappings":"AAuCA,MAAM,UAAU,4BAA4B,CAAC,CAAU;IACrD,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,EAAE;QACvC,OAAO,KAAK,CAAA;KACb;IACD,IAAI,CAAC,GAAI,CAA6B,CAAC,OAAO,CAAA;IAC9C,IAAI,OAAO,CAAC,KAAK,UAAU,EAAE;QAC3B,OAAO,KAAK,CAAA;KACb;IACD,CAAC,GAAI,CAA6B,CAAC,QAAQ,CAAA;IAC3C,IAAI,OAAO,CAAC,KAAK,UAAU,EAAE;QAC3B,OAAO,KAAK,CAAA;KACb;IACD,CAAC,GAAI,CAA6B,CAAC,kBAAkB,CAAA;IACrD,IAAI,OAAO,CAAC,KAAK,UAAU,EAAE;QAC3B,OAAO,KAAK,CAAA;KACb;IACD,CAAC,GAAI,CAA6B,CAAC,oBAAoB,CAAA;IACvD,IAAI,OAAO,CAAC,KAAK,UAAU,EAAE;QAC3B,OAAO,KAAK,CAAA;KACb;IACD,CAAC,GAAI,CAA6B,CAAC,MAAM,CAAA;IACzC,IAAI,OAAO,CAAC,KAAK,UAAU,EAAE;QAC3B,OAAO,KAAK,CAAA;KACb;IACD,OAAO,IAAI,CAAA;AACb,CAAC"}

package/dist/src/import/TransactionImportHandler.d.ts

@@ -0,0 +1,174 @@
+import { AccountAddress, Asset, AssetExchange, AssetTransfer, AssetType, Currency, Language, TransactionDescription, AccountNumber, ProcessConfig, ImportStateText, TextFileLine, SegmentId, NO_SEGMENT, ImportSegment, Directions, ImportConfig, BalanceSummaryEntry, DirectoryPath } from '@tasenor/common';
+import { TransactionUI } from './TransactionUI';
+import { TransactionRules } from './TransactionRules';
+import { TextFileProcessHandler } from './TextFileProcessHandler';
+import { Process, ProcessFile } from '../process';
+/**
+ * Core functionality for all transaction import handlers.
+ */
+export declare class TransactionImportHandler extends TextFileProcessHandler {
+    UI: TransactionUI;
+    rules: TransactionRules;
+    private analyzer;
+    constructor(name: string);
+    get path(): DirectoryPath;
+    /**
+     * By default, we don't support multifile.
+     * @param file
+     * @returns
+     */
+    canAppend(file: ProcessFile): boolean;
+    /**
+     * Get a single account balance.
+     * @param addr
+     */
+    getBalance(addr: AccountAddress): number;
+    /**
+     * Get the translation for the text to the currently configured language.
+     * @param text
+     * @returns
+     */
+    getTranslation(text: string, language: Language | undefined): Promise<string>;
+    /**
+     * Get the account having matching asset in their code.
+     * @param asset
+     * @returns
+     */
+    getAccountCanditates(addr: AccountAddress, config: ProcessConfig): Promise<AccountNumber[]>;
+    /**
+     * Construct grouping for the line data with columns defined using sub class that can generate unique ID per transaction.
+     * @param state
+     */
+    groupingById(state: ImportStateText<'segmented'>): Promise<ImportStateText<'segmented'>>;
+    /**
+     * Default parser for file data.
+     */
+    parse(state: ImportStateText<'initial'>, config?: ProcessConfig): Promise<ImportStateText<'segmented'>>;
+    /**
+     * Default segmentation is parsing CSV and then grouping by segment ID constructed for each line.
+     * @param state
+     * @param files
+     * @returns
+     */
+    segmentationCSV(process: Process, state: ImportStateText<'initial'>, files: ProcessFile[]): Promise<ImportStateText<'segmented'>>;
+    /**
+     * Hook to do some post proccessing for segmentation process. Collects standard fields.
+     * @param state
+     * @returns
+     */
+    segmentationPostProcess(state: ImportStateText<'segmented'>): Promise<ImportStateText<'segmented'>>;
+    segmentation(process: Process, state: ImportStateText<'initial'>, files: ProcessFile[]): Promise<ImportStateText<'segmented'>>;
+    /**
+     * Helper to dump segmentation results.
+     */
+    debugSegmentation(state: ImportStateText<'segmented'>): void;
+    /**
+     * Construct a hash for a text line usable as unique segment ID.
+     * @param line
+     */
+    hash(line: TextFileLine, columns?: string[] | undefined): SegmentId;
+    /**
+     * Segmentation by ID can use this function to group lines by their ID. By default the hash is used.
+     * @param line
+     */
+    segmentId(line: TextFileLine, columns?: string[] | undefined): SegmentId | typeof NO_SEGMENT;
+    /**
+     * Find out the timestamp from the line data if any.
+     * @param line
+     */
+    time(line: TextFileLine): Date | undefined;
+    /**
+     * Default classification constructs lines belonging to each segment and asks subclass to classify them.
+     *
+     * @param state
+     * @param files
+     * @returns
+     */
+    classification(process: Process, state: ImportStateText<'segmented'>, files: ProcessFile[]): Promise<ImportStateText<'classified'>>;
+    /**
+     * Helper to dump classification results.
+     */
+    debugClassification(state: ImportStateText<'classified'>): void;
+    /**
+     * By default, use rules to classify.
+     * @param lines
+     */
+    classifyLines(lines: TextFileLine[], config: ProcessConfig, segment: ImportSegment): Promise<TransactionDescription>;
+    /**
+     * Collect lines related to the segment.
+     * @param state
+     * @param segmentId
+     */
+    getLines(state: ImportStateText<'classified'>, segmentId: SegmentId): TextFileLine[] | null;
+    /**
+     * Check if all accounts are configured and if not, construct query UI for it.
+     * @param state
+     * @returns
+     */
+    needInputForAnalysis(state: ImportStateText<'classified'>, config: ProcessConfig): Promise<Directions | false>;
+    /**
+     * Study configured accounts and missing accounts and construct appropriate UI query for accounts.
+     * @param missing
+     * @param config
+     * @returns
+     */
+    directionsForMissingAccounts(missing: Set<AccountAddress>, config: ProcessConfig): Promise<Directions | false>;
+    /**
+     * Insert custom segments based on answer collection, if necessary.
+     */
+    createCustomSegments(state: ImportStateText<'classified'>, config: ImportConfig): Promise<ImportStateText<'classified'>>;
+    /**
+     * Sort the segments by their date.
+     * @param segments
+     * @returns
+     */
+    sortSegments(segments: Record<string, ImportSegment>): ImportSegment[];
+    /**
+     * Convert transfers to the actual transactions with account numbers.
+     * @param state
+     * @param files
+     */
+    analysis(process: Process, state: ImportStateText<'classified'>, files: ProcessFile[], config: ProcessConfig): Promise<ImportStateText<'analyzed'>>;
+    /**
+     * Analyze and construct transaction details from a transaction description.
+     * @param txs
+     */
+    analyze(txs: TransactionDescription, segment: ImportSegment, config: ProcessConfig, state: ImportStateText<'classified'>, debtAccounts: Record<AccountNumber, BalanceSummaryEntry>): Promise<TransactionDescription>;
+    /**
+     * Check if the resulting transactions needs to be recorded to loan account.
+     */
+    checkForLoan(result: TransactionDescription, debtAccounts: Record<AccountNumber, BalanceSummaryEntry>): Promise<TransactionDescription>;
+    /**
+     * Dump analysis results.
+     * @param state
+     */
+    debugAnalysis(state: ImportStateText<'analyzed'>): void;
+    /**
+     * Apply the result using the connector.
+     * @param state
+     * @param files
+     * @returns
+     */
+    execution(process: Process, state: ImportStateText<'analyzed'>, files: ProcessFile[]): Promise<ImportStateText<'executed'>>;
+    /**
+     * Ask VAT from connector.
+     * @param time
+     * @param reason
+     * @param asset
+     * @param currency
+     */
+    getVAT(time: Date, transfer: AssetTransfer, currency: Currency): Promise<null | number>;
+    /**
+     * Find the rate in the default currency for the asset.
+     * If there is information about rates inside the files, this function could be overridden and
+     * used for digging actual values. Those values can be collected during parse() call.
+     * @param time
+     * @param type
+     * @param asset
+     */
+    getRate(time: Date, type: AssetType, asset: Asset, currency: Currency, exchange: AssetExchange): Promise<number>;
+    /**
+     * Remove transactions created.
+     */
+    rollback(process: Process, state: ImportStateText<'executed'>): Promise<ImportStateText<'rolledback'>>;
+}