@vizabi/reader-ddfcsv 4.1.1 → 5.0.1

package/lib-web/src/ddf-csv.js

@@ -1,47 +1,47 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-var tslib_1 = require("tslib");
-var includes = require("lodash.includes");
-var cloneDeep = require("lodash.clonedeep");
-var isEmpty = require("lodash.isempty");
-var stripBom = require("strip-bom");
-var resource_selection_optimizer_1 = require("./resource-selection-optimizer");
-var ddfcsv_error_1 = require("./ddfcsv-error");
-var ddf_query_validator_1 = require("ddf-query-validator");
-var Papa = require("papaparse");
-var d3_time_format_1 = require("d3-time-format");
-var isValidNumeric = function (val) { return typeof val !== 'number' && !val ? false : true; };
+exports.ddfCsvReader = void 0;
+const includes = require("lodash.includes");
+const cloneDeep = require("lodash.clonedeep");
+const isEmpty = require("lodash.isempty");
+const stripBom = require("strip-bom");
+const resource_selection_optimizer_1 = require("./resource-selection-optimizer");
+const ddfcsv_error_1 = require("./ddfcsv-error");
+const ddf_query_validator_1 = require("ddf-query-validator");
+const Papa = require("papaparse");
+const d3_time_format_1 = require("d3-time-format");
+const isValidNumeric = val => typeof val !== 'number' && !val ? false : true;
 function ddfCsvReader(logger) {
-    var internalConcepts = [
+    const internalConcepts = [
         { concept: 'concept', concept_type: 'string', domain: null },
         { concept: 'concept_type', concept_type: 'string', domain: null }
     ];
-    var operators = new Map([
-        ['$and', function (row, predicates) { return predicates.every(function (p) { return applyFilterRow(row, p); }); }],
-        ['$or', function (row, predicates) { return predicates.some(function (p) { return applyFilterRow(row, p); }); }],
-        ['$not', function (row, predicate) { return !applyFilterRow(row, predicate); }],
-        ['$nor', function (row, predicates) { return !predicates.some(function (p) { return applyFilterRow(row, p); }); }],
-        ['$eq', function (rowValue, filterValue) { return rowValue == filterValue; }],
-        ['$ne', function (rowValue, filterValue) { return rowValue != filterValue; }],
-        ['$gt', function (rowValue, filterValue) { return isValidNumeric(rowValue) && rowValue > filterValue; }],
-        ['$gte', function (rowValue, filterValue) { return isValidNumeric(rowValue) && rowValue >= filterValue; }],
-        ['$lt', function (rowValue, filterValue) { return isValidNumeric(rowValue) && rowValue < filterValue; }],
-        ['$lte', function (rowValue, filterValue) { return isValidNumeric(rowValue) && rowValue <= filterValue; }],
-        ['$in', function (rowValue, filterValue) { return filterValue.has(rowValue); }],
-        ['$nin', function (rowValue, filterValue) { return !filterValue.has(rowValue); }],
+    const operators = new Map([
+        ['$and', (row, predicates) => predicates.every(p => applyFilterRow(row, p))],
+        ['$or', (row, predicates) => predicates.some(p => applyFilterRow(row, p))],
+        ['$not', (row, predicate) => !applyFilterRow(row, predicate)],
+        ['$nor', (row, predicates) => !predicates.some(p => applyFilterRow(row, p))],
+        ['$eq', (rowValue, filterValue) => rowValue == filterValue],
+        ['$ne', (rowValue, filterValue) => rowValue != filterValue],
+        ['$gt', (rowValue, filterValue) => isValidNumeric(rowValue) && rowValue > filterValue],
+        ['$gte', (rowValue, filterValue) => isValidNumeric(rowValue) && rowValue >= filterValue],
+        ['$lt', (rowValue, filterValue) => isValidNumeric(rowValue) && rowValue < filterValue],
+        ['$lte', (rowValue, filterValue) => isValidNumeric(rowValue) && rowValue <= filterValue],
+        ['$in', (rowValue, filterValue) => filterValue.has(rowValue)],
+        ['$nin', (rowValue, filterValue) => !filterValue.has(rowValue)],
     ]);
-    var keyValueLookup = new Map();
-    var resourcesLookup = new Map();
-    var optimalFilesSet = [];
-    var datapackage;
+    const keyValueLookup = new Map();
+    const resourcesLookup = new Map();
+    let optimalFilesSet = [];
+    let datapackage;
     function loadDataPackage(baseOptions) {
-        var datapackagePath = ddf_query_validator_1.getFilePath(baseOptions.basePath);
-        var _a = baseOptions.diagnostic.prepareDiagnosticFor('loadDataPackage'), debug = _a.debug, error = _a.error;
-        return new Promise(function (resolve, reject) {
+        const datapackagePath = (0, ddf_query_validator_1.getFilePath)(baseOptions.basePath);
+        const { debug, error } = baseOptions.diagnostic.prepareDiagnosticFor('loadDataPackage');
+        return new Promise((resolve, reject) => {
             if (datapackage) {
                 return resolve(datapackage);
             }
-            baseOptions.fileReader.readText(datapackagePath, function (err, data) {
+            baseOptions.fileReader.readText(datapackagePath, (err, data) => {
                 if (err) {
                     error('file reading', err);
                     return reject(new ddfcsv_error_1.DdfCsvError(ddfcsv_error_1.FILE_READING_ERROR, err, datapackagePath));
@@ -61,83 +61,58 @@ function ddfCsvReader(logger) {
             });
         });
     }
-    function loadConcepts(queryParam, options) {
-        return tslib_1.__awaiter(this, void 0, void 0, function () {
-            var error, conceptQuery, result, concepts, err_1;
-            return tslib_1.__generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        error = options.diagnostic.prepareDiagnosticFor('loadConcepts').error;
-                        setConceptsLookup(internalConcepts, options);
-                        conceptQuery = {
-                            select: { key: ['concept'], value: ['concept_type', 'domain'] },
-                            from: 'concepts'
-                        };
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 4, , 5]);
-                        return [4, queryData(conceptQuery, options)];
-                    case 2:
-                        concepts = _a.sent();
-                        buildConceptsLookup(concepts, options);
-                        return [4, reparseConcepts(options)];
-                    case 3:
-                        result = _a.sent();
-                        return [3, 5];
-                    case 4:
-                        err_1 = _a.sent();
-                        error('concepts processing', err_1);
-                        throw err_1;
-                    case 5: return [2, result];
-                }
-            });
-        });
+    async function loadConcepts(queryParam, options) {
+        const { error } = options.diagnostic.prepareDiagnosticFor('loadConcepts');
+        setConceptsLookup(internalConcepts, options);
+        const conceptQuery = {
+            select: { key: ['concept'], value: ['concept_type', 'domain'] },
+            from: 'concepts'
+        };
+        let result;
+        try {
+            const concepts = await queryData(conceptQuery, options);
+            buildConceptsLookup(concepts, options);
+            result = await reparseConcepts(options);
+        }
+        catch (err) {
+            error('concepts processing', err);
+            throw err;
+        }
+        return result;
     }
     function buildConceptsLookup(concepts, options) {
-        var entitySetMembershipConcepts = concepts
-            .filter(function (concept) { return concept.concept_type === 'entity_set'; })
-            .map(function (concept) { return ({
+        const entitySetMembershipConcepts = concepts
+            .filter(concept => concept.concept_type === 'entity_set')
+            .map(concept => ({
             concept: 'is--' + concept.concept,
             concept_type: 'boolean',
             domain: null
-        }); });
+        }));
         concepts = concepts
             .concat(entitySetMembershipConcepts)
             .concat(internalConcepts);
         setConceptsLookup(concepts, options);
     }
-    function reparseConcepts(_a) {
-        var conceptsLookup = _a.conceptsLookup;
-        var parsingFunctions = new Map([
-            ['boolean', function (str) { return str === 'true' || str === 'TRUE'; }],
-            ['measure', function (str) { return parseFloat(str); }]
+    function reparseConcepts({ conceptsLookup }) {
+        const parsingFunctions = new Map([
+            ['boolean', (str) => str === 'true' || str === 'TRUE'],
+            ['measure', (str) => parseFloat(str)]
         ]);
-        var resources = getResources(['concept']);
-        var resourceUpdates = tslib_1.__spread(resources).map(function (resource) {
-            return resource.data.then(function (response) {
-                var resourceConcepts = Object.keys(response.data[0]);
-                var parsingConcepts = new Map();
-                resourceConcepts.forEach(function (concept) {
-                    var type = conceptsLookup.get(concept).concept_type;
-                    var fn = parsingFunctions.get(type);
+        const resources = getResources(['concept']);
+        const resourceUpdates = [...resources].map(resource => {
+            return resource.data.then(response => {
+                const resourceConcepts = Object.keys(response.data[0]);
+                const parsingConcepts = new Map();
+                resourceConcepts.forEach(concept => {
+                    const type = conceptsLookup.get(concept).concept_type;
+                    const fn = parsingFunctions.get(type);
                     if (fn) {
                         parsingConcepts.set(concept, fn);
                     }
                 });
-                return response.data.forEach(function (row) {
-                    var e_1, _a;
-                    try {
-                        for (var parsingConcepts_1 = tslib_1.__values(parsingConcepts), parsingConcepts_1_1 = parsingConcepts_1.next(); !parsingConcepts_1_1.done; parsingConcepts_1_1 = parsingConcepts_1.next()) {
-                            var _b = tslib_1.__read(parsingConcepts_1_1.value, 2), concept = _b[0], parseFn = _b[1];
-                            row[concept] = parseFn(row[concept]);
-                        }
-                    }
-                    catch (e_1_1) { e_1 = { error: e_1_1 }; }
-                    finally {
-                        try {
-                            if (parsingConcepts_1_1 && !parsingConcepts_1_1.done && (_a = parsingConcepts_1.return)) _a.call(parsingConcepts_1);
-                        }
-                        finally { if (e_1) throw e_1.error; }
+                return response.data.forEach(row => {
+                    for (const [concept, parseFn] of parsingConcepts) {
+                        row[concept] = parseFn(row[concept]);
                     }
                 });
             });
@@ -146,84 +121,60 @@ function ddfCsvReader(logger) {
     }
     function setConceptsLookup(concepts, options) {
         options.conceptsLookup.clear();
-        concepts.forEach(function (row) { return options.conceptsLookup.set(row.concept, row); });
-    }
-    function query(queryParam, baseOptions) {
-        return tslib_1.__awaiter(this, void 0, void 0, function () {
-            var _a, warning, error, data, _b, appropriatePlugin, files, err_2;
-            return tslib_1.__generator(this, function (_c) {
-                switch (_c.label) {
-                    case 0:
-                        _a = baseOptions.diagnostic.prepareDiagnosticFor('query'), warning = _a.warning, error = _a.error;
-                        _c.label = 1;
-                    case 1:
-                        _c.trys.push([1, 12, , 13]);
-                        return [4, ddf_query_validator_1.validateQueryStructure(queryParam, baseOptions)];
-                    case 2:
-                        _c.sent();
-                        _b = baseOptions;
-                        return [4, loadDataPackage(baseOptions)];
-                    case 3:
-                        _b.datapackage = _c.sent();
-                        return [4, loadConcepts(queryParam, baseOptions)];
-                    case 4:
-                        _c.sent();
-                        return [4, ddf_query_validator_1.validateQueryDefinitions(queryParam, baseOptions)];
-                    case 5:
-                        _c.sent();
-                        if (!ddf_query_validator_1.isSchemaQuery(queryParam)) return [3, 7];
-                        return [4, querySchema(queryParam, baseOptions)];
-                    case 6:
-                        data = _c.sent();
-                        return [3, 11];
-                    case 7:
-                        appropriatePlugin = resource_selection_optimizer_1.getAppropriatePlugin(queryParam, baseOptions);
-                        if (!appropriatePlugin) return [3, 9];
-                        optimalFilesSet = [];
-                        return [4, appropriatePlugin.getRecommendedFilesSet()];
-                    case 8:
-                        files = _c.sent();
-                        optimalFilesSet = files;
-                        queryParam.optimalFilesSet = [].concat(files, queryParam.optimalFilesSet);
-                        warning('get custom optimal files list by a plugin', optimalFilesSet);
-                        _c.label = 9;
-                    case 9: return [4, queryData(queryParam, baseOptions)];
-                    case 10:
-                        data = _c.sent();
-                        _c.label = 11;
-                    case 11: return [3, 13];
-                    case 12:
-                        err_2 = _c.sent();
-                        error('general query error', err_2);
-                        throw err_2;
-                    case 13: return [2, data];
+        concepts.forEach(row => options.conceptsLookup.set(row.concept, row));
+    }
+    async function query(queryParam, baseOptions) {
+        const { warning, error } = baseOptions.diagnostic.prepareDiagnosticFor('query');
+        let data;
+        try {
+            await (0, ddf_query_validator_1.validateQueryStructure)(queryParam, baseOptions);
+            baseOptions.datapackage = await loadDataPackage(baseOptions);
+            await loadConcepts(queryParam, baseOptions);
+            await (0, ddf_query_validator_1.validateQueryDefinitions)(queryParam, baseOptions);
+            if ((0, ddf_query_validator_1.isSchemaQuery)(queryParam)) {
+                data = await querySchema(queryParam, baseOptions);
+            }
+            else {
+                const appropriatePlugin = (0, resource_selection_optimizer_1.getAppropriatePlugin)(queryParam, baseOptions);
+                if (appropriatePlugin) {
+                    optimalFilesSet = [];
+                    const files = await appropriatePlugin.getRecommendedFilesSet();
+                    optimalFilesSet = files;
+                    queryParam.optimalFilesSet = [].concat(files, queryParam.optimalFilesSet);
+                    warning('get custom optimal files list by a plugin', optimalFilesSet);
                 }
-            });
-        });
+                data = await queryData(queryParam, baseOptions);
+            }
+        }
+        catch (err) {
+            error('general query error', err);
+            throw err;
+        }
+        return data;
     }
     function queryData(queryParam, options) {
-        var debug = options.diagnostic.prepareDiagnosticFor('queryData').debug;
-        var _a = queryParam.select, _b = _a.key, key = _b === void 0 ? [] : _b, _c = _a.value, value = _c === void 0 ? [] : _c, _d = queryParam.from, from = _d === void 0 ? '' : _d, _e = queryParam.where, where = _e === void 0 ? {} : _e, _f = queryParam.join, join = _f === void 0 ? {} : _f, _g = queryParam.order_by, order_by = _g === void 0 ? [] : _g, language = queryParam.language;
-        var select = { key: key, value: value };
+        const { debug } = options.diagnostic.prepareDiagnosticFor('queryData');
+        const { select: { key = [], value = [] }, from = '', where = {}, join = {}, order_by = [], language } = queryParam;
+        const select = { key, value };
         debug('start all data loading', queryParam);
-        var projection = new Set(select.key.concat(select.value));
-        var filterFields = getFilterFields(where).filter(function (field) { return from === 'entities' || !projection.has(field); });
-        var resourcesPromise = loadResources(select.key, tslib_1.__spread(select.value, filterFields), language, options, queryParam);
-        var joinsPromise = getJoinFilters(join, queryParam, options);
-        var entitySetFilterPromise = getEntitySetFilter(select.key, queryParam, options);
+        const projection = new Set(select.key.concat(select.value));
+        const filterFields = getFilterFields(where).filter(field => from === 'entities' || !projection.has(field));
+        const resourcesPromise = loadResources(select.key, [...select.value, ...filterFields], language, options, queryParam);
+        const joinsPromise = getJoinFilters(join, queryParam, options);
+        const entitySetFilterPromise = getEntitySetFilter(select.key, queryParam, options);
         return Promise.all([resourcesPromise, entitySetFilterPromise, joinsPromise])
-            .then(function (_a) {
-            var _b = tslib_1.__read(_a, 3), resourceResponses = _b[0], entitySetFilter = _b[1], joinFilters = _b[2];
+            .then(([resourceResponses, entitySetFilter, joinFilters]) => {
             debug('finish all data loading', queryParam);
-            var whereResolved = processWhere(where, joinFilters);
-            var filter = mergeFilters(entitySetFilter, whereResolved);
+            const whereResolved = processWhere(where, joinFilters);
+            const filter = mergeFilters(entitySetFilter, whereResolved);
             debug('dataTables processing', queryParam);
-            var dataTables = resourceResponses
-                .map(function (response) { return processResourceResponse(response, select, filterFields, options); });
+            const dataTables = resourceResponses
+                .map(response => processResourceResponse(response, select, filterFields, options));
             debug('queryResult processing', queryParam);
-            var queryResult = joinData.apply(void 0, tslib_1.__spread([select.key, 'overwrite'], dataTables)).filter(function (row) { return applyFilterRow(row, filter); })
-                .map(function (row) { return fillMissingValues(row, projection); })
-                .map(function (row) { return projectRow(row, projection); });
+            const queryResult = joinData(select.key, 'overwrite', ...dataTables)
+                .filter(row => applyFilterRow(row, filter))
+                .map(row => fillMissingValues(row, projection))
+                .map(row => projectRow(row, projection));
             debug('result ordering', queryParam);
             orderData(queryResult, order_by);
             debug('final result is ready', queryParam);
@@ -231,33 +182,32 @@ function ddfCsvReader(logger) {
         });
     }
     function parseTime(result, options) {
-        var conceptsLookup = options.conceptsLookup;
-        var concepts = Object.keys(result[0] || {});
-        var timeConcepts = concepts.map(function (c) { return conceptsLookup.get(c) || {}; }).filter(function (co) { return co.concept_type == 'time'; });
-        timeConcepts.forEach(function (_a) {
-            var concept = _a.concept;
-            var parse = getTimeParser(concept, options);
-            result.forEach(function (row) {
+        const conceptsLookup = options.conceptsLookup;
+        const concepts = Object.keys(result[0] || {});
+        const timeConcepts = concepts.map(c => conceptsLookup.get(c) || {}).filter(co => co.concept_type == 'time');
+        timeConcepts.forEach(({ concept }) => {
+            const parse = getTimeParser(concept, options);
+            result.forEach(row => {
                 row[concept] = parse(row[concept]);
             });
         });
         return result;
     }
     function getTimeParser(concept, options) {
-        var error = options.diagnostic.prepareDiagnosticFor('queryData').error;
-        var parsers = {
-            year: d3_time_format_1.utcParse('%Y'),
-            month: d3_time_format_1.utcParse('%Y-%m'),
-            day: d3_time_format_1.utcParse('%Y%m%d'),
-            hour: d3_time_format_1.utcParse('%Y%m%dt%H'),
-            minute: d3_time_format_1.utcParse('%Y%m%dt%H%M'),
-            second: d3_time_format_1.utcParse('%Y%m%dt%H%M%S'),
-            week: d3_time_format_1.utcParse('%Yw%V'),
-            quarter: d3_time_format_1.utcParse('%Yq%q')
+        const { error } = options.diagnostic.prepareDiagnosticFor('queryData');
+        const parsers = {
+            year: (0, d3_time_format_1.utcParse)('%Y'),
+            month: (0, d3_time_format_1.utcParse)('%Y-%m'),
+            day: (0, d3_time_format_1.utcParse)('%Y%m%d'),
+            hour: (0, d3_time_format_1.utcParse)('%Y%m%dt%H'),
+            minute: (0, d3_time_format_1.utcParse)('%Y%m%dt%H%M'),
+            second: (0, d3_time_format_1.utcParse)('%Y%m%dt%H%M%S'),
+            week: (0, d3_time_format_1.utcParse)('%Yw%V'),
+            quarter: (0, d3_time_format_1.utcParse)('%Yq%q')
         };
         function tryParse(str) {
-            for (var i in parsers) {
-                var dateObject = parsers[i](str);
+            for (const i in parsers) {
+                const dateObject = parsers[i](str);
                 if (dateObject) {
                     return dateObject;
                 }
@@ -270,29 +220,28 @@ function ddfCsvReader(logger) {
         }
         if (!parsers[concept]) {
             error('No time parser found for time concept: ' + concept);
-            return function (str) { return str; };
+            return str => str;
         }
         return parsers[concept];
     }
-    function orderData(data, orderBy) {
-        if (orderBy === void 0) { orderBy = []; }
+    function orderData(data, orderBy = []) {
         if (orderBy.length === 0) {
             return;
         }
-        var orderNormalized = orderBy.map(function (orderPart) {
+        const orderNormalized = orderBy.map(orderPart => {
             if (typeof orderPart === 'string') {
                 return { concept: orderPart, direction: 1 };
             }
             else {
-                var concept = Object.keys(orderPart)[0];
-                var direction = (orderPart[concept] === 'asc' ? 1 : -1);
-                return { concept: concept, direction: direction };
+                const concept = Object.keys(orderPart)[0];
+                const direction = (orderPart[concept] === 'asc' ? 1 : -1);
+                return { concept, direction };
             }
         });
-        var n = orderNormalized.length;
-        data.sort(function (a, b) {
-            for (var i = 0; i < n; i++) {
-                var order = orderNormalized[i];
+        const n = orderNormalized.length;
+        data.sort((a, b) => {
+            for (let i = 0; i < n; i++) {
+                const order = orderNormalized[i];
                 if (a[order.concept] < b[order.concept]) {
                     return -1 * order.direction;
                 }
@@ -304,11 +253,11 @@ function ddfCsvReader(logger) {
         });
     }
     function processWhere(where, joinFilters) {
-        var result = {};
-        for (var field in where) {
-            var fieldValue = where[field];
+        const result = {};
+        for (const field in where) {
+            const fieldValue = where[field];
             if (includes(['$and', '$or', '$nor'], field)) {
-                result[field] = fieldValue.map(function (subFilter) { return processWhere(subFilter, joinFilters); });
+                result[field] = fieldValue.map(subFilter => processWhere(subFilter, joinFilters));
             }
             else if (field === '$in' || field === '$nin') {
                 result[field] = new Set(fieldValue);
@@ -325,62 +274,44 @@ function ddfCsvReader(logger) {
         }
         return result;
     }
-    function mergeFilters() {
-        var filters = [];
-        for (var _i = 0; _i < arguments.length; _i++) {
-            filters[_i] = arguments[_i];
-        }
-        return filters.reduce(function (a, b) {
+    function mergeFilters(...filters) {
+        return filters.reduce((a, b) => {
             a.$and.push(b);
             return a;
         }, { $and: [] });
     }
     function querySchema(queryParam, baseOptions) {
-        var _a = baseOptions.diagnostic.prepareDiagnosticFor('query'), debug = _a.debug, error = _a.error;
-        var getSchemaFromCollection = function (collectionPar) {
-            debug("get schema for collection " + collectionPar);
-            return baseOptions.datapackage.ddfSchema[collectionPar].map(function (_a) {
-                var primaryKey = _a.primaryKey, value = _a.value;
-                return ({ key: primaryKey, value: value });
-            });
+        const { debug, error } = baseOptions.diagnostic.prepareDiagnosticFor('query');
+        const getSchemaFromCollection = collectionPar => {
+            debug(`get schema for collection ${collectionPar}`);
+            return baseOptions.datapackage.ddfSchema[collectionPar].map(({ primaryKey, value }) => ({ key: primaryKey, value }));
         };
-        var collection = queryParam.from.split('.')[0];
+        const collection = queryParam.from.split('.')[0];
         if (baseOptions.datapackage.ddfSchema[collection]) {
             return getSchemaFromCollection(collection);
         }
         else if (collection === '*') {
             return Object.keys(baseOptions.datapackage.ddfSchema)
                 .map(getSchemaFromCollection)
-                .reduce(function (a, b) { return a.concat(b); });
+                .reduce((a, b) => a.concat(b));
         }
         else {
-            var message = "No valid collection (" + collection + ") for schema query";
+            const message = `No valid collection (${collection}) for schema query`;
             error(message);
             throwError(new ddfcsv_error_1.DdfCsvError(ddfcsv_error_1.DDF_ERROR, message));
         }
     }
     function fillMissingValues(row, projection) {
-        var e_2, _a;
-        try {
-            for (var projection_1 = tslib_1.__values(projection), projection_1_1 = projection_1.next(); !projection_1_1.done; projection_1_1 = projection_1.next()) {
-                var field = projection_1_1.value;
-                if (typeof row[field] === 'undefined') {
-                    row[field] = null;
-                }
-            }
-        }
-        catch (e_2_1) { e_2 = { error: e_2_1 }; }
-        finally {
-            try {
-                if (projection_1_1 && !projection_1_1.done && (_a = projection_1.return)) _a.call(projection_1);
+        for (const field of projection) {
+            if (typeof row[field] === 'undefined') {
+                row[field] = null;
             }
-            finally { if (e_2) throw e_2.error; }
         }
         return row;
     }
     function applyFilterRow(row, filter) {
-        return Object.keys(filter).every(function (filterKey) {
-            var operator = operators.get(filterKey);
+        return Object.keys(filter).every(filterKey => {
+            const operator = operators.get(filterKey);
             if (operator) {
                 return operator(row, filter[filterKey]);
             }
@@ -393,80 +324,65 @@ function ddfCsvReader(logger) {
         });
     }
     function getJoinFilters(join, queryParam, options) {
-        return Promise.all(Object.keys(join).map(function (joinID) { return getJoinFilter(joinID, join[joinID], queryParam, options); }))
-            .then(function (results) { return results.reduce(mergeObjects, {}); });
+        return Promise.all(Object.keys(join).map(joinID => getJoinFilter(joinID, join[joinID], queryParam, options)))
+            .then(results => results.reduce(mergeObjects, {}));
     }
     function mergeObjects(a, b) {
         return Object.assign(a, b);
     }
     function getJoinFilter(joinID, join, queryParam, options) {
-        var _a;
         if (options.conceptsLookup.get(join.key).concept_type === 'time') {
-            return Promise.resolve((_a = {}, _a[joinID] = join.where, _a));
+            return Promise.resolve({ [joinID]: join.where });
         }
         else {
             return query({
                 select: { key: [join.key] },
                 where: join.where,
                 from: options.conceptsLookup.has(join.key) ? 'entities' : 'concepts'
-            }, Object.assign({ joinID: joinID }, cloneDeep(options)))
-                .then(function (result) {
-                var _a, _b;
-                return (_a = {},
-                    _a[joinID] = (_b = {},
-                        _b[join.key] = {
-                            $in: new Set(result.map(function (row) { return row[join.key]; }))
-                        },
-                        _b),
-                    _a);
-            });
+            }, Object.assign({ joinID }, cloneDeep(options)))
+                .then(result => ({
+                [joinID]: {
+                    [join.key]: {
+                        $in: new Set(result.map(row => row[join.key]))
+                    }
+                }
+            }));
         }
     }
     function getFilterFields(filter) {
-        var fields = [];
-        for (var field in filter) {
+        const fields = [];
+        for (const field in filter) {
             if (includes(['$and', '$or', '$not', '$nor'], field)) {
-                filter[field].map(getFilterFields).forEach(function (subFields) { return fields.push.apply(fields, tslib_1.__spread(subFields)); });
+                filter[field].map(getFilterFields).forEach(subFields => fields.push(...subFields));
             }
             else {
                 fields.push(field);
             }
         }
-        return tslib_1.__spread(new Set(fields));
+        return [...new Set(fields)];
     }
     function filterConceptsByType(conceptTypes, queryKey, options) {
-        var e_3, _a;
-        var conceptStrings = queryKey || Array.from(options.conceptsLookup.keys());
-        var concepts = [];
-        try {
-            for (var conceptStrings_1 = tslib_1.__values(conceptStrings), conceptStrings_1_1 = conceptStrings_1.next(); !conceptStrings_1_1.done; conceptStrings_1_1 = conceptStrings_1.next()) {
-                var conceptString = conceptStrings_1_1.value;
-                var concept = options.conceptsLookup.get(conceptString);
-                if (includes(conceptTypes, concept.concept_type)) {
-                    concepts.push(concept);
-                }
+        const conceptStrings = queryKey || Array.from(options.conceptsLookup.keys());
+        const concepts = [];
+        for (const conceptString of conceptStrings) {
+            const concept = options.conceptsLookup.get(conceptString);
+            if (includes(conceptTypes, concept.concept_type)) {
+                concepts.push(concept);
             }
         }
-        catch (e_3_1) { e_3 = { error: e_3_1 }; }
-        finally {
-            try {
-                if (conceptStrings_1_1 && !conceptStrings_1_1.done && (_a = conceptStrings_1.return)) _a.call(conceptStrings_1);
-            }
-            finally { if (e_3) throw e_3.error; }
-        }
         return concepts;
     }
     function getEntityConceptRenameMap(queryKey, resourceKey, options) {
-        var resourceKeySet = new Set(resourceKey);
-        var entityConceptTypes = ['entity_set', 'entity_domain'];
-        var queryEntityConcepts = filterConceptsByType(entityConceptTypes, queryKey, options);
+        const resourceKeySet = new Set(resourceKey);
+        const entityConceptTypes = ['entity_set', 'entity_domain'];
+        const queryEntityConcepts = filterConceptsByType(entityConceptTypes, queryKey, options);
         if (queryEntityConcepts.length === 0) {
             return new Map();
         }
-        var allEntityConcepts = filterConceptsByType(entityConceptTypes, null, options);
+        const allEntityConcepts = filterConceptsByType(entityConceptTypes, null, options);
         return queryEntityConcepts
-            .map(function (concept) { return allEntityConcepts
-            .filter(function (lookupConcept) {
+            .map(concept => allEntityConcepts
+            .filter(lookupConcept => {
             if (concept.concept_type === 'entity_set') {
                 return resourceKeySet.has(lookupConcept.concept) &&
                     lookupConcept.concept !== concept.concept &&
@@ -479,114 +395,86 @@ function ddfCsvReader(logger) {
                     lookupConcept.domain === concept.concept;
             }
         })
-            .reduce(function (map, aliasConcept) { return map.set(aliasConcept.concept, concept.concept); }, new Map()); }).reduce(function (mapA, mapB) { return new Map(tslib_1.__spread(mapA, mapB)); }, new Map());
+            .reduce((map, aliasConcept) => map.set(aliasConcept.concept, concept.concept), new Map())).reduce((mapA, mapB) => new Map([...mapA, ...mapB]), new Map());
     }
     function getEntitySetFilter(conceptStrings, queryParam, options) {
-        var promises = filterConceptsByType(['entity_set'], conceptStrings, options)
-            .map(function (concept) { return query({
+        const promises = filterConceptsByType(['entity_set'], conceptStrings, options)
+            .map(concept => query({
             select: { key: [concept.domain], value: ['is--' + concept.concept] },
             from: 'entities'
         }, Object.assign({}, cloneDeep(options)))
-            .then(function (result) {
-            var _a;
-            return (_a = {},
-                _a[concept.concept] = {
-                    $in: new Set(result
-                        .filter(function (row) { return row['is--' + concept.concept]; })
-                        .map(function (row) { return row[concept.domain]; }))
-                },
-                _a);
-        }); });
-        return Promise.all(promises).then(function (results) {
-            return results.reduce(function (a, b) { return Object.assign(a, b); }, {});
+            .then(result => ({
+            [concept.concept]: {
+                $in: new Set(result
+                    .filter(row => row['is--' + concept.concept])
+                    .map(row => row[concept.domain]))
+            }
+        })));
+        return Promise.all(promises).then(results => {
+            return results.reduce((a, b) => Object.assign(a, b), {});
         });
     }
     function getResources(key, value) {
         if (!value || value.length === 0 || key[0] === value) {
-            return new Set(tslib_1.__spread(keyValueLookup
-                .get(createKeyString(key))
-                .values()).reduce(function (a, b) { return a.concat(b); }));
+            return new Set([...keyValueLookup
+                    .get(createKeyString(key))
+                    .values()
+            ].reduce((a, b) => a.concat(b)));
         }
         if (Array.isArray(value)) {
             return value
-                .map(function (singleValue) { return getResources(key, singleValue); })
-                .reduce(function (resultSet, resources) { return new Set(tslib_1.__spread(resultSet, resources)); }, new Set());
+                .map(singleValue => getResources(key, singleValue))
+                .reduce((resultSet, resources) => new Set([...resultSet, ...resources]), new Set());
         }
-        var oneKeyOneValueResourcesArray = keyValueLookup
+        let oneKeyOneValueResourcesArray = keyValueLookup
             .get(createKeyString(key))
             .get(value);
         if (oneKeyOneValueResourcesArray) {
             oneKeyOneValueResourcesArray = oneKeyOneValueResourcesArray
-                .filter(function (v) { return isEmpty(optimalFilesSet) || includes(optimalFilesSet, v.path); });
+                .filter(v => isEmpty(optimalFilesSet) || includes(optimalFilesSet, v.path));
         }
         return new Set(oneKeyOneValueResourcesArray);
     }
     function processResourceResponse(response, select, filterFields, options) {
-        var resourcePK = response.resource.schema.primaryKey;
-        var resourceProjection = new Set(tslib_1.__spread(resourcePK, select.value, filterFields));
-        var renameMap = getEntityConceptRenameMap(select.key, resourcePK, options);
+        const resourcePK = response.resource.schema.primaryKey;
+        const resourceProjection = new Set([...resourcePK, ...select.value, ...filterFields]);
+        const renameMap = getEntityConceptRenameMap(select.key, resourcePK, options);
         return response.data
-            .map(function (row) { return projectRow(row, resourceProjection); })
-            .map(function (row) { return renameHeaderRow(row, renameMap); });
+            .map(row => projectRow(row, resourceProjection))
+            .map(row => renameHeaderRow(row, renameMap));
     }
     function loadResources(key, value, language, options, queryParam) {
-        var debug = options.diagnostic.prepareDiagnosticFor('loadResource').debug;
-        var resources = getResources(key, value);
-        debug('resources list by query', { queryParam: queryParam, resources: tslib_1.__spread(resources) });
-        return Promise.all(tslib_1.__spread(resources).map(function (resource) { return loadResource(resource, language, options); }));
+        const { debug } = options.diagnostic.prepareDiagnosticFor('loadResource');
+        const resources = getResources(key, value);
+        debug('resources list by query', { queryParam, resources: [...resources] });
+        return Promise.all([...resources].map(resource => loadResource(resource, language, options)));
     }
     function projectRow(row, projectionSet) {
-        var e_4, _a;
-        var result = {};
-        try {
-            for (var _b = tslib_1.__values(Object.keys(row)), _c = _b.next(); !_c.done; _c = _b.next()) {
-                var concept = _c.value;
-                if (projectionSet.has(concept)) {
-                    result[concept] = row[concept];
-                }
+        const result = {};
+        for (const concept of Object.keys(row)) {
+            if (projectionSet.has(concept)) {
+                result[concept] = row[concept];
             }
         }
-        catch (e_4_1) { e_4 = { error: e_4_1 }; }
-        finally {
-            try {
-                if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
-            }
-            finally { if (e_4) throw e_4.error; }
-        }
         return result;
     }
     function renameHeaderRow(row, renameMap) {
-        var e_5, _a;
-        var result = {};
-        try {
-            for (var _b = tslib_1.__values(Object.keys(row)), _c = _b.next(); !_c.done; _c = _b.next()) {
-                var concept = _c.value;
-                result[renameMap.get(concept) || concept] = row[concept];
-            }
-        }
-        catch (e_5_1) { e_5 = { error: e_5_1 }; }
-        finally {
-            try {
-                if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
-            }
-            finally { if (e_5) throw e_5.error; }
+        const result = {};
+        for (const concept of Object.keys(row)) {
+            result[renameMap.get(concept) || concept] = row[concept];
         }
         return result;
     }
-    function joinData(key, joinMode) {
-        var data = [];
-        for (var _i = 2; _i < arguments.length; _i++) {
-            data[_i - 2] = arguments[_i];
-        }
+    function joinData(key, joinMode, ...data) {
         if (data.length === 1) {
             return data[0];
         }
-        var canonicalKey = key.slice(0).sort();
-        var dataMap = data.reduce(function (result, dataPar) {
-            dataPar.forEach(function (row) {
-                var keyString = canonicalKey.map(function (concept) { return row[concept]; }).join(',');
+        const canonicalKey = key.slice(0).sort();
+        const dataMap = data.reduce((result, dataPar) => {
+            dataPar.forEach(row => {
+                const keyString = canonicalKey.map(concept => row[concept]).join(',');
                 if (result.has(keyString)) {
-                    var resultRow = result.get(keyString);
+                    const resultRow = result.get(keyString);
                     joinRow(resultRow, row, joinMode);
                 }
                 else {
@@ -595,7 +483,7 @@ function ddfCsvReader(logger) {
             });
             return result;
         }, new Map());
-        return tslib_1.__spread(dataMap.values());
+        return [...dataMap.values()];
     }
     function joinRow(resultRow, sourceRow, mode) {
         switch (mode) {
@@ -603,18 +491,18 @@ function ddfCsvReader(logger) {
                 Object.assign(resultRow, sourceRow);
                 break;
             case 'translation':
-                for (var concept in sourceRow) {
+                for (const concept in sourceRow) {
                     if (sourceRow[concept] !== '') {
                         resultRow[concept] = sourceRow[concept];
                     }
                 }
                 break;
             case 'overwriteWithError':
-                for (var concept in sourceRow) {
+                for (const concept in sourceRow) {
                     if (resultRow[concept] !== undefined && resultRow[concept] !== sourceRow[concept]) {
-                        var sourceRowStr = JSON.stringify(sourceRow);
-                        var resultRowStr = JSON.stringify(resultRow);
-                        var errStr = "JOIN Error: two resources have different data for \"" + concept + "\": " + sourceRowStr + "," + resultRowStr;
+                        const sourceRowStr = JSON.stringify(sourceRow);
+                        const resultRowStr = JSON.stringify(resultRow);
+                        const errStr = `JOIN Error: two resources have different data for "${concept}": ${sourceRowStr},${resultRowStr}`;
                         throwError(new ddfcsv_error_1.DdfCsvError(ddfcsv_error_1.DDF_ERROR, errStr));
                     }
                     else {
@@ -625,76 +513,72 @@ function ddfCsvReader(logger) {
         }
     }
     function throwError(error) {
-        var currentLogger = logger || console;
+        const currentLogger = logger || console;
         currentLogger.error(error.message);
         throw error;
     }
-    function createKeyString(key, row) {
-        if (row === void 0) { row = false; }
-        var canonicalKey = key.slice(0).sort();
+    function createKeyString(key, row = false) {
+        const canonicalKey = key.slice(0).sort();
         if (!row) {
             return canonicalKey.join(',');
         }
         else {
-            return canonicalKey.map(function (concept) { return row[concept]; }).join(',');
+            return canonicalKey.map(concept => row[concept]).join(',');
         }
     }
     function loadResource(resource, language, options) {
-        var warning = options.diagnostic.prepareDiagnosticFor('loadResource').warning;
-        var filePromises = [];
+        const { warning } = options.diagnostic.prepareDiagnosticFor('loadResource');
+        const filePromises = [];
         if (typeof resource.data === 'undefined') {
             resource.data = loadFile(resource.path, options);
         }
         filePromises.push(resource.data);
-        var languageValid = typeof language !== 'undefined' && includes(getLanguages(options), language);
-        var languageLoaded = typeof resource.translations[language] !== 'undefined';
+        const languageValid = typeof language !== 'undefined' && includes(getLanguages(options), language);
+        const languageLoaded = typeof resource.translations[language] !== 'undefined';
         if (languageValid) {
             if (!languageLoaded) {
-                var translationPath_1 = "lang/" + language + "/" + resource.path;
-                resource.translations[language] = loadFile(translationPath_1, options)
-                    .catch(function (err) {
-                    warning("translation file " + translationPath_1, err);
+                const translationPath = `lang/${language}/${resource.path}`;
+                resource.translations[language] = loadFile(translationPath, options)
+                    .catch(err => {
+                    warning(`translation file ${translationPath}`, err);
                     return Promise.resolve({});
                 });
             }
             filePromises.push(resource.translations[language]);
         }
-        return Promise.all(filePromises).then(function (fileResponses) {
-            var filesData = fileResponses.map(function (resp) { return resp.data || []; });
-            var primaryKey = resource.schema.primaryKey;
-            var data = joinData.apply(void 0, tslib_1.__spread([primaryKey, 'translation'], filesData));
-            return { data: data, resource: resource };
+        return Promise.all(filePromises).then(fileResponses => {
+            const filesData = fileResponses.map(resp => resp.data || []);
+            const primaryKey = resource.schema.primaryKey;
+            const data = joinData(primaryKey, 'translation', ...filesData);
+            return { data, resource };
         });
     }
     function getLanguages(options) {
         if (!options.datapackage.translations) {
             return [];
         }
-        return options.datapackage.translations.map(function (lang) { return lang.id; });
+        return options.datapackage.translations.map(lang => lang.id);
     }
     function loadFile(filePath, options) {
-        var _a = options.diagnostic.prepareDiagnosticFor('loadFile'), debug = _a.debug, error = _a.error;
-        var fullFilePath = ddf_query_validator_1.getFilePath(options.basePath, filePath);
-        debug("start reading \"" + filePath + "\"");
-        return new Promise(function (resolve, reject) {
-            options.fileReader.readText(fullFilePath, function (err, data) {
+        const { debug, error } = options.diagnostic.prepareDiagnosticFor('loadFile');
+        const fullFilePath = (0, ddf_query_validator_1.getFilePath)(options.basePath, filePath);
+        debug(`start reading "${filePath}"`);
+        return new Promise((resolve, reject) => {
+            options.fileReader.readText(fullFilePath, (err, data) => {
                 if (err) {
-                    error("fail \"" + filePath + "\" reading", err);
+                    error(`fail "${filePath}" reading`, err);
                     return reject(new ddfcsv_error_1.DdfCsvError(ddfcsv_error_1.FILE_READING_ERROR, err, fullFilePath));
                 }
                 Papa.parse(stripBom(data), {
                     header: true,
                     skipEmptyLines: true,
-                    dynamicTyping: function (headerName) {
-                        var concept = options.conceptsLookup.get(headerName) || {};
-                        return includes(['boolean', 'measure'], concept.concept_type);
-                    },
-                    complete: function (result) {
-                        debug("finish reading \"" + filePath + "\"");
+                    dynamicTyping: true,
+                    complete: result => {
+                        debug(`finish reading "${filePath}"`);
                         resolve(result);
                     },
-                    error: function (parseErr) {
-                        error("fail \"" + filePath + "\" parsing", parseErr);
+                    error: parseErr => {
+                        error(`fail "${filePath}" parsing`, parseErr);
                         reject(new ddfcsv_error_1.DdfCsvError(ddfcsv_error_1.CSV_PARSING_ERROR, parseErr, filePath));
                     }
                 });
@@ -705,7 +589,7 @@ function ddfCsvReader(logger) {
         if (resourcesLookup.size > 0) {
             return resourcesLookup;
         }
-        datapackagePar.resources.forEach(function (resource) {
+        datapackagePar.resources.forEach(resource => {
             if (!Array.isArray(resource.schema.primaryKey)) {
                 resource.schema.primaryKey = [resource.schema.primaryKey];
             }
@@ -718,10 +602,10 @@ function ddfCsvReader(logger) {
         if (keyValueLookup.size > 0) {
             return keyValueLookup;
         }
-        for (var collection in datapackagePar.ddfSchema) {
-            datapackagePar.ddfSchema[collection].map(function (kvPair) {
-                var key = createKeyString(kvPair.primaryKey);
-                var resources = kvPair.resources.map(function (resourceName) { return resourcesLookup.get(resourceName); });
+        for (const collection in datapackagePar.ddfSchema) {
+            datapackagePar.ddfSchema[collection].map(kvPair => {
+                const key = createKeyString(kvPair.primaryKey);
+                const resources = kvPair.resources.map(resourceName => resourcesLookup.get(resourceName));
                 if (keyValueLookup.has(key)) {
                     keyValueLookup.get(key).set(kvPair.value, resources);
                 }
@@ -733,7 +617,7 @@ function ddfCsvReader(logger) {
         return keyValueLookup;
     }
     return {
-        query: query
+        query
     };
 }
 exports.ddfCsvReader = ddfCsvReader;