@vizabi/reader-ddfcsv 4.3.2 → 4.3.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vizabi/reader-ddfcsv",
-  "version": "4.3.2",
+  "version": "4.3.5",
   "description": "Vizabi DDFcsv reader",
   "author": "Vyacheslav Chub<vyacheslav.chub@valor-software.com>",
   "license": "GPL-3.0",
@@ -125,8 +125,8 @@
     "ts-loader": "4.3.0",
     "ts-node": "10.9.2",
     "tslib": " 2.6.2",
-    "tslint": "5.11.0",
-    "tslint-no-unused-expression-chai": "0.1.3",
+    "tslint": "6.1.3",
+    "tslint-no-unused-expression-chai": "0.1.4",
     "typescript": "5.4.5",
     "uglifyjs-webpack-plugin": "1.2.5",
     "webpack": "4.8.1",

package/src/ddf-csv.ts

@@ -41,6 +41,7 @@ export function ddfCsvReader (logger?: any) {
   let optimalFilesSet = [];
   let datapackage;
   let datapackagePromise;
+  let datasetWithConstraints = false;
 
   function loadDataPackage (baseOptions: IBaseReaderOptions): Promise<IDatapackage> {
     const datapackagePath = getFilePath(baseOptions.basePath);
@@ -165,8 +166,8 @@ export function ddfCsvReader (logger?: any) {
     concepts.forEach(row => options.conceptsLookup.set(row.concept, row));
   }
 
-  async function query (queryParam, baseOptions: IBaseReaderOptions) {
-    // console.log(JSON.stringify(queryParam, null, '\t'))
+  async function query (queryParam, _baseOptions: IBaseReaderOptions) {
+    const baseOptions = Object.assign({}, _baseOptions);
     const { warning, error } = baseOptions.diagnostic.prepareDiagnosticFor('query');
     let data;
 
@@ -180,10 +181,10 @@ export function ddfCsvReader (logger?: any) {
       if (isSchemaQuery(queryParam)) {
         data = await querySchema(queryParam, baseOptions);
       } else {
-        const appropriatePlugin = getAppropriatePlugin(this, queryParam, baseOptions);
+        const appropriatePlugin = datasetWithConstraints && getAppropriatePlugin(this, queryParam, baseOptions);
 
+        optimalFilesSet = [];
         if (appropriatePlugin) {
-          optimalFilesSet = [];
           const files = await appropriatePlugin.getRecommendedFilesSet();
           optimalFilesSet = files;
           queryParam.optimalFilesSet = [].concat(files, queryParam.optimalFilesSet);
@@ -201,7 +202,8 @@ export function ddfCsvReader (logger?: any) {
     return data;
   }
 
-  function queryData (queryParam, options: IBaseReaderOptions) {
+  function queryData (queryParam, _options: IBaseReaderOptions) {
+    const options = Object.assign({}, _options);
     const { debug } = options.diagnostic.prepareDiagnosticFor('queryData');
     const {
       select: { key = [], value = [] },
@@ -381,9 +383,37 @@ export function ddfCsvReader (logger?: any) {
     return result;
   }
 
+  function patchFilterForOrClause(filter) {
+
+    function processFilter(where) {
+      const whereKeys = Object.keys(where);
+      for (const key of whereKeys) {
+        if (key == "$or") {
+          where[key] = where[key].reduce((res, value) => {
+            const valueKeys = Object.keys(value);
+            if (valueKeys.length > 1) {
+              for (const key of valueKeys) {
+                res.push({ [key]: value[key] });
+              }
+            } else {
+              res.push(value);
+            }
+            return res;
+          }, []);
+        }
+      }
+      return where;
+    }
+
+    return processFilter(filter);
+  }
+
   function mergeFilters (...filters) {
     return filters.reduce((a, b) => {
-      a.$and.push(b);
+      if (!isEmpty(b)) {
+        patchFilterForOrClause(b);
+        a.$and.push(b);
+      }
 
       return a;
     }, { $and: [] });
@@ -461,7 +491,7 @@ export function ddfCsvReader (logger?: any) {
       return Promise.resolve({ [ joinID ]: join.where });
     } else {
       // entity concept
-      return query({
+      return queryData({
         select: { key: [ join.key ] },
         where: join.where,
         from: options.conceptsLookup.has(join.key) ? 'entities' : 'concepts'
@@ -557,7 +587,7 @@ export function ddfCsvReader (logger?: any) {
    */
   function getEntitySetFilter (conceptStrings, queryParam, options) {
     const promises = filterConceptsByType([ 'entity_set' ], conceptStrings, options)
-      .map(concept => query({
+      .map(concept => queryData({
           select: { key: [ concept.domain ], value: [ 'is--' + concept.concept ] },
           from: 'entities'
         }, Object.assign({}, options))
@@ -680,7 +710,7 @@ export function ddfCsvReader (logger?: any) {
 
           joinRow(resultRow, row, joinMode);
         } else {
-          result.set(keyString, row);
+          result.set(keyString, Object.assign({}, row));
         }
       });
 
@@ -835,7 +865,8 @@ export function ddfCsvReader (logger?: any) {
 
       const constraints = resource.schema.fields.reduce((result, field) => {
         if (field.constraints?.enum) {
-          result[field.name] = field.constraints.enum;
+          if (!datasetWithConstraints) datasetWithConstraints = true;
+          result[field.name] = field.constraints.enum.map(e => +e || e);
         }
         return result;
       }, {});
@@ -873,6 +904,7 @@ export function ddfCsvReader (logger?: any) {
 
   return {
     query,
+    queryData,
     loadFile
   };
 }

package/src/ddfcsv-reader.ts

@@ -15,7 +15,7 @@ export function prepareDDFCsvReaderObject (defaultResourceReader?: IResourceRead
     return {
       init (readerInfo) {
         // TODO: check validity of base path
-        this._basePath = githubPathAdapter(readerInfo.path);
+        this._basePath = githubPathAdapter(readerInfo.path || '');
 
         this._lastModified = readerInfo._lastModified;
         this.fileReader = externalResourceReader || defaultResourceReader;

package/src/resource-selection-optimizer/in-clause-under-conjunction.ts

@@ -1,9 +1,7 @@
-import * as path from 'path';
 import * as head from 'lodash.head';
 import * as values from 'lodash.values';
 import * as keys from 'lodash.keys';
 import * as get from 'lodash.get';
-import * as flattenDeep from 'lodash.flattendeep';
 import * as isEmpty from 'lodash.isempty';
 import * as startsWith from 'lodash.startswith';
 import * as includes from 'lodash.includes';
@@ -12,8 +10,6 @@ import { DdfCsvError } from '../ddfcsv-error';
 import { IDatapackage, IResourceSelectionOptimizer, IResourceRead, IBaseReaderOptions } from '../interfaces';
 import { QueryFeature, featureDetectors, IQuery } from 'ddf-query-validator';
 
-const Papa = require('papaparse');
-
 const WHERE_KEYWORD = 'where';
 const JOIN_KEYWORD = 'join';
 const KEY_IN = '$in';
@@ -21,10 +17,6 @@ const KEY_NIN = '$nin';
 const KEY_AND = '$and';
 const KEY_OR = '$or';
 
-const getFirstConditionClause = clause => head(values(clause));
-const getFirstKey = obj => head(keys(obj));
-const isOneKeyBased = obj => keys(obj).length === 1;
-
 export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
   private flow: any = {};
   private fileReader: IResourceRead;
@@ -44,11 +36,7 @@ export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
   isMatched(): boolean {
     this.flow.joinObject = get(this.query, JOIN_KEYWORD);
 
-    const relatedFeatures = compact(featureDetectors.map(detector => detector(this.query, this.conceptsLookup)));
-
-    return this.query.from === "datapoints";
-    // return includes(relatedFeatures, QueryFeature.WhereClauseBasedOnConjunction) &&
-    //   includes(relatedFeatures, QueryFeature.ConjunctionPartFromWhereClauseCorrespondsToJoin);
+    return this.query.from === "datapoints" && this.flow.joinObject;
   }
 
   async getRecommendedFilesSet(): Promise<string[]> {
@@ -59,7 +47,8 @@ export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
 
       let result;
       try {
-        this.collectProcessableClauses();
+        this.flow.processableClauses = await this.collectProcessableClauses();
+        if (!this.flow.processableClauses) return [];
         this.collectEntityFilesNames();
         const data = await this.collectEntities();
         this.fillEntityValuesHash(data);
@@ -80,24 +69,26 @@ export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
     }
   }
 
-  private collectProcessableClauses(): InClauseUnderConjunction {
-    const joinKeys = keys(this.flow.joinObject);
-
-    this.flow.processableClauses = [];
+  private collectProcessableClauses(): Promise<any> {
+    const joinKeys = keys(this.flow.joinObject).filter(
+      key => ["entity_domain", "entity_set"].includes(this.options.conceptsLookup.get(key.slice(1))?.concept_type));
+    if (!joinKeys.length) return Promise.resolve(false);
 
-    for (const joinKey of joinKeys) {
+    return Promise.all(joinKeys.map(joinKey => {
+      const key = this.flow.joinObject[joinKey].key;
       const where = get(this.flow.joinObject, `${joinKey}.${WHERE_KEYWORD}`, {});
 
-      if (this.singleAndField(where)) {
-        this.flow.processableClauses.push(...flattenDeep(where[KEY_AND].map(el => this.getProcessableClauses(el))));
-      } else if (this.singleOrField(where)) {
-        this.flow.processableClauses.push(...flattenDeep(where[KEY_OR].map(el => this.getProcessableClauses(el))));
-      } else {
-        this.flow.processableClauses.push(...this.getProcessableClauses(where));
-      }
-    }
+      return this.parent.queryData({
+        select: { key: [key] },
+        where,
+        from: this.options.conceptsLookup.has(key) ? 'entities' : 'concepts'
+      }, Object.assign({ joinID: joinKey }, this.options))
+        .then(result => ({
+          key,
+          entities: new Set(result.map(row => row[ key ]))
+        }));
+    }));
 
-    return this;
   }
 
   private collectEntityFilesNames(): InClauseUnderConjunction {
@@ -107,15 +98,15 @@ export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
 
     for (const schemaResourceRecord of this.datapackage.ddfSchema.entities) {
       for (const clause of this.flow.processableClauses) {
-        const primaryKey = getFirstKey(clause);
+        const key = clause.key;
 
-        if (head(schemaResourceRecord.primaryKey) === primaryKey) {
+        if (head(schemaResourceRecord.primaryKey) === key) {
           for (const resourceName of schemaResourceRecord.resources) {
             const resource = this.options.resourcesLookup.get(resourceName);
 
             this.flow.entityResources.add(resource);
             this.flow.entityFilesNames.add(resource.path);
-            this.flow.fileNameToPrimaryKeyHash.set(resource.path, primaryKey);
+            this.flow.fileNameToPrimaryKeyHash.set(resource.path, key);
           }
         }
       }
@@ -162,30 +153,14 @@ export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
   }
 
   private getFilesGroupsQueryClause(): InClauseUnderConjunction {
-    const getEntitiesExcept = (entityValuesToExclude: string[]): string[] => {
-      const result = [];
-
-      for (const entityKey of this.flow.entityValueToDomainHash.keys()) {
-        if (!includes(entityValuesToExclude, entityKey)) {
-          result.push(entityKey);
-        }
-      }
-
-      return result;
-    };
     const filesGroupsByClause = new Map();
 
     for (const clause of this.flow.processableClauses) {
       const filesGroupByClause = {
-        entities: this.flow.entityFilesNames,
-        datapoints: new Set(),
-        concepts: new Set()
+        datapoints: new Set()
       };
-      const firstConditionClause = getFirstConditionClause(clause);
-      const entityValuesFromClause = firstConditionClause[KEY_IN] || getEntitiesExcept(firstConditionClause[KEY_NIN]);
 
-      for (const entityValueFromClause of entityValuesFromClause) {
-        //filesGroupByClause.entities.add(this.flow.entityValueToFileHash.get(entityValueFromClause));
+      for (const entityValueFromClause of clause.entities) {
 
         const entitiesByQuery = this.flow.entityValueToDomainHash.get(entityValueFromClause);
 
@@ -194,13 +169,12 @@ export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
             for (const resourceName of schemaResourceRecord.resources) {
               if (includes(schemaResourceRecord.primaryKey, entityByQuery)) {
                 const resource = this.options.resourcesLookup.get(resourceName);
-                const constraint = resource.constraints[entityByQuery];
+                const constraint = resource.constraints?.[entityByQuery];
                 if ( constraint ) {
                   if (constraint.includes(entityValueFromClause)) {
                     filesGroupByClause.datapoints.add(resource.path);
                   }
-                }                
-                else {
+                } else {
                   filesGroupByClause.datapoints.add(resource.path);
                 }
               }
@@ -209,13 +183,7 @@ export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
         }
       }
 
-      for (const schemaResourceRecord of this.datapackage.ddfSchema.concepts) {
-        for (const resourceName of schemaResourceRecord.resources) {
-          filesGroupByClause.concepts.add(this.options.resourcesLookup.get(resourceName).path);
-        }
-      }
-
-      filesGroupsByClause.set(clause, filesGroupByClause);
+      filesGroupsByClause.set(clause.key, filesGroupByClause);
     }
 
     this.flow.filesGroupsByClause = filesGroupsByClause;
@@ -224,59 +192,29 @@ export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
   }
 
   private getOptimalFilesGroup(): string[] {
-    const clauseKeys = this.flow.filesGroupsByClause.keys();
 
-    let appropriateClauseKey;
-    let appropriateClauseSize;
-
-    for (const key of clauseKeys) {
-      const size = this.flow.filesGroupsByClause.get(key).datapoints.size +
-        this.flow.filesGroupsByClause.get(key).entities.size +
-        this.flow.filesGroupsByClause.get(key).concepts.size;
+    const entities = this.flow.entityFilesNames;
 
-      if (!appropriateClauseKey || size < appropriateClauseSize) {
-        appropriateClauseKey = key;
-        appropriateClauseSize = size;
+    const concepts = new Set();
+    for (const schemaResourceRecord of this.datapackage.ddfSchema.concepts) {
+      for (const resourceName of schemaResourceRecord.resources) {
+        concepts.add(this.options.resourcesLookup.get(resourceName).path);
       }
     }
 
-    if (!this.flow.filesGroupsByClause.get(appropriateClauseKey)) {
-      return [];
-    }
-
-    return [
-      ...Array.from(this.flow.filesGroupsByClause.get(appropriateClauseKey).concepts),
-      ...Array.from(this.flow.filesGroupsByClause.get(appropriateClauseKey).entities),
-      ...Array.from(this.flow.filesGroupsByClause.get(appropriateClauseKey).datapoints)
-    ] as string[];
-  }
-
-  private getProcessableClauses(clause) {
-    const result = [];
-    const clauseKeys = keys(clause);
+    const clauseKeys = this.flow.filesGroupsByClause.keys();
+    let datapoints = Array.from(this.flow.filesGroupsByClause.get(clauseKeys.next().value).datapoints);
 
     for (const key of clauseKeys) {
-      if (!startsWith(key, '$') && isOneKeyBased(clause[key])) {
-        // attention! this functionality process only first clause
-        // for example, { geo: { '$in': ['world'] } }
-        // in this example { geo: { '$in': ['world'] }, foo: { '$in': ['bar', 'baz'] }  }]
-        // foo: { '$in': ['bar', 'baz'] } will NOT be processed
-        const conditionKey = head(keys(clause[key]));
-
-        if (conditionKey === KEY_IN || conditionKey === KEY_NIN) {
-          result.push(clause);
-        }
-      }
+      datapoints = this.intersectArray(datapoints, Array.from(this.flow.filesGroupsByClause.get(key).datapoints));
     }
 
-    return result;
+    return [...Array.from(concepts), ...Array.from(entities)].concat(datapoints) as string[];
   }
 
-  private singleAndField(clause): boolean {
-    return isOneKeyBased(clause) && !!get(clause, KEY_AND);
-  }
 
-  private singleOrField(clause): boolean {
-    return isOneKeyBased(clause) && !!get(clause, KEY_OR);
+  private intersectArray(array1, array2) {
+    return array1.filter(value => array2.includes(value));
   }
+
 }

package/test/definition/datapoints-definition.spec.ts

@@ -232,7 +232,7 @@ describe('Datapoints definition errors in query', () => {
             where: {
               country_code: {
                 $in: [
-                  '900'
+                  900
                 ]
               }
             }

package/test/main.spec.ts

@@ -127,8 +127,10 @@ describe('General errors in ddfcsv reader', () => {
 
         expect(error.file).to.equal(expectedPath);
         expect(error.name).to.equal('DdfCsvError');
-        expect(error.message).to.equal(`JSON parsing error [filepath: ${expectedPath}]. Unexpected token ( in JSON at position 0.`);
-        expect(error.details).to.equal('Unexpected token ( in JSON at position 0');
+        expect(error.message).to.equal(`JSON parsing error [filepath: ${expectedPath}]. Unexpected token '(', "(
+    "nam"... is not valid JSON.`);
+        expect(error.details).to.equal(`Unexpected token '(', "(
+    "nam"... is not valid JSON`);
       }
     });
   });
@@ -245,8 +247,10 @@ describe('General errors in ddfcsv reader', () => {
 
         expect(error.file).to.equal(expectedPath);
         expect(error.name).to.equal('DdfCsvError');
-        expect(error.message).to.equal(`JSON parsing error [filepath: ${expectedPath}]. Unexpected token ( in JSON at position 0.`);
-        expect(error.details).to.equal('Unexpected token ( in JSON at position 0');
+        expect(error.message).to.equal(`JSON parsing error [filepath: ${expectedPath}]. Unexpected token '(', "(
+    "nam"... is not valid JSON.`);
+        expect(error.details).to.equal(`Unexpected token '(', "(
+    "nam"... is not valid JSON`);
       }
     });
   });