@vizabi/reader-ddfcsv 4.5.3 → 4.5.5

package/src/resource-selection-optimizer/in-clause-under-conjunction.ts

@@ -1,220 +1,220 @@
-import * as head from 'lodash.head';
-import * as values from 'lodash.values';
-import * as keys from 'lodash.keys';
-import * as get from 'lodash.get';
-import * as isEmpty from 'lodash.isempty';
-import * as startsWith from 'lodash.startswith';
-import * as includes from 'lodash.includes';
-import * as compact from 'lodash.compact';
-import { DdfCsvError } from '../ddfcsv-error';
-import { IDatapackage, IResourceSelectionOptimizer, IResourceRead, IBaseReaderOptions } from '../interfaces';
-import { QueryFeature, featureDetectors, IQuery } from '@vizabi/ddf-query-validator';
-
-const WHERE_KEYWORD = 'where';
-const JOIN_KEYWORD = 'join';
-const KEY_IN = '$in';
-const KEY_NIN = '$nin';
-const KEY_AND = '$and';
-const KEY_OR = '$or';
-
-export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
-  private flow: any = {};
-  private fileReader: IResourceRead;
-  private datasetPath: string;
-  private query: IQuery;
-  private datapackage: IDatapackage;
-  private conceptsLookup;
-
-  constructor(private parent, queryParam, private options: IBaseReaderOptions) {
-    this.fileReader = options.fileReader;
-    this.datasetPath = options.basePath;
-    this.query = queryParam;
-    this.datapackage = options.datapackage;
-    this.conceptsLookup = options.conceptsLookup;
-  }
-
-  isMatched(): boolean {
-    this.flow.joinObject = get(this.query, JOIN_KEYWORD);
-
-    return this.query.from === "datapoints" && this.flow.joinObject;
-  }
-
-  async getRecommendedFilesSet(): Promise<string[]> {
-    const { debug, error, warning } = this.options.diagnostic.prepareDiagnosticFor('getRecommendedFilesSet');
-
-    if (this.isMatched()) {
-      debug('plugin matched');
-
-      let result;
-      try {
-        this.flow.processableClauses = await this.collectProcessableClauses();
-        if (!this.flow.processableClauses) return [];
-        this.collectEntityFilesNames();
-        const data = await this.collectEntities();
-        this.fillEntityValuesHash(data);
-        this.getFilesGroupsQueryClause();
-        result = this.getOptimalFilesGroup();
-      } catch (err) {
-        error('wrong data processing', err);
-        return [];
-      }
-
-      debug('recommended files found', result);
-
-      return result;
-    } else {
-      const message = `Plugin "InClauseUnderConjunction" is not matched!`;
-      warning(message);
-      throw new DdfCsvError(message, 'InClauseUnderConjunction plugin');
-    }
-  }
-
-  private collectProcessableClauses(): Promise<any> {
-    const joinKeys = keys(this.flow.joinObject).filter(
-      key => ["entity_domain", "entity_set"].includes(this.options.conceptsLookup.get(key.slice(1))?.concept_type));
-    if (!joinKeys.length) return Promise.resolve(false);
-
-    return Promise.all(joinKeys.map(joinKey => {
-      const key = this.flow.joinObject[joinKey].key;
-      const where = get(this.flow.joinObject, `${joinKey}.${WHERE_KEYWORD}`, {});
-
-      return this.parent.queryData({
-        select: { key: [key] },
-        where,
-        from: this.options.conceptsLookup.has(key) ? 'entities' : 'concepts'
-      }, Object.assign({ joinID: joinKey }, this.options))
-        .then(result => ({
-          key,
-          entities: new Set(result.map(row => row[ key ]))
-        }));
-    }));
-
-  }
-
-  private collectEntityFilesNames(): InClauseUnderConjunction {
-    this.flow.entityFilesNames = new Set();
-    this.flow.entityResources = new Set();
-    this.flow.fileNameToPrimaryKeyHash = new Map();
-
-    for (const schemaResourceRecord of this.datapackage.ddfSchema.entities) {
-      for (const clause of this.flow.processableClauses) {
-        const key = clause.key;
-
-        if (head(schemaResourceRecord.primaryKey) === key) {
-          for (const resourceName of schemaResourceRecord.resources) {
-            const resource = this.options.resourcesLookup.get(resourceName);
-
-            this.flow.entityResources.add(resource);
-            this.flow.entityFilesNames.add(resource.path);
-            this.flow.fileNameToPrimaryKeyHash.set(resource.path, key);
-          }
-        }
-      }
-    }
-
-    return this;
-  }
-
-  private collectEntities(): Promise<any> {
-    const self = this;
-    const actions = [...self.flow.entityResources].map(resource => {
-      return (resource.data || (resource.data = self.parent.loadFile(resource.path, self.options)))
-        .then(data => ({result: data, file: resource.path}));
-    });
-
-    return Promise.all(actions);
-  }
-
-  private fillEntityValuesHash(entitiesData): InClauseUnderConjunction {
-    const getSubdomainsFromRecord = record => compact(keys(record)
-      .filter(key => startsWith(key, 'is--') && (record[key] === true))
-      .map(key => key.replace(/^is--/, '')));
-
-    this.flow.entityValueToFileHash = new Map();
-    this.flow.entityValueToDomainHash = new Map();
-
-    for (const entityFileDescriptor of entitiesData) {
-      for (const entityRecord of entityFileDescriptor.result.data) {
-        const primaryKeyForThisFile = this.flow.fileNameToPrimaryKeyHash.get(entityFileDescriptor.file);
-        const domainsForCurrentRecord = [...getSubdomainsFromRecord(entityRecord)];
-
-        if (isEmpty(domainsForCurrentRecord)) {
-          domainsForCurrentRecord.push(primaryKeyForThisFile);
-        }
-
-        const primaryKeyCellValue = entityRecord[primaryKeyForThisFile] || entityRecord[domainsForCurrentRecord[0]];
-
-        this.flow.entityValueToDomainHash.set(primaryKeyCellValue, domainsForCurrentRecord);
-        this.flow.entityValueToFileHash.set(primaryKeyCellValue, entityFileDescriptor.file);
-      }
-    }
-
-    return this;
-  }
-
-  private getFilesGroupsQueryClause(): InClauseUnderConjunction {
-    const filesGroupsByClause = new Map();
-
-    for (const clause of this.flow.processableClauses) {
-      const filesGroupByClause = {
-        datapoints: new Set()
-      };
-
-      for (const entityValueFromClause of clause.entities) {
-
-        const entitiesByQuery = this.flow.entityValueToDomainHash.get(entityValueFromClause);
-
-        for (const entityByQuery of entitiesByQuery) {
-          for (const schemaResourceRecord of this.datapackage.ddfSchema.datapoints) {
-            for (const resourceName of schemaResourceRecord.resources) {
-              if (includes(schemaResourceRecord.primaryKey, entityByQuery)) {
-                const resource = this.options.resourcesLookup.get(resourceName);
-                const constraint = resource.constraints?.[entityByQuery];
-                if ( constraint ) {
-                  if (constraint.includes(entityValueFromClause)) {
-                    filesGroupByClause.datapoints.add(resource.path);
-                  }
-                } else {
-                  filesGroupByClause.datapoints.add(resource.path);
-                }
-              }
-            }
-          }
-        }
-      }
-
-      filesGroupsByClause.set(clause.key, filesGroupByClause);
-    }
-
-    this.flow.filesGroupsByClause = filesGroupsByClause;
-
-    return this;
-  }
-
-  private getOptimalFilesGroup(): string[] {
-
-    const entities = this.flow.entityFilesNames;
-
-    const concepts = new Set();
-    for (const schemaResourceRecord of this.datapackage.ddfSchema.concepts) {
-      for (const resourceName of schemaResourceRecord.resources) {
-        concepts.add(this.options.resourcesLookup.get(resourceName).path);
-      }
-    }
-
-    const clauseKeys = this.flow.filesGroupsByClause.keys();
-    let datapoints = Array.from(this.flow.filesGroupsByClause.get(clauseKeys.next().value).datapoints);
-
-    for (const key of clauseKeys) {
-      datapoints = this.intersectArray(datapoints, Array.from(this.flow.filesGroupsByClause.get(key).datapoints));
-    }
-
-    return [...Array.from(concepts), ...Array.from(entities)].concat(datapoints) as string[];
-  }
-
-
-  private intersectArray(array1, array2) {
-    return array1.filter(value => array2.includes(value));
-  }
-
-}
+import head from 'lodash-es/head';
+import values from 'lodash-es/values';
+import keys from 'lodash-es/keys';
+import get from 'lodash-es/get';
+import isEmpty from 'lodash-es/isEmpty';
+import startsWith from 'lodash-es/startsWith';
+import includes from 'lodash-es/includes';
+import compact from 'lodash-es/compact';
+import { DdfCsvError } from '../ddfcsv-error';
+import { IDatapackage, IResourceSelectionOptimizer, IResourceRead, IBaseReaderOptions } from '../interfaces';
+import { QueryFeature, featureDetectors, IQuery } from '@vizabi/ddf-query-validator';
+
+const WHERE_KEYWORD = 'where';
+const JOIN_KEYWORD = 'join';
+const KEY_IN = '$in';
+const KEY_NIN = '$nin';
+const KEY_AND = '$and';
+const KEY_OR = '$or';
+
+export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
+  private flow: any = {};
+  private fileReader: IResourceRead;
+  private datasetPath: string;
+  private query: IQuery;
+  private datapackage: IDatapackage;
+  private conceptsLookup;
+
+  constructor(private parent, queryParam, private options: IBaseReaderOptions) {
+    this.fileReader = options.fileReader;
+    this.datasetPath = options.basePath;
+    this.query = queryParam;
+    this.datapackage = options.datapackage;
+    this.conceptsLookup = options.conceptsLookup;
+  }
+
+  isMatched(): boolean {
+    this.flow.joinObject = get(this.query, JOIN_KEYWORD);
+
+    return this.query.from === "datapoints" && this.flow.joinObject;
+  }
+
+  async getRecommendedFilesSet(): Promise<string[]> {
+    const { debug, error, warning } = this.options.diagnostic.prepareDiagnosticFor('getRecommendedFilesSet');
+
+    if (this.isMatched()) {
+      debug('plugin matched');
+
+      let result;
+      try {
+        this.flow.processableClauses = await this.collectProcessableClauses();
+        if (!this.flow.processableClauses) return [];
+        this.collectEntityFilesNames();
+        const data = await this.collectEntities();
+        this.fillEntityValuesHash(data);
+        this.getFilesGroupsQueryClause();
+        result = this.getOptimalFilesGroup();
+      } catch (err) {
+        error('wrong data processing', err);
+        return [];
+      }
+
+      debug('recommended files found', result);
+
+      return result;
+    } else {
+      const message = `Plugin "InClauseUnderConjunction" is not matched!`;
+      warning(message);
+      throw new DdfCsvError(message, 'InClauseUnderConjunction plugin');
+    }
+  }
+
+  private collectProcessableClauses(): Promise<any> {
+    const joinKeys = keys(this.flow.joinObject).filter(
+      key => ["entity_domain", "entity_set"].includes(this.options.conceptsLookup.get(key.slice(1))?.concept_type));
+    if (!joinKeys.length) return Promise.resolve(false);
+
+    return Promise.all(joinKeys.map(joinKey => {
+      const key = this.flow.joinObject[joinKey].key;
+      const where = get(this.flow.joinObject, `${joinKey}.${WHERE_KEYWORD}`, {});
+
+      return this.parent.queryData({
+        select: { key: [key] },
+        where,
+        from: this.options.conceptsLookup.has(key) ? 'entities' : 'concepts'
+      }, Object.assign({ joinID: joinKey }, this.options))
+        .then(result => ({
+          key,
+          entities: new Set(result.map(row => row[ key ]))
+        }));
+    }));
+
+  }
+
+  private collectEntityFilesNames(): InClauseUnderConjunction {
+    this.flow.entityFilesNames = new Set();
+    this.flow.entityResources = new Set();
+    this.flow.fileNameToPrimaryKeyHash = new Map();
+
+    for (const schemaResourceRecord of this.datapackage.ddfSchema.entities) {
+      for (const clause of this.flow.processableClauses) {
+        const key = clause.key;
+
+        if (head(schemaResourceRecord.primaryKey) === key) {
+          for (const resourceName of schemaResourceRecord.resources) {
+            const resource = this.options.resourcesLookup.get(resourceName);
+
+            this.flow.entityResources.add(resource);
+            this.flow.entityFilesNames.add(resource.path);
+            this.flow.fileNameToPrimaryKeyHash.set(resource.path, key);
+          }
+        }
+      }
+    }
+
+    return this;
+  }
+
+  private collectEntities(): Promise<any> {
+    const self = this;
+    const actions = [...self.flow.entityResources].map(resource => {
+      return (resource.data || (resource.data = self.parent.loadFile(resource.path, self.options)))
+        .then(data => ({result: data, file: resource.path}));
+    });
+
+    return Promise.all(actions);
+  }
+
+  private fillEntityValuesHash(entitiesData): InClauseUnderConjunction {
+    const getSubdomainsFromRecord = record => compact(keys(record)
+      .filter(key => startsWith(key, 'is--') && (record[key] === true))
+      .map(key => key.replace(/^is--/, '')));
+
+    this.flow.entityValueToFileHash = new Map();
+    this.flow.entityValueToDomainHash = new Map();
+
+    for (const entityFileDescriptor of entitiesData) {
+      for (const entityRecord of entityFileDescriptor.result.data) {
+        const primaryKeyForThisFile = this.flow.fileNameToPrimaryKeyHash.get(entityFileDescriptor.file);
+        const domainsForCurrentRecord = [...getSubdomainsFromRecord(entityRecord)];
+
+        if (isEmpty(domainsForCurrentRecord)) {
+          domainsForCurrentRecord.push(primaryKeyForThisFile);
+        }
+
+        const primaryKeyCellValue = entityRecord[primaryKeyForThisFile] || entityRecord[domainsForCurrentRecord[0]];
+
+        this.flow.entityValueToDomainHash.set(primaryKeyCellValue, domainsForCurrentRecord);
+        this.flow.entityValueToFileHash.set(primaryKeyCellValue, entityFileDescriptor.file);
+      }
+    }
+
+    return this;
+  }
+
+  private getFilesGroupsQueryClause(): InClauseUnderConjunction {
+    const filesGroupsByClause = new Map();
+
+    for (const clause of this.flow.processableClauses) {
+      const filesGroupByClause = {
+        datapoints: new Set()
+      };
+
+      for (const entityValueFromClause of clause.entities) {
+
+        const entitiesByQuery = this.flow.entityValueToDomainHash.get(entityValueFromClause);
+
+        for (const entityByQuery of entitiesByQuery) {
+          for (const schemaResourceRecord of this.datapackage.ddfSchema.datapoints) {
+            for (const resourceName of schemaResourceRecord.resources) {
+              if (includes(schemaResourceRecord.primaryKey, entityByQuery)) {
+                const resource = this.options.resourcesLookup.get(resourceName);
+                const constraint = resource.constraints?.[entityByQuery];
+                if ( constraint ) {
+                  if (constraint.includes(entityValueFromClause)) {
+                    filesGroupByClause.datapoints.add(resource.path);
+                  }
+                } else {
+                  filesGroupByClause.datapoints.add(resource.path);
+                }
+              }
+            }
+          }
+        }
+      }
+
+      filesGroupsByClause.set(clause.key, filesGroupByClause);
+    }
+
+    this.flow.filesGroupsByClause = filesGroupsByClause;
+
+    return this;
+  }
+
+  private getOptimalFilesGroup(): string[] {
+
+    const entities = this.flow.entityFilesNames;
+
+    const concepts = new Set();
+    for (const schemaResourceRecord of this.datapackage.ddfSchema.concepts) {
+      for (const resourceName of schemaResourceRecord.resources) {
+        concepts.add(this.options.resourcesLookup.get(resourceName).path);
+      }
+    }
+
+    const clauseKeys = this.flow.filesGroupsByClause.keys();
+    let datapoints = Array.from(this.flow.filesGroupsByClause.get(clauseKeys.next().value).datapoints);
+
+    for (const key of clauseKeys) {
+      datapoints = this.intersectArray(datapoints, Array.from(this.flow.filesGroupsByClause.get(key).datapoints));
+    }
+
+    return [...Array.from(concepts), ...Array.from(entities)].concat(datapoints) as string[];
+  }
+
+
+  private intersectArray(array1, array2) {
+    return array1.filter(value => array2.includes(value));
+  }
+
+}

package/src/resource-selection-optimizer/index.ts

@@ -1,11 +1,11 @@
-import { InClauseUnderConjunction } from './in-clause-under-conjunction';
-import * as head from 'lodash.head';
-import { IBaseReaderOptions, IResourceSelectionOptimizer } from '../interfaces';
-
-export function getAppropriatePlugin(parent, queryParam, options: IBaseReaderOptions): IResourceSelectionOptimizer {
-  const plugins = [
-    new InClauseUnderConjunction(parent, queryParam, options)
-  ];
-
-  return head(plugins.filter(plugin => plugin.isMatched()));
-}
+import { InClauseUnderConjunction } from './in-clause-under-conjunction';
+import head from 'lodash-es/head';
+import { IBaseReaderOptions, IResourceSelectionOptimizer } from '../interfaces';
+
+export function getAppropriatePlugin(parent, queryParam, options: IBaseReaderOptions): IResourceSelectionOptimizer {
+  const plugins = [
+    new InClauseUnderConjunction(parent, queryParam, options)
+  ];
+
+  return head(plugins.filter(plugin => plugin.isMatched()));
+}

package/src/test-cases/entities.ts

@@ -1,52 +1,52 @@
-import {
-  BASE_PATH,
-  EMPTY_TRANSLATIONS_PATH,
-  WS_TESTING_PATH
-} from '../../test/common';
-import {
-  CONCEPT_TYPE_ENTITY_DOMAIN,
-  CONCEPT_TYPE_ENTITY_SET,
-  isEntityDomainOrSet,
-  RESERVED_CONCEPT,
-  RESERVED_CONCEPT_TYPE,
-  RESERVED_DOMAIN,
-  RESERVED_DRILL_UP
-} from '@vizabi/ddf-query-validator';
-
-import * as map from 'lodash.map';
-import * as flatMap from 'lodash.flatmap';
-import * as isNil from 'lodash.isnil';
-import * as flatten from 'lodash.flatten';
-
-const ALL_OPERATORS: string[] = [null, '.', '.is--'];
-const ALL_CONCEPTS: string[] = ['', ' ', RESERVED_CONCEPT, RESERVED_CONCEPT_TYPE, RESERVED_DOMAIN, RESERVED_DRILL_UP, 'company', 'english_speaking', 'company_scale', 'name', 'anno', 'lines_of_code', 'region', 'country', 'latitude', 'longitude', 'full_name_changed', 'project', 'domain', 'additional_column', 'meeting_style', 'popular_appeal', 'methodology'];
-const ALL_ENTITY_SETS_AND_DOMAINS: string[] = ['company', 'english_speaking', 'company_scale', 'region', 'project'];
-
-export const initData = {};
-export const testsDescriptors = ALL_ENTITY_SETS_AND_DOMAINS.reduce((acc, entitySetOrDomain: string) => {
-  const BINARY_OPERATORS = [...flatMap(ALL_CONCEPTS, (parent: string) => map(ALL_CONCEPTS, (child) => [parent, child]))];
-  const UNARY_OPERATORS = [...ALL_CONCEPTS];
-
-  const description = `Autogenerated tests for ${entitySetOrDomain}`;
-
-  acc[description] = flatten(
-    ALL_OPERATORS.map((operator: string) => {
-      const CONCEPTS_CLAUSE: any[] = isNil(operator) ? UNARY_OPERATORS : map([...BINARY_OPERATORS], (args: string[]) => args.join(operator));
-
-      return CONCEPTS_CLAUSE.map((clause: string) => ({
-        itTitle: `should be fine for ${clause}`,
-        query: {
-          repositoryPath: BASE_PATH + WS_TESTING_PATH + '/master-HEAD',
-          dataset: WS_TESTING_PATH,
-          from: 'entities',
-          select: {key: [entitySetOrDomain]},
-          where: {
-            [clause]: true
-          }
-        }
-      }));
-    })
-  );
-
-  return acc;
-}, {});
+import {
+  BASE_PATH,
+  EMPTY_TRANSLATIONS_PATH,
+  WS_TESTING_PATH
+} from '../../test/common';
+import {
+  CONCEPT_TYPE_ENTITY_DOMAIN,
+  CONCEPT_TYPE_ENTITY_SET,
+  isEntityDomainOrSet,
+  RESERVED_CONCEPT,
+  RESERVED_CONCEPT_TYPE,
+  RESERVED_DOMAIN,
+  RESERVED_DRILL_UP
+} from '@vizabi/ddf-query-validator';
+
+import map from 'lodash-es/map';
+import flatMap from 'lodash-es/flatMap';
+import isNil from 'lodash-es/isNil';
+import flatten from 'lodash-es/flatten';
+
+const ALL_OPERATORS: string[] = [null, '.', '.is--'];
+const ALL_CONCEPTS: string[] = ['', ' ', RESERVED_CONCEPT, RESERVED_CONCEPT_TYPE, RESERVED_DOMAIN, RESERVED_DRILL_UP, 'company', 'english_speaking', 'company_scale', 'name', 'anno', 'lines_of_code', 'region', 'country', 'latitude', 'longitude', 'full_name_changed', 'project', 'domain', 'additional_column', 'meeting_style', 'popular_appeal', 'methodology'];
+const ALL_ENTITY_SETS_AND_DOMAINS: string[] = ['company', 'english_speaking', 'company_scale', 'region', 'project'];
+
+export const initData = {};
+export const testsDescriptors = ALL_ENTITY_SETS_AND_DOMAINS.reduce((acc, entitySetOrDomain: string) => {
+  const BINARY_OPERATORS = [...flatMap(ALL_CONCEPTS, (parent: string) => map(ALL_CONCEPTS, (child) => [parent, child]))];
+  const UNARY_OPERATORS = [...ALL_CONCEPTS];
+
+  const description = `Autogenerated tests for ${entitySetOrDomain}`;
+
+  acc[description] = flatten(
+    ALL_OPERATORS.map((operator: string) => {
+      const CONCEPTS_CLAUSE: any[] = isNil(operator) ? UNARY_OPERATORS : map([...BINARY_OPERATORS], (args: string[]) => args.join(operator));
+
+      return CONCEPTS_CLAUSE.map((clause: string) => ({
+        itTitle: `should be fine for ${clause}`,
+        query: {
+          repositoryPath: BASE_PATH + WS_TESTING_PATH + '/master-HEAD',
+          dataset: WS_TESTING_PATH,
+          from: 'entities',
+          select: {key: [entitySetOrDomain]},
+          where: {
+            [clause]: true
+          }
+        }
+      }));
+    })
+  );
+
+  return acc;
+}, {});

package/test/common.ts

@@ -1,5 +1,5 @@
 import * as chai from 'chai';
-import * as _ from 'lodash';
+import * as _ from 'lodash-es';
 
 const expect = chai.expect;
 

package/test/definition/datapoints-definition.spec.ts

@@ -1,6 +1,6 @@
 import * as chai from 'chai';
 import { getDDFCsvReaderObject } from '../../src/index';
-import * as _ from 'lodash';
+// import * as _ from 'lodash';
 import {
   BASE_PATH,
   BIG_PATH,