@sis-cc/dotstatsuite-components 9.1.2 → 9.2.2

package/src/rules/src/preparators/getDisplay.js

@@ -0,0 +1,20 @@
+import * as R from 'ramda';
+
+export const getDisplay = ({ noDisplayIndexes = [], notDisplayedCodes = {} }) => {
+  const annotationsValidator = R.pipe(
+    R.propOr([], 'annotations'),
+    R.intersection(noDisplayIndexes),
+    R.isEmpty
+  );
+
+  const blacklistValidator = R.ifElse(
+    R.has('values'),
+    ({ id }) => R.anyPass([
+      R.complement(R.has)(id),
+      R.hasPath([id, 'values'])
+    ])(notDisplayedCodes),
+    ({ id, parent }) => R.complement(R.hasPath)([parent, 'values', id], notDisplayedCodes)
+  );
+
+  return R.allPass([annotationsValidator, blacklistValidator]);
+};

package/src/rules/src/preparators/getObservations.js

@@ -0,0 +1,19 @@
+import * as R from 'ramda';
+
+export const getObservations = (sdmxJson) => R.pipe(
+  R.pathOr({}, ['data', 'dataSets', 0, 'observations']),
+  R.mapObjIndexed(
+    (observation, observationKey) => {
+      const value = R.head(observation);
+      const attrValuesIndexes = R.tail(observation);
+      const dimValuesIndexes = R.split(':', observationKey);
+
+      return ({
+        key: observationKey,
+        value,
+        attrValuesIndexes,
+        dimValuesIndexes
+      });
+    },
+  )
+)(sdmxJson);

package/src/rules/src/table/factories/getCells.js

@@ -1,5 +1,4 @@
 import * as R from 'ramda';
-import { getCellValue } from './getCellValue'; 
 import { dimensionValueDisplay } from '../../dimension-utils';
 
 const getAttrLabel = (display) => (attribute) => `${display(attribute)}: ${display(R.propOr({}, 'value', attribute))}`;
@@ -82,13 +81,10 @@ export const getCells = (data, display, customAttributes, unitsProps) => R.mapOb
       R.lensProp('flags'),
       getFlags(
         customAttributes, dimensionValueDisplay(display), unitsProps
-      )(R.pick(['attributes', 'units'], observation))
-    ),
-    R.set(R.lensProp('intValue'), R.prop('value')(observation)),
-    R.set(
-      R.lensProp('value'),
-      getCellValue(observation)
+      )(observation)
     ),
+    R.set(R.lensProp('intValue'), R.prop('value', observation)),
+    R.set(R.lensProp('value'), R.prop('formattedValue', observation)),
     R.assoc('indexedDimValIds', R.prop('indexedDimValIds', observation))
   )({}),
   R.propOr({}, 'observations', data)

package/src/rules/src/table/factories/getLayoutData.js

@@ -1,203 +1,169 @@
 import * as R from 'ramda';
-import { getSortedLayoutData } from './getSortedLayoutData';
-import { getCuratedCells } from './getCuratedCells';
 
-const layoutDataToKey = R.pipe(
-  R.addIndex(R.map)((entry, index) => ({ ...entry, f: R.path([index, 'value', '__index']) })),
-  R.map(R.prop('f')),
-  getters => R.converge((...results) => R.join(':', results), getters), 
+const indexesToLayoutData = (dimensions, indexes) => R.addIndex(R.map)(
+  (valueIndex, dimensionIndex) => {
+    const dimension = R.nth(dimensionIndex, dimensions);
+    return ({
+      dimension: R.pick(['id', 'name', '__index'], dimension),
+      value: R.nth(Number(valueIndex), R.propOr([], 'values', dimension))
+    });
+  },
+  indexes
 );
 
-export const refineSectionsData = (sectionsData, extractedKeys, curatedData, dataToKey) => R.pipe(
-  R.map(
-    (section) => {
-      const sectionKey = dataToKey.sections(R.head(section));
-      return R.over(
-        R.lensIndex(1),
-        R.filter(
-          rowData => {
-            const rowKey = dataToKey.rows(rowData);
-            return R.pipe(
-              R.path([sectionKey, rowKey]),
-              R.omit(extractedKeys),
-              R.isEmpty,
-              R.not
-            )(curatedData);
-          }
-        )
-      )(section);
-    }
-  ),
-  R.filter(
-    R.pipe(R.nth(1), R.isEmpty, R.not)
-  )
-)(sectionsData);
-
-export const refineHeaderData = (headerData, extractedKeys, curatedData, dataToKey) => R.filter( // extracted : { [sectionKey]: [...rowKeys] }
-  (header) => {
-    const headerKey = dataToKey.header(header);
-    return R.pipe(
-      R.prop(headerKey),
-      (sections) => {
-        return R.pickBy(
-          (rows, section) => R.pipe(
-            R.omit(R.propOr([], section, extractedKeys)),
-            R.isEmpty,
-            R.not
-          )(rows),
-          sections
-        );
-      },
-      R.isEmpty,
-      R.not
-    )(curatedData);
-  }
-)(headerData);
-
-export const truncateSectionRows = (n, sectionsData, dataToKey) => {
-  const [rest, extracted] = R.pipe(
-    R.last, // last section
-    R.nth(1), // rows
-    R.splitAt(R.negate(n)) // truncate from the end
-  )(sectionsData);
-
-  const truncatedData = R.isEmpty(rest)
-    ? R.dropLast(1, sectionsData) // remove section if no more rows
-    : R.adjust(-1, R.adjust(1, R.always(rest)), sectionsData)
-
-  const sectionKey = dataToKey.sections(R.pipe(R.last, R.head)(sectionsData));
-  const rowKeys = R.map(dataToKey.rows, extracted);
-
-  const extractedLength = R.length(extracted);
-  if (extractedLength < n) {
-    const { truncated, extractedKeys } = truncateSectionRows(
-      n - extractedLength,
-      truncatedData,
-      dataToKey
-    );
-    return ({ truncated, extractedKeys: { ...extractedKeys, [sectionKey]: rowKeys } });
-  }
-
-  return ({ truncated: truncatedData, extractedKeys: { [sectionKey]: rowKeys } });
+const parseParents = (value, previous) => {
+  const previousParents = R.pathOr([], ['value', 'parents'], previous);
+  const indexedParents = R.indexBy(R.identity, value.parents);
+
+  const refinedParents = R.reduce(
+    (acc, id) => {
+      if (R.has(id, indexedParents)) {
+        return R.append(id, acc);
+      }
+      return acc;
+    },
+    [],
+    R.append(R.path(['value', 'id'], previous), previousParents)
+  );
+  const lastParent = R.last(refinedParents);
+  const missingParents = R.takeLastWhile(id => id !== lastParent, value.parents);
+
+  return ({ parents: refinedParents, missingIds: missingParents });
 };
 
-export const truncateHeader = (n, headerData, dataToKey) => {
-  const [truncated, extracted] = R.splitAt(R.negate(n))(headerData);
-  return ({ truncated, extractedKeys: R.map(dataToKey.header, extracted) });
-};
-
-const refineLength = length => length === 0 ? 1 : length;
-
-const truncateLayout = (isVertical) => R.ifElse(
-  R.always(isVertical),
-  truncateSectionRows,
-  truncateHeader,
-);
-
-const refineLayout = (isVertical) => R.ifElse(
-  R.always(isVertical),
-  refineHeaderData,
-  refineSectionsData
-)
+const getRowData = (indexes, dimension) => {
+  return R.pipe(
+    R.propOr([], 'values'),
+    R.props(indexes),
+    R.reduce(
+      ({ data, missingParents }, _value) => {
+        const raw = {
+          dimension: R.pick(['id', 'name', '__index'], dimension),
+          value: _value,
+        };
+        if (R.isNil(_value.parents) || R.isEmpty(_value.parents)) {
+          return ({ data: R.append([raw], data), missingParents: {} });
+        }
+        const previous = R.head(R.last(data) || []);
+        if (R.last(_value.parents) === R.path(['value', 'id'], previous)) { //previous is parent
+          const value = R.set(
+            R.lensPath(['value', 'parents']),
+            R.append(
+              R.path(['value', 'id'], previous),
+              R.pathOr([], ['value', 'parents'], previous)
+            )
+          )(raw);
+          return ({ data: R.append([value], data), missingParents: {} });
+        }
+
+        const previousParent = R.pipe(R.pathOr([], ['value', 'parents']), R.last)(previous);
+        if (R.last(_value.parents) === previousParent) { //previous is bro
+          const value = R.set(
+            R.lensPath(['value']),
+            {
+              ..._value,
+              name: R.isEmpty(missingParents) ? _value.name : `${missingParents.name} > ${_value.name}`,
+              parents: R.pathOr([], ['value', 'parents'], previous)
+            }
+          )(raw);
+          return ({ data: R.append([value], data), missingParents });
+        }
+        const { parents, missingIds } = parseParents(_value, previous);
+        const _missingParents = R.reduce(
+          (acc, value) => {
+            if (value.id !== R.head(acc.missingIds)) {
+              return acc;
+            }
+            const _missingIds = R.tail(acc.missingIds);
+            if (R.isEmpty(acc.name)) {
+              return ({ name: value.name, missingIds: _missingIds });
+            }
+            return ({
+              name: `${acc.name} > ${value.name}`,
+              missingIds: _missingIds
+            });
+          },
+          { name: '', missingIds },
+          dimension.values
+        );
 
-const getShape = isVertical => isVertical ? ['header', 'sections', 'rows'] : ['sections', 'rows', 'header'];
+        const value = R.set(
+          R.lensProp('value'),
+          {
+            ..._value,
+            parents,
+            name: R.isEmpty(missingIds) ? _value.name : `${_missingParents.name} > ${_value.name}`,
+          }
+        )(raw);
 
-const getRefinedLayout = (isVertical, truncated, refined) => R.ifElse(
-  R.always(isVertical),
-  R.pipe(R.assoc('sectionsData', truncated), R.assoc('headerData', refined)),
-  R.pipe(R.assoc('headerData', truncated), R.assoc('sectionsData', refined)),
-)({});
+        return ({ data: R.append([value], data), missingParents: R.isEmpty(missingIds) ? {} : _missingParents });
+      },
+      { data: [], missingParents: {} }
+    ),
+    R.prop('data')
+  )(dimension);
+};
 
-const segregateLayout = (isVertical) => R.ifElse(
-  R.always(isVertical),
-  R.props(['sectionsData', 'headerData']),
-  R.props(['headerData', 'sectionsData'])
-);
+const getRowsData = (rowsIndexes, dimensions) => {
+  const dimension = R.head(dimensions);
 
-export const getLayoutData = ({ layout, observations, limit }) => {
-  const layoutData = getSortedLayoutData(layout, observations);
-  const { headerData, sectionsData } = layoutData;
-  if (R.isNil(limit) || limit === 0 || R.all(R.isEmpty, [headerData, sectionsData])) {
-    return R.assoc('truncated', false, layoutData);
+  if (R.length(dimensions) === 1) {
+    return getRowData(R.unnest(rowsIndexes), R.head(dimensions));
   }
 
-  //number of dimensions in header
-  const headerDimCount = R.pipe(
-    R.head, // first column
-    R.when(R.isNil, R.always([])),
-    R.length // number of dims
-  )(headerData);
-
-  //number of columns for values
-  const headerValuesCount = R.pipe(
-    R.length,
-    R.when(R.equals(0), R.always(1))
-  )(headerData);
-
-  //total of cells in header
-  const headerCellsCount = headerDimCount * (headerValuesCount + 1);
-
-  // number of dimensions in a row
-  const rowDimCount = R.pipe(
-    R.head, // firstSection
-    R.last, // rows,
-    R.head, // first row
-    R.length
-  )(sectionsData);
-
-  // number of cells in a row
-  const rowCellsCount = rowDimCount + headerValuesCount + 1;
-
-  // number of rows
-  const rowsCount = R.pipe(
-    R.map(R.last),
-    R.unnest,
-    R.length,
-    R.add(1)
-  )(sectionsData);
-
-  // total of cells in all rows
-  const rowsCellsCount = rowCellsCount * rowsCount;
-
-  // number of sections cells
-  const sectionsCellsCount = R.ifElse(
-    R.pipe(R.head, R.head, R.length, R.equals(0)),
-    R.always(0),
-    R.length
-  )(sectionsData);
-
-  const total = rowsCellsCount + sectionsCellsCount + headerCellsCount;
-
-  const excess = total - limit;
-  if (excess <= 0) {
-    return R.assoc('truncated', false, layoutData);
-  }
-
-  // total of cells in one column
-  const columnCellsCount = headerDimCount + rowsCount;
-
-  const isVertical = columnCellsCount > rowCellsCount
-
-  const [toTruncate, toRefine] = segregateLayout(isVertical)(layoutData);
-
-  const cutLength = R.pipe(
-    R.ifElse(R.identity, R.always(rowCellsCount), R.always(columnCellsCount)),
-    R.divide(excess),
-    Math.ceil
-  )(isVertical);
+  const groupedIndexes = R.reduce(
+    (acc, indexes) => {
+      const headIndex = R.head(indexes);
+      const last = R.last(acc.heads);
+      if (last === headIndex) {
+        return ({
+          ...acc,
+          tails: R.over(
+            R.lensIndex(-1),
+            R.append(R.tail(indexes))
+          )(acc.tails)
+        });
+      }
+      return ({
+        heads: R.append(headIndex, acc.heads),
+        tails: R.append([R.tail(indexes)], acc.tails)
+      });
+    },
+    { heads: [], tails: [] },
+    rowsIndexes
+  );
+
+  const headsDatas = getRowData(groupedIndexes.heads, R.head(dimensions));
+
+  const res = R.addIndex(R.map)(
+    (tail, i) => {
+      const tailData = getRowsData(tail, R.tail(dimensions));
+      return R.map(
+        R.prepend(R.head(headsDatas[i])),
+        tailData
+      );
+    },
+    groupedIndexes.tails
+  );
+
+  return R.unnest(res);
+};
 
-  const dataToKey = R.mapObjIndexed(layoutDataToKey, layout);
+export const getLayoutData = (layoutIndexes, layout) => {
+  const { header, sections, truncated } = layoutIndexes;
+  const headerData = R.map(i => indexesToLayoutData(layout.header, i), header);
 
-  const { truncated, extractedKeys } = truncateLayout(isVertical)(cutLength, toTruncate, dataToKey);
+  const sectionsData = R.map(
+    ([sectionIndexes, rows]) => {
+      const sectionData = indexesToLayoutData(layout.sections, sectionIndexes);
 
-  const curatedData = R.pipe(
-    getShape,
-    (shape) => getCuratedCells({ layout, observations, shape })
-  )(isVertical);
 
-  const refined = refineLayout(isVertical)(toRefine, extractedKeys, curatedData, dataToKey);
+      const rowsData = getRowsData(rows, layout.rows);
 
-  const result = getRefinedLayout(isVertical, truncated, refined);
+      return [sectionData, rowsData];
+    },
+    sections
+  );
 
-  return R.assoc('truncated', true, result);
+  return ({ headerData, sectionsData, truncated });
 };

package/src/rules/src/table/factories/getSortedLayoutIndexes.js

@@ -0,0 +1,108 @@
+import * as R from 'ramda';
+
+/*
+
+const layout = {
+  header: [
+    { ...dim, values: [] } // values already sorted regarding order annotation
+  ],
+  rows: [],
+  sections: []
+};
+
+const observations = {
+  [obsKey]: {
+    ...obs,
+    orderedDimIndexes: [sortedValIndex]
+  }
+}
+
+*/
+
+const getLayoutPivots = layout => {
+  return R.mapObjIndexed(
+    dimensions => R.props(R.pluck('__index', dimensions)),
+  )(layout);
+};
+
+const comparator = size => (a, b) => {
+  let i = 0;
+  while (a[i] === b[i] && i < size) {
+    i++;
+  }
+  return a[i] - b[i];
+};
+
+// simple call to ramda uniq method is very slow regarding performance 
+const uniqIndexes = (indexes) => R.pipe(
+  R.reduce(
+    (acc, i) => {
+      const key = R.join(':', i);
+      if (R.has(key, acc.keys)) {
+        return acc;
+      }
+      return ({
+        indexes: R.append(i, acc.indexes),
+        keys: R.assoc(key, key, acc.keys)
+      });
+    },
+    { indexes: [], keys: {} },
+  ),
+  R.prop('indexes')
+)(indexes);
+
+export const getSortedLayoutIndexes = (layout, observations) => {
+  const dimIndexes = R.pipe(R.values, R.pluck('orderedDimIndexes'))(observations);
+
+  const headerLength = R.length(layout.header);
+
+  const sectionLength = R.length(layout.sections);
+  const rowLength = R.length(layout.rows);
+
+  const layoutPivots = getLayoutPivots(layout);
+
+  const { header, sections } = R.pipe(
+    R.reduce(
+      (acc, indexes) => {
+        const headerIndexes = layoutPivots.header(indexes);
+        const sectionsIndexes = layoutPivots.sections(indexes);
+        const rowsIndexes = layoutPivots.rows(indexes);
+
+        return ({
+          header: R.append(headerIndexes, acc.header),
+          sections: R.append(
+            R.concat(sectionsIndexes, rowsIndexes),
+            acc.sections
+          )
+        });
+      },
+      { header: [], sections: [] },
+    ),
+    R.mapObjIndexed(uniqIndexes),
+    R.evolve({
+      header: R.sort(comparator(headerLength)),
+      sections: R.sort(comparator(sectionLength + rowLength))
+    }),
+    indexes => ({
+      ...indexes,
+      sections: R.reduce(
+        (acc, i) => {
+          const [sectionIndexes, rowIndexes] = R.splitAt(sectionLength, i);
+          const previousSecIndexes = R.pipe(R.nth(-1), i => R.isNil(i) ? null : R.head(i))(acc);
+          if (R.equals(sectionIndexes, previousSecIndexes)) {
+            return R.over(
+              R.lensIndex(-1),
+              R.over(R.lensIndex(1), R.append(rowIndexes))
+            )(acc);
+          }
+
+          return R.append([sectionIndexes, [rowIndexes]], acc);
+        },
+        [],
+        indexes.sections
+      )
+    })
+  )(dimIndexes);
+
+  return ({ header, sections });
+};

package/src/rules/src/table/factories/getTableData.js

@@ -3,15 +3,17 @@ import { withFlatHierarchy, isTimePeriodDimension } from '@sis-cc/dotstatsuite-s
 import { getCells } from './getCells';
 import { getConfirmedSeriesAttributesIds } from './getConfirmedSeriesAttributesIds';
 import { getCuratedCells } from './getCuratedCells';
-import { getLayoutData } from './getLayoutData';
 import { getLayoutDataWithFlags } from './getLayoutWithFlags';
-import { refineLayoutHierarchy } from './refineLayoutHierarchy';
 
 import { getUnitsInLayout } from '../units/getUnitsinLayout';
 import { appendUnitsInLayoutData } from '../units/appendUnitsInLayoutData';
 import { cleanUnitsInLayoutData } from '../units/cleanUnitsInLayoutData';
 
 
+import { getSortedLayoutIndexes } from './getSortedLayoutIndexes';
+import { refineLayoutSize } from './refineLayoutSize';
+import { getLayoutData } from './getLayoutData';
+
 export const getTableProps = ({ data, layoutIds, display, customAttributes, limit, isTimeInverted }) => {
   const seriesAttributes = R.propOr({}, 'seriesAttributes', data);
 
@@ -59,8 +61,8 @@ export const getTableProps = ({ data, layoutIds, display, customAttributes, limi
         R.lensProp('values'),
         R.sortWith([R.ascend(R.prop('__indexPosition')), R.ascend(R.prop('order'))]),
       )(dim),
-      R.when( // refine hierarchy on last row
-        R.pipe(R.prop('id'), R.equals(R.last(R.propOr([], 'rows', layoutIds)))),
+      R.when( // refine hierarchy on rows
+        R.pipe(R.prop('id'), R.flip(R.includes)(R.propOr([], 'rows', layoutIds))),
         withFlatHierarchy
       )
     ),
@@ -79,11 +81,9 @@ export const getTableProps = ({ data, layoutIds, display, customAttributes, limi
   )(data);
 
   const layoutData = R.pipe(
-    getLayoutData,
-    R.over(
-      R.lensProp('sectionsData'),
-      sectionsData => refineLayoutHierarchy(sectionsData, R.last(R.prop('rows', layout)))
-    ),
+    getSortedLayoutIndexes,
+    refineLayoutSize({ layout, observations: data.observations, limit: R.when(R.isNil, R.always(0))(limit) }),
+    indexes => getLayoutData(indexes, layout),
     appendUnitsInLayoutData({
       display, unitDimension, unitsDisplay: unitsLevelDisplay, unitsSeries,
       unitsDefinitionCodes, unitsIndexes: unitsAttachmentIndexesInLayout,
@@ -91,7 +91,7 @@ export const getTableProps = ({ data, layoutIds, display, customAttributes, limi
     }),
     getLayoutDataWithFlags(seriesAttributesValues, display, customAttributes),
     cleanUnitsInLayoutData({ unitsDisplay: unitsLevelDisplay, unitsLayoutIndexes: unitsIndexesInLayout })
-  )({ layout, observations: cells, limit: R.when(R.isNil, R.always(0))(limit) });
+  )(layout, R.propOr({}, 'observations', data));
   return ({
     cells: getCuratedCells({ layout, observations: cells, shape: ['header', 'sections', 'rows'] }),
     layout,