@sis-cc/dotstatsuite-components 9.1.0 → 9.2.0

package/src/rules/src/table/factories/refineLayoutSize.js

@@ -0,0 +1,224 @@
+import * as R from 'ramda';
+
+const getPivotKey = indexes => R.pipe(
+  R.props(indexes),
+  R.join(':')
+);
+
+const getPivots = (layout, shape) => R.pipe(
+  R.props(shape),
+  R.map(R.pipe(R.pluck('__index'), getPivotKey))
+)(layout);
+
+const indexWithPivots = (pivots, observations) => R.pipe(
+  R.values,
+  R.reduce(
+    (acc, obs) => {
+      const path = R.map(
+        pivot => pivot(obs.orderedDimIndexes),
+        pivots
+      );
+      return R.over(
+        R.lensPath(path),
+        R.ifElse(R.isNil, R.always([obs]), R.append(obs))
+      )(acc);
+    },
+    {}
+  )
+)(observations);
+
+export const getCuratedCells = ({ layout, observations, shape }) => {
+  const pivots = getPivots(layout, shape);
+  return indexWithPivots(pivots, observations);
+};
+
+export const refineSections = (sections, extractedKeys, curatedObs) => R.pipe(
+  R.map(
+    (section) => {
+      const sectionKey = R.join(':', R.head(section) || []);
+      return R.over(
+        R.lensIndex(1),
+        R.filter(
+          rowData => {
+            const rowKey = R.join(':', rowData);
+            return R.pipe(
+              R.path([sectionKey, rowKey]),
+              R.omit(extractedKeys),
+              R.isEmpty,
+              R.not
+            )(curatedObs);
+          }
+        )
+      )(section);
+    }
+  ),
+  R.filter(
+    R.pipe(R.nth(1), R.isEmpty, R.not)
+  )
+)(sections);
+
+export const refineHeader = (header, extractedKeys, curatedObs) => R.filter( // extracted : { [sectionKey]: [...rowKeys] }
+  (header) => {
+    const headerKey = R.join(':', header);
+    return R.pipe(
+      R.prop(headerKey),
+      (sections) => {
+        return R.pickBy(
+          (rows, section) => R.pipe(
+            R.omit(R.propOr([], section, extractedKeys)),
+            R.isEmpty,
+            R.not
+          )(rows),
+          sections
+        );
+      },
+      R.isEmpty,
+      R.not
+    )(curatedObs);
+  }
+)(header);
+
+export const truncateSectionRows = (n, sectionsData) => {
+  const [rest, extracted] = R.pipe(
+    R.last, // last section
+    R.nth(1), // rows
+    R.splitAt(R.negate(n)) // truncate from the end
+  )(sectionsData);
+
+  const truncatedData = R.isEmpty(rest)
+    ? R.dropLast(1, sectionsData) // remove section if no more rows
+    : R.adjust(-1, R.adjust(1, R.always(rest)), sectionsData)
+
+  const sectionKey = R.join(':', R.pipe(R.last, R.head)(sectionsData) || []);
+  const rowKeys = R.map(R.join(':'), extracted);
+
+  const extractedLength = R.length(extracted);
+  if (extractedLength < n) {
+    const { truncated, extractedKeys } = truncateSectionRows(
+      n - extractedLength,
+      truncatedData
+    );
+    return ({ truncated, extractedKeys: { ...extractedKeys, [sectionKey]: rowKeys } });
+  }
+
+  return ({ truncated: truncatedData, extractedKeys: { [sectionKey]: rowKeys } });
+};
+
+export const truncateHeader = (n, headerData) => {
+  const [truncated, extracted] = R.splitAt(R.negate(n))(headerData);
+  return ({ truncated, extractedKeys: R.map(R.join(':'), extracted) });
+};
+
+const refineLength = length => length === 0 ? 1 : length;
+
+const truncateLayout = (isVertical) => R.ifElse(
+  R.always(isVertical),
+  truncateSectionRows,
+  truncateHeader,
+);
+
+const refineLayout = (isVertical) => R.ifElse(
+  R.always(isVertical),
+  refineHeader,
+  refineSections
+)
+
+const getShape = isVertical => isVertical ? ['header', 'sections', 'rows'] : ['sections', 'rows', 'header'];
+
+const getRefinedLayout = (isVertical, truncated, refined) => R.ifElse(
+  R.always(isVertical),
+  R.pipe(R.assoc('sections', truncated), R.assoc('header', refined)),
+  R.pipe(R.assoc('header', truncated), R.assoc('sections', refined)),
+)({});
+
+const segregateLayout = (isVertical) => R.ifElse(
+  R.always(isVertical),
+  R.props(['sections', 'header']),
+  R.props(['header', 'sections'])
+);
+
+export const refineLayoutSize = ({ layout, observations, limit }) => layoutIndexes => {
+  const { header, sections } = layoutIndexes;
+  if (R.isNil(limit) || limit === 0 || R.all(R.isEmpty, [header, sections])) {
+    return R.assoc('truncated', false, layoutIndexes);
+  }
+
+  //number of dimensions in header
+  const headerDimCount = R.pipe(
+    R.head, // first column
+    R.when(R.isNil, R.always([])),
+    R.length // number of dims
+  )(header);
+
+  //number of columns for values
+  const headerValuesCount = R.pipe(
+    R.length,
+    R.when(R.equals(0), R.always(1))
+  )(header);
+
+  //total of cells in header
+  const headerCellsCount = headerDimCount * (headerValuesCount + 1);
+
+  // number of dimensions in a row
+  const rowDimCount = R.pipe(
+    R.head, // firstSection
+    R.last, // rows,
+    R.head, // first row
+    R.length
+  )(sections);
+
+  // number of cells in a row
+  const rowCellsCount = rowDimCount + headerValuesCount + 1;
+
+  // number of rows
+  const rowsCount = R.pipe(
+    R.map(R.last),
+    R.unnest,
+    R.length,
+    R.add(1)
+  )(sections);
+
+  // total of cells in all rows
+  const rowsCellsCount = rowCellsCount * rowsCount;
+
+  // number of sections cells
+  const sectionsCellsCount = R.ifElse(
+    R.pipe(R.head, R.head, R.length, R.equals(0)),
+    R.always(0),
+    R.length
+  )(sections);
+
+  const total = rowsCellsCount + sectionsCellsCount + headerCellsCount;
+
+  const excess = total - limit;
+  if (excess <= 0) {
+    return R.assoc('truncated', false, layoutIndexes);
+  }
+
+  // total of cells in one column
+  const columnCellsCount = headerDimCount + rowsCount;
+
+  const isVertical = columnCellsCount > rowCellsCount
+
+  const [toTruncate, toRefine] = segregateLayout(isVertical)(layoutIndexes);
+
+
+  const cutLength = R.pipe(
+    R.ifElse(R.identity, R.always(rowCellsCount), R.always(columnCellsCount)),
+    R.divide(excess),
+    Math.ceil
+  )(isVertical);
+
+  const { truncated, extractedKeys } = truncateLayout(isVertical)(cutLength, toTruncate);
+
+  const curatedObs = R.pipe(
+    getShape,
+    (shape) => getCuratedCells({ layout, observations, shape })
+  )(isVertical);
+
+  const refined = refineLayout(isVertical)(toRefine, extractedKeys, curatedObs);
+
+  const result = getRefinedLayout(isVertical, truncated, refined);
+
+  return R.assoc('truncated', true, result);
+};

package/src/rules/src/table/preparators/prepareData.js

@@ -3,11 +3,9 @@ import { getObservationsType } from '../../properties/getObservationsType';
 import { parseDimensionsIds } from './parseDimensionsIds';
 import { getNoDisplayAnnotationsIndexes } from './getNoDisplayAnnotationsIndexes';
 import { getNoDisplayCodes } from './getNoDisplayCodes';
-import { getDisplay } from './getDisplay';
 import { parseAttributes } from './parseAttributes';
 import { getDimensionsAttributesRegisters } from './getDimensionsAttributesRegisters';
 import { parseAttributesValuesFromObservations } from './parseAttributesValuesFromObservations';
-import { parseObservations } from './parseObservations';
 import { getOneValueDimensions } from './getOneValueDimensions';
 import { getManyValuesDimensions } from './getManyValuesDimensions';
 import { getAttributeValue } from './getAttributeValue';
@@ -17,8 +15,11 @@ import { getUnitsDefinition } from '../units/getUnitsDefinition';
 import { getUnitsArtefacts } from '../units/getUnitsArtefacts';
 import { getHeaderUnits } from '../units/getHeaderUnits';
 import { getAttachmentSeriesIndexes } from '../units/getAttachmentSeriesIndexes';
-import { getUnitsSeriesOccurences } from '../units/getUnitsSeriesOccurences';
-import { refineDimSeriesUnits } from '../units/refineDimSeriesUnits';
+import { getUnitsSeries } from '../units/getUnitsSeries';
+
+import { getDisplay } from '../../preparators/getDisplay';
+import { getObservations } from '../../preparators/getObservations';
+import { enhanceObservations } from '../../preparators/enhanceObservations';
 
 export const prepareData = (sdmxJson, customAttributes, unitsProps={}) => {
   const {
@@ -30,10 +31,7 @@ export const prepareData = (sdmxJson, customAttributes, unitsProps={}) => {
     defaultCodes, annotationsDefinitionCodes, data: sdmxJson
   });
   const _customAttributes = R.assoc('units', unitsDefinitionCodes, customAttributes);
-  const dimensions = R.pipe(
-    R.pathOr([], ['data', 'structure', 'dimensions', 'observation']),
-    R.addIndex(R.map)((dim, index) => R.assoc('__index', index, dim))
-  )(sdmxJson);
+  const dimensions = R.pathOr([], ['data', 'structure', 'dimensions', 'observation'], sdmxJson);
 
   const dimensionsIndexedByIds = R.indexBy(R.prop('id'), dimensions);
 
@@ -60,40 +58,33 @@ export const prepareData = (sdmxJson, customAttributes, unitsProps={}) => {
     const attachmentSeriesIndexes = getAttachmentSeriesIndexes({ dimensions: _dimensions, unitsArtefacts });
   //------------------------------------------------------------------
 
+  const attributes = R.pathOr([], ['data', 'structure', 'attributes', 'observation'], sdmxJson);
+
   const { attributesIndexedByIds, attributesIdsIndexedByTargets } = parseAttributes({
     attributes: R.concat(
-      R.pathOr([], ['data', 'structure', 'attributes', 'observation'], sdmxJson),
+      attributes,
       R.pathOr([], ['data', 'structure', 'attributes', 'dataSet'], sdmxJson),
     ),
     parsedDimensionsIds,
     getDisplay: _getDisplay,
     customAttributes: _customAttributes,
   });
-  
+
   const seriesAttributesIds = R.propOr([], 'series', attributesIdsIndexedByTargets);
   const observationsAttributesIds = R.propOr([], 'observations', attributesIdsIndexedByTargets);
-  const observations = parseObservations(
-    R.pathOr({}, ['data', 'dataSets', 0, 'observations'], sdmxJson),
-    dimensions,
-    R.pick(R.concat(seriesAttributesIds, observationsAttributesIds), attributesIndexedByIds),
-    R.pipe(
-      R.pick(['prefscale', 'decimals']),
-      R.mapObjIndexed(R.flip(R.prop)(attributesIndexedByIds))
-    )(attributesIdsIndexedByTargets),
-    unitsArtefacts,
-    _getDisplay
-  );
+  const options = {
+    attachmentSeriesIndexes,
+    customAttributes,
+    noDisplayIndexes,
+    notDisplayedCodes,
+    attributesIds: R.concat(seriesAttributesIds, observationsAttributesIds),
+    unitsIds: unitsDefinitionCodes,
+  };
+  
+  const _observations = getObservations(sdmxJson);
+  const observations = enhanceObservations(dimensions, _observations, attributes, options);
   //------------------------------------------------------------------
-  const { rawUnitsSeries, obs } = getUnitsSeriesOccurences({
-    unitsArtefacts, attachmentSeriesIndexes
-  })(observations);
-  const unitsSeries = refineDimSeriesUnits({
-    unitsArtefacts: R.merge(
-      R.pathOr({}, ['dimensions', 'series'], unitsArtefacts),
-      R.pathOr({}, ['attributes', 'series'], unitsArtefacts)
-    ),
-    rejectedValueIds
-  })(rawUnitsSeries);
+  const unitsSeries = getUnitsSeries(observations);
   const unitsAttachmentSeriesIds = R.pipe(
     R.props(attachmentSeriesIndexes),
     R.pluck('id')
@@ -153,7 +144,7 @@ export const prepareData = (sdmxJson, customAttributes, unitsProps={}) => {
       unitsAttachmentSeriesIds,
       unitsSeries,
     },
-    observations: obs,
+    observations,
     observationsType: getObservationsType(sdmxJson.data),
     dataflowAttributes,
     dataflowName: R.path(['data', 'structure', 'name'], sdmxJson),

package/src/rules/src/table/units/appendUnitsInLayoutDataEntry.js

@@ -16,26 +16,23 @@ export const appendUnitsInLayoutDataEntry = ({
     R.join(':'),
     R.flip(R.propOr({}))(unitsSeries)
   )(layoutDataEntry);
+
   const unitsData = {
     dimension: unitDimension,
-    value: {
-      id: 'VAL',
-      unitsLabel: R.pipe(
-        R.props(unitsDefinitionCodes),
-        R.reduce(
-          (label, unit) => {
-            if (R.isNil(unit) || R.includes(R.path(['value', 'id'], unit), rejectedValueIds)) {
-              return label;
-            }
-            const unitLabel = dimensionValueDisplay(display)(R.propOr({}, 'value', unit));
-            return R.isEmpty(label)
-              ? unitLabel
-              : `${label}, ${unitLabel}`;
-          },
-          ''
-        )
-      )(unitsSerie)
-    }
+    value: R.pipe(
+      R.props(unitsDefinitionCodes),
+      R.reduce((acc, unit) => {
+        const value = R.prop('value', unit);
+        if (R.isNil(unit) || R.includes(value.id, rejectedValueIds)) {
+          return acc;
+        }
+        const unitLabel = dimensionValueDisplay(display)(R.propOr({}, 'value', unit));
+        return R.isEmpty(acc.id)
+          ? { id: value.id, unitsLabel: unitLabel }
+          : { id: `${acc.id} - ${value.id}` , unitsLabel: `${acc.unitsLabel}, ${unitLabel}` };
+      }, { id: '', unitsLabel: '' })
+    )(unitsSerie)
   };
+
   return R.append(unitsData, layoutDataEntry);
 };

package/src/rules/src/table/units/getAttachmentSeriesIndexes.js

@@ -21,6 +21,7 @@ export const getAttachmentSeriesIndexes = ({
     R.props(R.concat(unitsDimensionsCodes, targettedDimensionsCodes)),
     R.filter(dim => !R.isNil(dim)),
     R.pluck('index'),
-    R.uniq
+    R.uniq,
+    R.sortBy(R.identity)
   )(dimensions);
 };

package/src/rules/src/table/units/getUnitsSeries.js

@@ -0,0 +1,20 @@
+import * as R from 'ramda';
+
+/*
+  observations = { [obsKey]: { units: { serieKey: '', [id]: { id, name, value } } } };
+*/
+
+export const getUnitsSeries = (observations) => {
+  let series = {};
+  R.map(obs => {
+    series = R.over(R.lensProp(obs.units.serieKey), R.ifElse(R.isNil, R.always([obs.units]), R.append(obs.units)))(series);
+  }, observations);
+
+  return R.map(
+    serie => R.reduce(
+      (acc, units) => R.mergeWith((a, b) => R.path(['value', 'index'], a) === R.path(['value', 'index'], b) ? a : null, acc, units),
+      {},
+      serie
+    )
+  )(series);
+};

package/src/rules/src/v8-transformer.js

@@ -112,7 +112,7 @@ export const dataTransformer = (dataNew, options = {}) => {
   const getDimObservations = R.propOr([], 'observation');
   const dimObservations = getDimObservations(dimensions);
 
-  const getObservations = locale => R.reduce((acc, observation) => {
+  const getObservations = locale => R.addIndex(R.reduce)((acc, observation, obsIndex) => {
     const id = R.prop('id')(observation);
     const dimAnnotations = getRelationnalAnnotations(R.propOr([],'annotations')(observation))(annotations)
     const isHidden = R.includes(id)(hiddenIds) ? hiddenFormat : getIsHidden(dimAnnotations);
@@ -140,12 +140,14 @@ export const dataTransformer = (dataNew, options = {}) => {
     const sortedValues = R.pipe(
       getReccursiveChildren,
       R.flatten,
-      R.addIndex(R.map)((value, index) => R.assoc('__indexPosition', index, value))
+      R.addIndex(R.map)((value, index) => R.assoc('__indexPosition', index, value)),
+      R.sortBy(R.prop('__index'))
     )('##ROOT');
 
     return ({
       observation: R.append({
         ...observation,
+        __index: obsIndex,
         name: getLocalisedProp('names', observation),
         values: sortedValues,
         role: R.isNil(observation.roles) ? null : R.head(observation.roles),

package/test/appendUnitsInLayoutDataEntry.test.js

@@ -29,7 +29,7 @@ describe('appendUnitsInLayoutDataEntry tests', () => {
       unitsDefinitionCodes: [],
       unitsIndexes: [],
       partialUnitSerieIndexes: []
-    })([])).to.deep.equal([{ dimension: {}, value: { id: 'VAL', unitsLabel: '' } }]);
+    })([])).to.deep.equal([{ dimension: {}, value: { id: '', unitsLabel: '' } }]);
   });
   it('complete test', () => {
     expect(appendUnitsInLayoutDataEntry({
@@ -43,7 +43,7 @@ describe('appendUnitsInLayoutDataEntry tests', () => {
       { dimension: { id: 'D1', __index: 2 }, value: { id: 'V', __index: 0 } },
       { dimension: { id: 'D2', __index: 0 }, value: { id: 'V', __index: 0 } },
       { dimension: { id: 'D3', __index: 4 }, value: { id: 'V', __index: 0 } },
-      { dimension: { id: 'U' }, value: { id: 'VAL', unitsLabel: 'U3V, U1V' } }
+      { dimension: { id: 'U' }, value: { id: 'U3V - U1V', unitsLabel: 'U3V, U1V' } }
     ]);
   });
   it('rejectedValues test', () => {
@@ -59,7 +59,7 @@ describe('appendUnitsInLayoutDataEntry tests', () => {
         { dimension: { id: 'D1', __index: 2 }, value: { id: 'V', __index: 0 } },
         { dimension: { id: 'D2', __index: 0 }, value: { id: 'V', __index: 0 } },
         { dimension: { id: 'D3', __index: 4 }, value: { id: 'V', __index: 0 } },
-        { dimension: { id: 'U' }, value: { id: 'VAL', unitsLabel: 'U1V' } }
+        { dimension: { id: 'U' }, value: { id: 'U1V', unitsLabel: 'U1V' } }
     ]);
   })
 });