@erxes/api-shared 0.0.0-stage → 3.2.19-2

package/src/core-modules/properties/filter.test.ts

@@ -0,0 +1,74 @@
+import { buildPropertyFilter } from './filter';
+import { toPropertyRowKey } from './keys';
+
+const encode = (fieldId: string, operator: string, value?: string) =>
+  value === undefined
+    ? `${encodeURIComponent(fieldId)}:${operator}`
+    : `${encodeURIComponent(fieldId)}:${operator}:${encodeURIComponent(value)}`;
+
+const school = toPropertyRowKey('edu', 'school');
+const year = toPropertyRowKey('edu', 'year');
+
+describe('buildPropertyFilter', () => {
+  it('builds a plain field condition', () => {
+    expect(buildPropertyFilter(encode('plan', 'eq', 'pro'))).toEqual([
+      { 'propertiesData.plan': { $in: ['pro'] } },
+    ]);
+  });
+
+  it('matches one entry of a repeating group', () => {
+    expect(buildPropertyFilter(encode(school, 'eq', 'MUIS'))).toEqual([
+      { 'propertiesData.g:edu': { $elemMatch: { school: { $in: ['MUIS'] } } } },
+    ]);
+  });
+
+  it('makes both conditions hold for one shared entry', () => {
+    const filter = buildPropertyFilter(
+      [encode(school, 'eq', 'MUIS'), encode(year, 'gte', '2020')].join(';'),
+    );
+
+    expect(filter).toEqual([
+      {
+        'propertiesData.g:edu': {
+          $elemMatch: { school: { $in: ['MUIS'] }, year: { $gte: 2020 } },
+        },
+      },
+    ]);
+  });
+
+  it('lets any-row conditions stand on separate entries', () => {
+    const filter = buildPropertyFilter(
+      [
+        encode(toPropertyRowKey('edu', 'school', true), 'eq', 'MUIS'),
+        encode(toPropertyRowKey('edu', 'year', true), 'gte', '2020'),
+      ].join(';'),
+    );
+
+    expect(filter).toEqual([
+      { 'propertiesData.g:edu': { $elemMatch: { school: { $in: ['MUIS'] } } } },
+      { 'propertiesData.g:edu': { $elemMatch: { year: { $gte: 2020 } } } },
+    ]);
+  });
+
+  it('keeps entries of different groups apart', () => {
+    const filter = buildPropertyFilter(
+      [
+        encode(school, 'eq', 'MUIS'),
+        encode(toPropertyRowKey('work', 'company'), 'eq', 'erxes'),
+      ].join(';'),
+    );
+
+    expect(filter).toHaveLength(2);
+  });
+
+  it('keeps a value containing the separators intact', () => {
+    expect(buildPropertyFilter(encode('plan', 'contains', 'a:b/c'))).toEqual([
+      { 'propertiesData.plan': { $regex: 'a:b/c', $options: 'i' } },
+    ]);
+  });
+
+  it('reads nothing from an empty input', () => {
+    expect(buildPropertyFilter('')).toEqual([]);
+    expect(buildPropertyFilter(null)).toEqual([]);
+  });
+});

package/src/core-modules/properties/filter.ts

@@ -0,0 +1,156 @@
+import {
+  parsePropertyDataKey,
+  propertyDataPath,
+  toPropertyGroupKey,
+} from './keys';
+import { mergePropertyRowConditions } from './mergeRows';
+import {
+  isPropertyFilterOperator,
+  MULTI_VALUE_PROPERTY_OPERATORS,
+  PROPERTY_FILTER_OPERATOR_BY_TYPE,
+  PROPERTY_FILTER_OPERATORS,
+} from './operators';
+import {
+  IPropertyFilterCondition,
+  IPropertyFilterOptions,
+  PropertyFilterQuery,
+  PropertyFilterOperator,
+} from './types';
+
+// `fieldId:operator:value`, joined by `;`. Every segment is percent-encoded, so
+// a compound `g:<groupId>/<fieldId>` key survives the round trip.
+const CONDITION_SEP = ';';
+const PART_SEP = ':';
+const VALUE_SEP = ',';
+
+const decodeConditionValue = (
+  operator: PropertyFilterOperator,
+  raw: string,
+): unknown => {
+  if (MULTI_VALUE_PROPERTY_OPERATORS.includes(operator)) {
+    return raw
+      .split(VALUE_SEP)
+      .filter(Boolean)
+      .map((value) => decodeURIComponent(value));
+  }
+
+  return decodeURIComponent(raw);
+};
+
+export const parsePropertyFilterConditions = (
+  encoded?: string | null,
+): IPropertyFilterCondition[] => {
+  if (!encoded) {
+    return [];
+  }
+
+  return String(encoded)
+    .split(CONDITION_SEP)
+    .map((entry) => entry.trim())
+    .filter(Boolean)
+    .reduce<IPropertyFilterCondition[]>((conditions, entry) => {
+      const firstSep = entry.indexOf(PART_SEP);
+
+      if (firstSep === -1) {
+        return conditions;
+      }
+
+      const fieldId = decodeURIComponent(entry.slice(0, firstSep));
+      const rest = entry.slice(firstSep + 1);
+      const secondSep = rest.indexOf(PART_SEP);
+      const rawOperator = secondSep === -1 ? rest : rest.slice(0, secondSep);
+
+      if (!fieldId || !isPropertyFilterOperator(rawOperator)) {
+        return conditions;
+      }
+
+      const condition: IPropertyFilterCondition = {
+        fieldId,
+        operator: rawOperator,
+      };
+
+      if (secondSep !== -1) {
+        condition.value = decodeConditionValue(
+          rawOperator,
+          rest.slice(secondSep + 1),
+        );
+      }
+
+      conditions.push(condition);
+
+      return conditions;
+    }, []);
+};
+
+export const buildPropertyFilterConditions = (
+  conditions: IPropertyFilterCondition[],
+  options?: IPropertyFilterOptions,
+): PropertyFilterQuery[] => {
+  const { prefix } = options || {};
+
+  const built: PropertyFilterQuery[] = [];
+  const standalone = new Set<PropertyFilterQuery>();
+
+  for (const condition of conditions || []) {
+    const { fieldId, value, type = '' } = condition || {};
+
+    if (!condition || !fieldId) {
+      continue;
+    }
+
+    const operator =
+      condition.operator ||
+      PROPERTY_FILTER_OPERATOR_BY_TYPE[type] ||
+      'contains';
+
+    const build = PROPERTY_FILTER_OPERATORS[operator];
+
+    if (!build) {
+      continue;
+    }
+
+    const key = parsePropertyDataKey(fieldId);
+
+    if (key.kind === 'row') {
+      const inner = build(key.fieldId, value);
+
+      if (inner) {
+        const property = {
+          [propertyDataPath(toPropertyGroupKey(key.groupId), prefix)]: {
+            $elemMatch: inner,
+          },
+        };
+
+        if (key.anyRow) {
+          standalone.add(property);
+        }
+
+        built.push(property);
+      }
+
+      continue;
+    }
+
+    const property = build(propertyDataPath(key.fieldId, prefix), value);
+
+    if (property) {
+      built.push(property);
+    }
+  }
+
+  return mergePropertyRowConditions(
+    built,
+    (property) => !standalone.has(property),
+  );
+};
+
+export const buildPropertyFilter = (
+  input?: string | IPropertyFilterCondition[] | null,
+  options?: IPropertyFilterOptions,
+): PropertyFilterQuery[] =>
+  buildPropertyFilterConditions(
+    typeof input === 'string' || input == null
+      ? parsePropertyFilterConditions(input)
+      : input,
+    options,
+  );

package/src/core-modules/properties/importExport.ts

@@ -0,0 +1,159 @@
+import {
+  isPropertyGroupKey,
+  parsePropertyColumnKey,
+  ParsedPropertyColumnKey,
+  propertyDataPath,
+  propertyGroupIdFromKey,
+  toPropertyGroupKey,
+  toPropertyRowColumnKey,
+} from './keys';
+
+type TPropertyData = Record<string, unknown>;
+
+export const readPropertyDataColumn = (
+  propertiesData: TPropertyData | undefined,
+  column: ParsedPropertyColumnKey,
+): unknown => {
+  if (!propertiesData) {
+    return undefined;
+  }
+
+  if (column.kind === 'field') {
+    return propertiesData[column.fieldId];
+  }
+
+  const rows = propertiesData[toPropertyGroupKey(column.groupId)];
+
+  return Array.isArray(rows)
+    ? (rows[column.index - 1] as TPropertyData | undefined)?.[column.fieldId]
+    : undefined;
+};
+
+export const buildPropertyDataColumns = (
+  propertiesData: TPropertyData,
+  formatValue: (value: unknown) => string,
+): Record<string, string> => {
+  const columns: Record<string, string> = {};
+
+  for (const [key, value] of Object.entries(propertiesData || {})) {
+    if (value === undefined || value === null) {
+      continue;
+    }
+
+    if (!isPropertyGroupKey(key) || !Array.isArray(value)) {
+      columns[propertyDataPath(key)] = formatValue(value);
+      continue;
+    }
+
+    const groupId = propertyGroupIdFromKey(key);
+
+    value.forEach((row, offset) => {
+      for (const [fieldId, rowValue] of Object.entries(
+        (row || {}) as TPropertyData,
+      )) {
+        if (fieldId === '_id' || rowValue === undefined || rowValue === null) {
+          continue;
+        }
+
+        columns[toPropertyRowColumnKey(groupId, fieldId, offset + 1)] =
+          formatValue(rowValue);
+      }
+    });
+  }
+
+  return columns;
+};
+
+/** Reassembles numbered columns into rows. Consumes the columns it reads. */
+export const collectPropertyDataFromColumns = (
+  doc: Record<string, unknown>,
+): TPropertyData => {
+  const propertiesData: TPropertyData = {};
+  const rowsByGroup = new Map<string, Map<number, TPropertyData>>();
+
+  for (const key of Object.keys(doc)) {
+    const column = parsePropertyColumnKey(key);
+
+    if (!column) {
+      continue;
+    }
+
+    const value = doc[key];
+
+    delete doc[key];
+
+    if (value === undefined || value === null || value === '') {
+      continue;
+    }
+
+    if (column.kind === 'field') {
+      propertiesData[column.fieldId] = value;
+      continue;
+    }
+
+    const rows = rowsByGroup.get(column.groupId) || new Map();
+
+    rows.set(column.index, {
+      ...(rows.get(column.index) || {}),
+      [column.fieldId]: value,
+    });
+
+    rowsByGroup.set(column.groupId, rows);
+  }
+
+  for (const [groupId, rows] of rowsByGroup) {
+    const ordered = [...rows.entries()]
+      .sort(([left], [right]) => left - right)
+      .map(([, row]) => row);
+
+    if (ordered.length) {
+      propertiesData[toPropertyGroupKey(groupId)] = ordered;
+    }
+  }
+
+  return propertiesData;
+};
+
+/**
+ * Fold imported properties onto the ones a record already carries.
+ *
+ * An import row only holds the columns that particular file happened to
+ * include, so writing `propertiesData` wholesale would erase every property
+ * the file left out. Repeating groups merge position by position for the same
+ * reason: a file that fills row 2 must not blank out row 2's other fields, and
+ * rows the file never reached are kept as they were.
+ */
+export const mergePropertyData = (
+  existing: TPropertyData | undefined,
+  incoming: TPropertyData,
+): TPropertyData => {
+  const merged: TPropertyData = { ...(existing || {}) };
+
+  for (const [key, value] of Object.entries(incoming)) {
+    if (!isPropertyGroupKey(key) || !Array.isArray(value)) {
+      merged[key] = value;
+      continue;
+    }
+
+    const existingRows = Array.isArray(merged[key])
+      ? (merged[key] as TPropertyData[])
+      : [];
+
+    merged[key] = [
+      ...value.map((row, offset) => {
+        const existingRow = existingRows[offset];
+
+        return {
+          ...(existingRow || {}),
+          ...((row || {}) as TPropertyData),
+          // A row keeps the identity it was created with; an imported row
+          // carries no id of its own and must not mint a new one.
+          ...(existingRow?._id ? { _id: existingRow._id } : {}),
+        };
+      }),
+      ...existingRows.slice(value.length),
+    ];
+  }
+
+  return merged;
+};

package/src/core-modules/properties/index.ts

@@ -0,0 +1,9 @@
+export * from './filter';
+export * from './importExport';
+export * from './keys';
+export * from './mergeRows';
+export * from './operators';
+export * from './options';
+export * from './setupPropertyProducers';
+export * from './types';
+export * from './valueUsage';

package/src/core-modules/properties/keys.test.ts

@@ -0,0 +1,71 @@
+import {
+  parsePropertyColumnKey,
+  parsePropertyDataKey,
+  toPropertyGroupKey,
+  toPropertyRowColumnKey,
+  toPropertyRowKey,
+} from './keys';
+
+// nanoid may contain `_` and `-`, so ids are exercised with both
+const GROUP = 'V1StGXR8_Z';
+const FIELD = 'Gm3K8ni3-r2i';
+
+describe('property data keys', () => {
+  it('round trips a row key', () => {
+    expect(parsePropertyDataKey(toPropertyRowKey(GROUP, FIELD))).toEqual({
+      kind: 'row',
+      groupId: GROUP,
+      fieldId: FIELD,
+      anyRow: false,
+    });
+  });
+
+  it('round trips an any-row key', () => {
+    expect(parsePropertyDataKey(toPropertyRowKey(GROUP, FIELD, true))).toEqual({
+      kind: 'row',
+      groupId: GROUP,
+      fieldId: FIELD,
+      anyRow: true,
+    });
+  });
+
+  it('treats a bare group key as a plain field', () => {
+    expect(parsePropertyDataKey(toPropertyGroupKey(GROUP))).toEqual({
+      kind: 'field',
+      fieldId: toPropertyGroupKey(GROUP),
+    });
+  });
+
+  it('round trips a numbered column key', () => {
+    expect(
+      parsePropertyColumnKey(toPropertyRowColumnKey(GROUP, FIELD, 3)),
+    ).toEqual({
+      kind: 'row',
+      groupId: GROUP,
+      fieldId: FIELD,
+      index: 3,
+    });
+  });
+
+  it('defaults an unnumbered row column to the first entry', () => {
+    expect(
+      parsePropertyColumnKey(`propertiesData.g:${GROUP}/${FIELD}`),
+    ).toEqual({
+      kind: 'row',
+      groupId: GROUP,
+      fieldId: FIELD,
+      index: 1,
+    });
+  });
+
+  it('reads a plain column', () => {
+    expect(parsePropertyColumnKey(`propertiesData.${FIELD}`)).toEqual({
+      kind: 'field',
+      fieldId: FIELD,
+    });
+  });
+
+  it('ignores a column that is not property data', () => {
+    expect(parsePropertyColumnKey('tagIds')).toBeNull();
+  });
+});

package/src/core-modules/properties/keys.ts

@@ -0,0 +1,128 @@
+// A `propertiesData` key is a bare field id or a prefixed group id. `:`, `/`
+// and `~` never occur in a nanoid, so no split can hit an id.
+export const PROPERTIES_DATA = 'propertiesData';
+
+export const PROPERTY_GROUP_KEY_PREFIX = 'g:';
+
+export const PROPERTY_ROW_KEY_SEP = '/';
+
+// Same key, but the condition stands on its own row instead of joining its
+// siblings in one $elemMatch.
+export const PROPERTY_ANY_ROW_KEY_SEP = '~';
+
+export const toPropertyGroupKey = (groupId: string) =>
+  `${PROPERTY_GROUP_KEY_PREFIX}${groupId}`;
+
+export const isPropertyGroupKey = (key: string) =>
+  key.startsWith(PROPERTY_GROUP_KEY_PREFIX);
+
+export const propertyGroupIdFromKey = (key: string) =>
+  key.slice(PROPERTY_GROUP_KEY_PREFIX.length);
+
+export const propertyDataPath = (key: string, prefix = PROPERTIES_DATA) =>
+  `${prefix}.${key}`;
+
+export const toPropertyRowKey = (
+  groupId: string,
+  fieldId: string,
+  anyRow = false,
+) =>
+  `${toPropertyGroupKey(groupId)}${
+    anyRow ? PROPERTY_ANY_ROW_KEY_SEP : PROPERTY_ROW_KEY_SEP
+  }${fieldId}`;
+
+export type ParsedPropertyDataKey =
+  | { kind: 'field'; fieldId: string }
+  | { kind: 'row'; groupId: string; fieldId: string; anyRow: boolean };
+
+export const parsePropertyDataKey = (key: string): ParsedPropertyDataKey => {
+  if (!isPropertyGroupKey(key)) {
+    return { kind: 'field', fieldId: key };
+  }
+
+  const rest = propertyGroupIdFromKey(key);
+  const sameRowSep = rest.indexOf(PROPERTY_ROW_KEY_SEP);
+  const anyRowSep = rest.indexOf(PROPERTY_ANY_ROW_KEY_SEP);
+
+  const sep =
+    sameRowSep === -1 || (anyRowSep !== -1 && anyRowSep < sameRowSep)
+      ? anyRowSep
+      : sameRowSep;
+
+  if (sep === -1) {
+    return { kind: 'field', fieldId: key };
+  }
+
+  return {
+    kind: 'row',
+    groupId: rest.slice(0, sep),
+    fieldId: rest.slice(sep + 1),
+    anyRow: rest[sep] === PROPERTY_ANY_ROW_KEY_SEP,
+  };
+};
+
+export const isPropertyDataPath = (field: string, prefix = PROPERTIES_DATA) =>
+  field.includes(`${prefix}.`);
+
+export const fieldIdFromPropertyDataPath = (
+  field: string,
+  prefix = PROPERTIES_DATA,
+) => field.replace(`${prefix}.`, '');
+
+export const propertyDataExistsFilter = (
+  fieldIds: string[],
+  prefix = PROPERTIES_DATA,
+) => ({
+  $or: fieldIds.map((fieldId) => ({
+    [propertyDataPath(fieldId, prefix)]: { $exists: true },
+  })),
+});
+
+export const propertyDataRegexFilter = (
+  fieldId: string,
+  regex: RegExp,
+  prefix = PROPERTIES_DATA,
+) => ({ [propertyDataPath(fieldId, prefix)]: { $regex: regex } });
+
+// A row cannot fit one spreadsheet cell, so each gets a numbered column.
+export const PROPERTY_ROW_INDEX_SEP = '#';
+
+export const toPropertyRowColumnKey = (
+  groupId: string,
+  fieldId: string,
+  index: number,
+) =>
+  `${propertyDataPath(
+    toPropertyRowKey(groupId, fieldId),
+  )}${PROPERTY_ROW_INDEX_SEP}${index}`;
+
+export type ParsedPropertyColumnKey =
+  | { kind: 'field'; fieldId: string }
+  | { kind: 'row'; groupId: string; fieldId: string; index: number };
+
+export const parsePropertyColumnKey = (
+  columnKey: string,
+): ParsedPropertyColumnKey | null => {
+  if (!columnKey.startsWith(`${PROPERTIES_DATA}.`)) {
+    return null;
+  }
+
+  const rest = columnKey.slice(PROPERTIES_DATA.length + 1);
+  const sep = rest.lastIndexOf(PROPERTY_ROW_INDEX_SEP);
+  const index = sep === -1 ? NaN : Number(rest.slice(sep + 1));
+
+  const key = parsePropertyDataKey(
+    Number.isInteger(index) && index > 0 ? rest.slice(0, sep) : rest,
+  );
+
+  if (key.kind !== 'row') {
+    return { kind: 'field', fieldId: key.fieldId };
+  }
+
+  return {
+    kind: 'row',
+    groupId: key.groupId,
+    fieldId: key.fieldId,
+    index: Number.isInteger(index) && index > 0 ? index : 1,
+  };
+};

package/src/core-modules/properties/mergePropertyData.test.ts

@@ -0,0 +1,65 @@
+import { mergePropertyData } from './importExport';
+import { toPropertyGroupKey } from './keys';
+
+const GROUP = toPropertyGroupKey('grp1');
+
+describe('mergePropertyData', () => {
+  it('keeps properties the import did not mention', () => {
+    const merged = mergePropertyData(
+      { loyalty: 'Gold', size: '42' },
+      { size: '44' },
+    );
+
+    expect(merged).toEqual({ loyalty: 'Gold', size: '44' });
+  });
+
+  it('starts from nothing when the record has no properties yet', () => {
+    expect(mergePropertyData(undefined, { loyalty: 'Gold' })).toEqual({
+      loyalty: 'Gold',
+    });
+  });
+
+  it('merges a repeating row field by field', () => {
+    const merged = mergePropertyData(
+      { [GROUP]: [{ _id: 'row1', street: 'Old', city: 'UB' }] },
+      { [GROUP]: [{ street: 'New' }] },
+    );
+
+    expect(merged[GROUP]).toEqual([{ _id: 'row1', street: 'New', city: 'UB' }]);
+  });
+
+  it('keeps rows the import did not reach', () => {
+    const merged = mergePropertyData(
+      {
+        [GROUP]: [
+          { _id: 'row1', street: 'First' },
+          { _id: 'row2', street: 'Second' },
+        ],
+      },
+      { [GROUP]: [{ street: 'Updated' }] },
+    );
+
+    expect(merged[GROUP]).toEqual([
+      { _id: 'row1', street: 'Updated' },
+      { _id: 'row2', street: 'Second' },
+    ]);
+  });
+
+  it('appends rows beyond what the record had', () => {
+    const merged = mergePropertyData(
+      { [GROUP]: [{ _id: 'row1', street: 'First' }] },
+      { [GROUP]: [{ street: 'First' }, { _id: 'fresh', street: 'Second' }] },
+    );
+
+    expect(merged[GROUP]).toEqual([
+      { _id: 'row1', street: 'First' },
+      { _id: 'fresh', street: 'Second' },
+    ]);
+  });
+
+  it('replaces a plain value rather than merging into it', () => {
+    const merged = mergePropertyData({ tags: ['a', 'b'] }, { tags: ['c'] });
+
+    expect(merged.tags).toEqual(['c']);
+  });
+});