@erxes/api-shared 0.0.0-stage → 3.2.19-2

package/src/core-modules/properties/mergeRows.ts

@@ -0,0 +1,93 @@
+import { PropertyFilterQuery } from './types';
+
+const isPlainObject = (value: unknown): value is Record<string, unknown> =>
+  !!value && typeof value === 'object' && !Array.isArray(value);
+
+type RowClause = { path: string; inner: Record<string, unknown> };
+
+const asRowClause = (condition: PropertyFilterQuery): RowClause | null => {
+  const keys = Object.keys(condition);
+
+  if (keys.length !== 1) {
+    return null;
+  }
+
+  const [path] = keys;
+  const value = condition[path];
+
+  if (!isPlainObject(value) || Object.keys(value).length !== 1) {
+    return null;
+  }
+
+  return isPlainObject(value.$elemMatch)
+    ? { path, inner: value.$elemMatch }
+    : null;
+};
+
+const canAbsorb = (
+  open: Record<string, unknown>,
+  inner: Record<string, unknown>,
+) =>
+  Object.entries(inner).every(([leaf, value]) => {
+    if (!(leaf in open)) {
+      return true;
+    }
+
+    const current = open[leaf];
+
+    return (
+      isPlainObject(current) &&
+      isPlainObject(value) &&
+      Object.keys(value).every((operator) => !(operator in current))
+    );
+  });
+
+const absorb = (
+  open: Record<string, unknown>,
+  inner: Record<string, unknown>,
+) => {
+  for (const [leaf, value] of Object.entries(inner)) {
+    const current = open[leaf];
+
+    open[leaf] =
+      isPlainObject(current) && isPlainObject(value)
+        ? { ...current, ...value }
+        : value;
+  }
+};
+
+/**
+ * Folds same-group conditions into one `$elemMatch`, so they hold for the same
+ * entry rather than for any two. Only safe for conditions joined by AND;
+ * `canMerge` opts a condition out and leaves it on its own entry.
+ */
+export const mergePropertyRowConditions = <T extends PropertyFilterQuery>(
+  conditions: T[],
+  canMerge: (condition: T) => boolean = () => true,
+): T[] => {
+  const merged: T[] = [];
+  const openByPath = new Map<string, Record<string, unknown>>();
+
+  for (const condition of conditions) {
+    const clause = canMerge(condition) ? asRowClause(condition) : null;
+
+    if (!clause) {
+      merged.push(condition);
+      continue;
+    }
+
+    const open = openByPath.get(clause.path);
+
+    if (open && canAbsorb(open, clause.inner)) {
+      absorb(open, clause.inner);
+      continue;
+    }
+
+    const inner = { ...clause.inner };
+
+    openByPath.set(clause.path, inner);
+    merged.push({ [clause.path]: { $elemMatch: inner } } as T);
+  }
+
+  return merged;
+};

package/src/core-modules/properties/operators.ts

@@ -0,0 +1,91 @@
+import { escapeRegExp } from '../../utils';
+import { PropertyFilterQuery, PropertyFilterOperator } from './types';
+
+const isEmpty = (value: unknown) =>
+  value === undefined || value === null || value === '';
+
+const toArray = (value: unknown): unknown[] =>
+  Array.isArray(value) ? value : isEmpty(value) ? [] : [value];
+
+const regex = (value: string) => ({
+  $regex: escapeRegExp(value),
+  $options: 'i',
+});
+
+const fileTypes = (value: unknown): string[] =>
+  toArray(value)
+    .map((item) => String(item ?? '').trim())
+    .filter(Boolean);
+
+const asNumberOrString = (value: unknown): number | string => {
+  const text = String(value ?? '');
+
+  return text !== '' && !isNaN(Number(text)) ? Number(text) : text;
+};
+
+const eqValues = (value: unknown): unknown[] => {
+  const normalized = asNumberOrString(value);
+
+  return typeof normalized === 'number'
+    ? [normalized, String(value)]
+    : [normalized];
+};
+
+export const PROPERTY_FILTER_OPERATORS: Record<
+  PropertyFilterOperator,
+  (path: string, value: unknown) => PropertyFilterQuery | null
+> = {
+  eq: (path, value) => ({ [path]: { $in: eqValues(value) } }),
+  ne: (path, value) => ({ [path]: { $nin: eqValues(value) } }),
+  gt: (path, value) => ({ [path]: { $gt: asNumberOrString(value) } }),
+  gte: (path, value) => ({ [path]: { $gte: asNumberOrString(value) } }),
+  lt: (path, value) => ({ [path]: { $lt: asNumberOrString(value) } }),
+  lte: (path, value) => ({ [path]: { $lte: asNumberOrString(value) } }),
+  isTrue: (path) => ({ [path]: { $in: [true, 'true', 'Yes', 'yes'] } }),
+  isFalse: (path) => ({ [path]: { $in: [false, 'false', 'No', 'no', null] } }),
+  isSet: (path) => ({ [path]: { $exists: true, $nin: [null, '', []] } }),
+  isNotSet: (path) => ({
+    $or: [{ [path]: { $exists: false } }, { [path]: { $in: [null, ''] } }],
+  }),
+  contains: (path, value) =>
+    isEmpty(value) ? null : { [path]: regex(String(value)) },
+  doesNotContain: (path, value) =>
+    isEmpty(value) ? null : { [path]: { $not: regex(String(value)) } },
+  in: (path, value) =>
+    toArray(value).length ? { [path]: { $in: toArray(value) } } : null,
+  notIn: (path, value) =>
+    toArray(value).length ? { [path]: { $nin: toArray(value) } } : null,
+  fileType: (path, value) => {
+    const types = fileTypes(value);
+
+    return types.length
+      ? {
+          [`${path}.type`]: {
+            $regex: types.map(escapeRegExp).join('|'),
+            $options: 'i',
+          },
+        }
+      : null;
+  },
+};
+
+export const PROPERTY_FILTER_OPERATOR_BY_TYPE: Record<
+  string,
+  PropertyFilterOperator
+> = {
+  boolean: 'isTrue',
+  number: 'eq',
+  date: 'eq',
+  multiSelect: 'in',
+  check: 'in',
+};
+
+export const isPropertyFilterOperator = (
+  operator: string,
+): operator is PropertyFilterOperator => operator in PROPERTY_FILTER_OPERATORS;
+
+export const MULTI_VALUE_PROPERTY_OPERATORS: PropertyFilterOperator[] = [
+  'in',
+  'notIn',
+  'fileType',
+];

package/src/core-modules/properties/options.test.ts

@@ -0,0 +1,119 @@
+import { SegmentNode } from '../segments/nodes';
+import { reconcilePropertyOptions, segmentFiltersByOption } from './options';
+
+const option = (value: string, deprecated?: boolean) => ({
+  label: value.toUpperCase(),
+  value,
+  ...(deprecated === undefined ? {} : { deprecated }),
+});
+
+describe('reconcilePropertyOptions', () => {
+  const stored = [option('a'), option('b'), option('c')];
+
+  it('keeps what the form sends, label edits included', () => {
+    const incoming = [
+      { label: 'Renamed', value: 'a' },
+      option('b'),
+      option('c'),
+    ];
+
+    expect(reconcilePropertyOptions(stored, incoming, [])).toEqual(incoming);
+  });
+
+  it('drops a removed option no record holds', () => {
+    const result = reconcilePropertyOptions(
+      stored,
+      [option('a'), option('b')],
+      [{ value: 'a', count: 4 }],
+    );
+
+    expect(result.map(({ value }) => value)).toEqual(['a', 'b']);
+  });
+
+  it('archives a removed option a record still holds', () => {
+    const result = reconcilePropertyOptions(
+      stored,
+      [option('a')],
+      [
+        { value: 'b', count: 2 },
+        { value: 'c', count: 0 },
+      ],
+    );
+
+    expect(result).toEqual([option('a'), option('b', true)]);
+  });
+
+  it('archives every removed option when they could not be counted', () => {
+    const result = reconcilePropertyOptions(stored, [option('a')], null);
+
+    expect(result).toEqual([option('a'), option('b', true), option('c', true)]);
+  });
+
+  it('adds a new option as given', () => {
+    const result = reconcilePropertyOptions(
+      stored,
+      [...stored, option('d')],
+      [],
+    );
+
+    expect(result.map(({ value }) => value)).toEqual(['a', 'b', 'c', 'd']);
+  });
+});
+
+describe('segmentFiltersByOption', () => {
+  const field = (fieldKey: string, value: string | string[]) =>
+    ({
+      kind: 'field',
+      contentType: 'core:contacts.customers',
+      fieldKey,
+      operator: 'in',
+      value,
+    }) as SegmentNode;
+
+  const group = (...children: SegmentNode[]): SegmentNode => ({
+    kind: 'group',
+    conjunction: 'and',
+    children,
+  });
+
+  it('finds an option named in a nested condition', () => {
+    const root = group(
+      field('state', 'customer'),
+      group(field('propertiesData.f1', ['o1', 'o3'])),
+    );
+
+    expect(segmentFiltersByOption(root, 'f1', 'o3')).toBe(true);
+    expect(segmentFiltersByOption(root, 'f1', 'o2')).toBe(false);
+  });
+
+  it('matches a single value and a repeating group row', () => {
+    expect(
+      segmentFiltersByOption(
+        group(field('propertiesData.f1', 'o1')),
+        'f1',
+        'o1',
+      ),
+    ).toBe(true);
+    expect(
+      segmentFiltersByOption(
+        group(field('propertiesData.g:edu/f1', 'o1')),
+        'f1',
+        'o1',
+      ),
+    ).toBe(true);
+  });
+
+  it('ignores another field holding the same value', () => {
+    expect(
+      segmentFiltersByOption(
+        group(field('propertiesData.f2', 'o1')),
+        'f1',
+        'o1',
+      ),
+    ).toBe(false);
+  });
+
+  it('passes over a segment carried over without conditions', () => {
+    expect(segmentFiltersByOption(undefined, 'f1', 'o1')).toBe(false);
+  });
+});

package/src/core-modules/properties/options.ts

@@ -0,0 +1,55 @@
+import { SegmentNode } from '../segments/nodes';
+import { walkSegmentNodes } from '../segments/walkNodes';
+import { parsePropertyDataKey, PROPERTIES_DATA } from './keys';
+import { IPropertyValueCounts } from './valueUsage';
+
+type TOption = { value: string; deprecated?: boolean | null };
+
+// An option a record still holds is archived; one nobody chose simply goes.
+export const reconcilePropertyOptions = <T extends TOption>(
+  stored: T[] = [],
+  incoming: T[] = [],
+  // null when it could not be counted; every dropped option is then kept.
+  byOption: IPropertyValueCounts['byOption'],
+): T[] => {
+  const incomingValues = new Set(incoming.map(({ value }) => value));
+  const heldValues = new Set(
+    (byOption || []).filter(({ count }) => count > 0).map(({ value }) => value),
+  );
+
+  const archived = stored
+    .filter(({ value }) => !incomingValues.has(value))
+    .filter(({ value }) => !byOption || heldValues.has(value))
+    .map((option) => ({ ...option, deprecated: true }));
+
+  return [...incoming, ...archived];
+};
+
+const PREFIX = `${PROPERTIES_DATA}.`;
+
+// Whether a segment's conditions name one option of a property, at any depth
+// and on a plain field or a repeating group's row alike.
+export const segmentFiltersByOption = (
+  root: SegmentNode | null | undefined,
+  fieldId: string,
+  value: string,
+) => {
+  if (!root) {
+    return false;
+  }
+
+  for (const node of walkSegmentNodes(root)) {
+    if (node.kind !== 'field' || !node.fieldKey.startsWith(PREFIX)) {
+      continue;
+    }
+
+    const key = parsePropertyDataKey(node.fieldKey.slice(PREFIX.length));
+    const values = Array.isArray(node.value) ? node.value : [node.value];
+
+    if (key.fieldId === fieldId && values.map(String).includes(value)) {
+      return true;
+    }
+  }
+
+  return false;
+};

package/src/core-modules/properties/setupPropertyProducers.ts

@@ -0,0 +1,32 @@
+import { initTRPC } from '@trpc/server';
+import * as trpcExpress from '@trpc/server/adapters/express';
+import { Express } from 'express';
+import { z } from 'zod';
+import { createTRPCContext } from '../../utils';
+import {
+  PropertyValueUsageInput,
+  TPropertyProducers,
+  TPropertyValueUsageHandler,
+} from './valueUsage';
+
+// Records live in the plugin, so core asks it which of them hold a value.
+export const initPropertyProducers = (
+  app: Express,
+  valueUsage: TPropertyValueUsageHandler,
+) => {
+  const t = initTRPC.context<{ subdomain: string }>().create();
+
+  const router = t.router({
+    [TPropertyProducers.VALUE_USAGE]: t.procedure
+      .input(z.object({ subdomain: z.string(), data: PropertyValueUsageInput }))
+      .query(({ input }) => valueUsage(input)),
+  });
+
+  app.use(
+    '/properties',
+    trpcExpress.createExpressMiddleware({
+      router,
+      createContext: createTRPCContext(async (_subdomain, context) => context),
+    }),
+  );
+};

package/src/core-modules/properties/types.ts

@@ -0,0 +1,31 @@
+export type PropertyFilterOperator =
+  | 'eq'
+  | 'ne'
+  | 'contains'
+  | 'doesNotContain'
+  | 'gt'
+  | 'gte'
+  | 'lt'
+  | 'lte'
+  | 'isTrue'
+  | 'isFalse'
+  | 'in'
+  | 'notIn'
+  | 'isSet'
+  | 'isNotSet'
+  | 'fileType';
+
+export interface IPropertyFilterCondition {
+  /** Either `<fieldId>` or `g:<groupId>/<fieldId>` for a repeating group. */
+  fieldId: string;
+  type?: string;
+  operator?: PropertyFilterOperator;
+  value?: unknown;
+}
+
+export interface IPropertyFilterOptions {
+  /** Defaults to `propertiesData`. */
+  prefix?: string;
+}
+
+export type PropertyFilterQuery = Record<string, unknown>;

package/src/core-modules/properties/valueUsage.test.ts

@@ -0,0 +1,158 @@
+import { Collection } from 'mongoose';
+import {
+  measurePropertyValueCounts,
+  measurePropertyValueSamples,
+  measurePropertyValueUsage,
+  TPropertyValueUsageInput,
+  UNKNOWN_VALUE_SAMPLES,
+} from './valueUsage';
+
+const timeout = () =>
+  Promise.reject(new Error('operation exceeded time limit'));
+
+// Only what the helpers call; each query can be made to give up on its own.
+const fakeCollection = ({
+  docs = [] as Record<string, unknown>[],
+  count = 0 as number | (() => Promise<number>),
+  options = [] as { _id: unknown; count: number }[] | (() => Promise<never>),
+} = {}) => {
+  const collection = {
+    find: jest.fn(() => ({ toArray: async () => docs })),
+    countDocuments: jest.fn(() =>
+      typeof count === 'function' ? count() : Promise.resolve(count),
+    ),
+    aggregate: jest.fn(() => ({
+      toArray: () =>
+        typeof options === 'function' ? options() : Promise.resolve(options),
+    })),
+  };
+
+  return { collection, typed: collection as unknown as Collection };
+};
+
+const input = (
+  overrides: Partial<TPropertyValueUsageInput> = {},
+): TPropertyValueUsageInput => ({
+  contentType: 'core:customer',
+  path: 'propertiesData.f1',
+  part: 'samples',
+  ...overrides,
+});
+
+const label = (doc: Record<string, unknown>) =>
+  typeof doc.name === 'string' ? doc.name : '';
+
+describe('measurePropertyValueSamples', () => {
+  it('labels the first records holding a value', async () => {
+    const { collection, typed } = fakeCollection({
+      docs: [{ _id: 'a', name: 'Bold' }, { _id: 'b' }],
+    });
+
+    const result = await measurePropertyValueSamples(typed, input(), label);
+
+    expect(result).toEqual({
+      known: true,
+      // a record without a name still shows, by its id
+      samples: [
+        { _id: 'a', label: 'Bold' },
+        { _id: 'b', label: 'b' },
+      ],
+    });
+    expect(collection.find).toHaveBeenCalledWith(
+      { 'propertiesData.f1': { $exists: true } },
+      expect.objectContaining({ limit: 20 }),
+    );
+  });
+
+  it('narrows to one option when a value is given', async () => {
+    const { collection, typed } = fakeCollection();
+
+    await measurePropertyValueSamples(typed, input({ value: 'o1' }), label);
+
+    expect(collection.find).toHaveBeenCalledWith(
+      { 'propertiesData.f1': 'o1' },
+      expect.anything(),
+    );
+  });
+
+  it('reads as unknown when the scan gives up', async () => {
+    const { collection, typed } = fakeCollection();
+    collection.find.mockReturnValue({ toArray: timeout } as never);
+
+    expect(await measurePropertyValueSamples(typed, input(), label)).toBe(
+      UNKNOWN_VALUE_SAMPLES,
+    );
+  });
+});
+
+describe('measurePropertyValueCounts', () => {
+  const counts = input({ part: 'counts', optionValues: ['o1', 'o2'] });
+
+  it('counts the records and each option', async () => {
+    const { typed } = fakeCollection({
+      count: 3,
+      options: [{ _id: 'o1', count: 2 }],
+    });
+
+    expect(await measurePropertyValueCounts(typed, counts)).toEqual({
+      known: true,
+      count: 3,
+      capped: false,
+      byOption: [{ value: 'o1', count: 2 }],
+    });
+  });
+
+  it('marks a count at the cap as capped', async () => {
+    const { typed } = fakeCollection({ count: 1000 });
+
+    expect(await measurePropertyValueCounts(typed, counts)).toMatchObject({
+      count: 1000,
+      capped: true,
+    });
+  });
+
+  it('keeps the total when the per-option tally gives up', async () => {
+    const { typed } = fakeCollection({ count: 3, options: timeout });
+
+    expect(await measurePropertyValueCounts(typed, counts)).toEqual({
+      known: true,
+      count: 3,
+      capped: false,
+      byOption: null,
+    });
+  });
+
+  it('keeps the per-option tally when the total gives up', async () => {
+    const { typed } = fakeCollection({
+      count: timeout,
+      options: [{ _id: 'o2', count: 1 }],
+    });
+
+    expect(await measurePropertyValueCounts(typed, counts)).toEqual({
+      known: false,
+      count: 0,
+      capped: false,
+      byOption: [{ value: 'o2', count: 1 }],
+    });
+  });
+
+  it('skips the tally for a field without options or for one option', async () => {
+    const { collection, typed } = fakeCollection({ count: 1 });
+
+    await measurePropertyValueCounts(typed, input({ part: 'counts' }));
+    await measurePropertyValueCounts(typed, { ...counts, value: 'o1' });
+
+    expect(collection.aggregate).not.toHaveBeenCalled();
+  });
+});
+
+describe('measurePropertyValueUsage', () => {
+  it('answers only the part it was asked for', async () => {
+    const { collection, typed } = fakeCollection({ count: 2 });
+
+    await measurePropertyValueUsage(typed, input({ part: 'counts' }), label);
+
+    expect(collection.find).not.toHaveBeenCalled();
+    expect(collection.countDocuments).toHaveBeenCalled();
+  });
+});