@erxes/api-shared 0.0.0-stage → 3.2.19-1

package/src/core-modules/segments/mongoFilter.ts

@@ -0,0 +1,355 @@
+import { parsePropertyDataKey, toPropertyGroupKey } from '../properties/keys';
+import { mergePropertyRowConditions } from '../properties/mergeRows';
+import {
+  PERIOD_OPERATORS,
+  SegmentOperator,
+  normalizeSegmentOperator,
+} from './operators';
+import { SegmentFieldMeta, SegmentFieldNamespace } from './fieldMeta';
+import {
+  SEGMENT_MEMBERSHIP_FIELD,
+  SegmentFieldNode,
+  SegmentNode,
+  SegmentValue,
+} from './nodes';
+import {
+  anniversaryRanges,
+  DEFAULT_SEGMENT_TIME_ZONE,
+  shiftZonedDays,
+  zonedDate,
+  zonedDayStart,
+  zonedPeriodRange,
+} from './zonedTime';
+
+export type SegmentMongoFilter = Record<string, unknown>;
+
+export type SegmentCompileContext = {
+  fields: SegmentFieldMeta[];
+  namespaces?: SegmentFieldNamespace[];
+  now?: Date;
+  timeZone?: string;
+};
+
+export type SegmentCompileResult = {
+  filter: SegmentMongoFilter;
+  unsupported: string[];
+};
+
+const MINUTE_MS = 60_000;
+
+const escapeRegex = (value: string) =>
+  value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+const toArray = (value: SegmentValue): SegmentValue[] =>
+  Array.isArray(value) ? value : [value];
+
+const toNumber = (value: SegmentValue | undefined): number | undefined => {
+  if (value === undefined) {
+    return undefined;
+  }
+
+  const parsed = Number(String(value).replace(/,/g, ''));
+
+  return Number.isNaN(parsed) ? undefined : parsed;
+};
+
+const toDate = (value: SegmentValue): Date | undefined => {
+  const parsed = value instanceof Date ? value : new Date(String(value));
+
+  return Number.isNaN(parsed.getTime()) ? undefined : parsed;
+};
+
+const minuteBucket = (
+  value: SegmentValue,
+  now: Date,
+  offset: 1 | -1,
+): Record<string, Date> | undefined => {
+  const amount = toNumber(value);
+
+  if (amount === undefined) {
+    return undefined;
+  }
+
+  const target =
+    Math.floor((now.getTime() + offset * amount * MINUTE_MS) / MINUTE_MS) *
+    MINUTE_MS;
+
+  return { $gte: new Date(target), $lte: new Date(target + MINUTE_MS - 1) };
+};
+
+const dayBucket = (
+  value: SegmentValue,
+  now: Date,
+  timeZone: string,
+  offset: 1 | -1,
+): Record<string, Date> | undefined => {
+  const amount = toNumber(value);
+
+  if (amount === undefined) {
+    return undefined;
+  }
+
+  const day = shiftZonedDays(zonedDate(now, timeZone), offset * amount);
+
+  return {
+    $gte: zonedDayStart(day, timeZone),
+    $lt: zonedDayStart(shiftZonedDays(day, 1), timeZone),
+  };
+};
+
+const anniversaryOn = (
+  path: string,
+  value: SegmentValue | undefined,
+  now: Date,
+  timeZone: string,
+  offset: 0 | 1 | -1,
+): SegmentMongoFilter | undefined => {
+  const amount = offset === 0 ? 0 : toNumber(value);
+
+  if (amount === undefined) {
+    return undefined;
+  }
+
+  const target = shiftZonedDays(zonedDate(now, timeZone), offset * amount);
+
+  return {
+    $or: anniversaryRanges(target, timeZone).map((range) => ({
+      [path]: { $gte: range.gte, $lt: range.lt },
+    })),
+  };
+};
+
+const ANNIVERSARY_OFFSETS: Partial<Record<SegmentOperator, 0 | 1 | -1>> = {
+  [SegmentOperator.AnniversaryToday]: 0,
+  [SegmentOperator.AnniversaryFromNow]: 1,
+  [SegmentOperator.AnniversaryAgo]: -1,
+};
+
+const compareOn = (
+  operator: SegmentOperator,
+  value: SegmentValue | undefined,
+  now: Date,
+  timeZone: string,
+): Record<string, unknown> | undefined => {
+  switch (operator) {
+    case SegmentOperator.IsSet:
+      return { $nin: [null, []], $exists: true };
+    case SegmentOperator.IsNotSet:
+      return { $in: [null, []] };
+    case SegmentOperator.IsTrue:
+      return { $in: [true, 'true'] };
+    case SegmentOperator.IsFalse:
+      return { $in: [false, 'false'] };
+    default:
+      break;
+  }
+
+  const period = PERIOD_OPERATORS[operator];
+
+  if (period) {
+    const { gte, lt } = zonedPeriodRange(now, timeZone, period);
+
+    return { $gte: gte, $lt: lt };
+  }
+
+  if (value === undefined) {
+    return undefined;
+  }
+
+  switch (operator) {
+    case SegmentOperator.Equals:
+      return { $eq: value };
+    case SegmentOperator.NotEquals:
+      return { $ne: value };
+    case SegmentOperator.Contains:
+      return { $regex: escapeRegex(String(value)), $options: 'i' };
+    case SegmentOperator.NotContains:
+      return {
+        $not: new RegExp(escapeRegex(String(value)), 'i'),
+      };
+
+    case SegmentOperator.In:
+      return { $in: toArray(value) };
+    case SegmentOperator.NotIn:
+      return { $nin: toArray(value) };
+
+    case SegmentOperator.NumberGt: {
+      const parsed = toNumber(value);
+      return parsed === undefined ? undefined : { $gte: parsed };
+    }
+    case SegmentOperator.NumberLt: {
+      const parsed = toNumber(value);
+      return parsed === undefined ? undefined : { $lte: parsed };
+    }
+    case SegmentOperator.DateGte: {
+      const parsed = toDate(value);
+      return parsed === undefined ? undefined : { $gte: parsed };
+    }
+    case SegmentOperator.DateLte: {
+      const parsed = toDate(value);
+      return parsed === undefined ? undefined : { $lte: parsed };
+    }
+
+    case SegmentOperator.MinutesFromNow:
+      return minuteBucket(value, now, 1);
+    case SegmentOperator.MinutesAgo:
+      return minuteBucket(value, now, -1);
+    case SegmentOperator.DaysFromNow:
+      return dayBucket(value, now, timeZone, 1);
+    case SegmentOperator.DaysAgo:
+      return dayBucket(value, now, timeZone, -1);
+
+    default:
+      return undefined;
+  }
+};
+
+const compileField = (
+  node: SegmentFieldNode,
+  context: SegmentCompileContext,
+  now: Date,
+): SegmentMongoFilter | undefined => {
+  const operator = normalizeSegmentOperator(node.operator);
+  const timeZone = context.timeZone || DEFAULT_SEGMENT_TIME_ZONE;
+  const anniversary = ANNIVERSARY_OFFSETS[operator];
+
+  const declared = context.fields.find((field) => field.key === node.fieldKey);
+
+  if (declared) {
+    if (declared.kind !== 'projected') {
+      return undefined;
+    }
+
+    if (anniversary !== undefined) {
+      return anniversaryOn(
+        declared.path,
+        node.value,
+        now,
+        timeZone,
+        anniversary,
+      );
+    }
+
+    const comparison = compareOn(operator, node.value, now, timeZone);
+
+    return comparison ? { [declared.path]: comparison } : undefined;
+  }
+
+  const [prefix, ...rest] = node.fieldKey.split('.');
+  const namespace = (context.namespaces || []).find(
+    (candidate) => candidate.prefix === prefix,
+  );
+
+  if (!namespace || !rest.length) {
+    return undefined;
+  }
+
+  const entryKey = rest.join('.');
+  const key = parsePropertyDataKey(entryKey);
+
+  // the leaf path is relative to one row, so the comparison goes inside it
+  if (key.kind === 'row') {
+    const rowsPath = `${namespace.path}.${toPropertyGroupKey(key.groupId)}`;
+
+    if (anniversary !== undefined) {
+      const branches = anniversaryOn(
+        key.fieldId,
+        node.value,
+        now,
+        timeZone,
+        anniversary,
+      );
+
+      return branches ? { [rowsPath]: { $elemMatch: branches } } : undefined;
+    }
+
+    const rowComparison = compareOn(operator, node.value, now, timeZone);
+
+    return rowComparison
+      ? { [rowsPath]: { $elemMatch: { [key.fieldId]: rowComparison } } }
+      : undefined;
+  }
+
+  // The values are one object keyed by field id, so a namespaced field is the
+  // dotted path it reads as - the same shape any stored field compiles to.
+  const path = `${namespace.path}.${entryKey}`;
+
+  if (anniversary !== undefined) {
+    const branches = anniversaryOn(
+      path,
+      node.value,
+      now,
+      timeZone,
+      anniversary,
+    );
+
+    return branches ? branches : undefined;
+  }
+
+  const comparison = compareOn(operator, node.value, now, timeZone);
+
+  return comparison ? { [path]: comparison } : undefined;
+};
+
+const compileNode = (
+  node: SegmentNode,
+  context: SegmentCompileContext,
+  now: Date,
+  unsupported: string[],
+): SegmentMongoFilter | undefined => {
+  if (node.kind === 'field') {
+    const compiled = compileField(node, context, now);
+
+    if (!compiled) {
+      unsupported.push(`${node.contentType}.${node.fieldKey}`);
+    }
+
+    return compiled;
+  }
+
+  if (node.kind === 'relation') {
+    unsupported.push(node.relationKey);
+    return undefined;
+  }
+
+  if (node.kind === 'segment') {
+    return {
+      [SEGMENT_MEMBERSHIP_FIELD]: node.exclude
+        ? { $ne: node.segmentId }
+        : node.segmentId,
+    };
+  }
+
+  const compiled = node.children
+    .map((child) => compileNode(child, context, now, unsupported))
+    .filter((child): child is SegmentMongoFilter => Boolean(child));
+
+  if (!compiled.length) {
+    return undefined;
+  }
+
+  // only siblings that all have to hold can share one entry
+  const children =
+    node.conjunction === 'or' ? compiled : mergePropertyRowConditions(compiled);
+
+  if (children.length === 1) {
+    return children[0];
+  }
+
+  return node.conjunction === 'or' ? { $or: children } : { $and: children };
+};
+
+export const compileSegmentMongoFilter = (
+  node: SegmentNode,
+  context: SegmentCompileContext,
+): SegmentCompileResult => {
+  const unsupported: string[] = [];
+  const filter = compileNode(
+    node,
+    context,
+    context.now || new Date(),
+    unsupported,
+  );
+
+  return { filter: filter || {}, unsupported };
+};

package/src/core-modules/segments/nodeRefs.ts

@@ -0,0 +1,29 @@
+import {
+  SegmentFieldNode,
+  SegmentRelationNode,
+  SEGMENT_MEMBERSHIP_FIELD,
+} from './nodes';
+
+export const segmentReferenceRef = (subjectType: string): string =>
+  `${subjectType}.${SEGMENT_MEMBERSHIP_FIELD}`;
+
+export const segmentFieldRef = (node: SegmentFieldNode): string =>
+  `${node.contentType}.${node.fieldKey}`;
+
+const hashValue = (value: unknown): string => {
+  const source = JSON.stringify(value);
+  let hash = 0x811c9dc5;
+
+  for (let index = 0; index < source.length; index++) {
+    hash ^= source.charCodeAt(index);
+    hash = Math.imul(hash, 0x01000193) >>> 0;
+  }
+
+  return hash.toString(16).padStart(8, '0');
+};
+
+export const segmentRelationRef = (node: SegmentRelationNode): string =>
+  `${node.relationKey}#${hashValue({
+    measure: node.measure,
+    child: node.child,
+  })}`;

package/src/core-modules/segments/nodes.ts

@@ -0,0 +1,48 @@
+import { SegmentOperator } from './operators';
+
+export type SegmentValue = string | number | boolean | Date | string[];
+
+export type SegmentGroupNode = {
+  kind: 'group';
+  conjunction: 'and' | 'or';
+  children: SegmentNode[];
+};
+
+export type SegmentFieldNode = {
+  kind: 'field';
+  contentType: string;
+  fieldKey: string;
+  operator: SegmentOperator;
+  value?: SegmentValue;
+  meta?: Record<string, string>;
+};
+
+export type SegmentMeasure =
+  | { op: 'exists' }
+  | { op: 'none' }
+  | { op: 'count' }
+  | { op: 'sum' | 'avg' | 'min' | 'max'; fieldKey: string };
+
+export type SegmentRelationNode = {
+  kind: 'relation';
+  relationKey: string;
+  measure: SegmentMeasure;
+  child?: SegmentNode;
+  operator?: SegmentOperator;
+  value?: SegmentValue;
+};
+
+export type SegmentReferenceNode = {
+  kind: 'segment';
+  segmentId: string;
+  exclude?: boolean;
+};
+
+export type SegmentNode =
+  | SegmentGroupNode
+  | SegmentFieldNode
+  | SegmentRelationNode
+  | SegmentReferenceNode;
+
+/** The field every record carries its membership in, added by `schemaWrapper`. */
+export const SEGMENT_MEMBERSHIP_FIELD = 'segmentIds';

package/src/core-modules/segments/operators.ts

@@ -0,0 +1,243 @@
+import type { ZonedPeriod } from './zonedTime';
+
+export enum SegmentOperator {
+  Equals = 'e',
+  NotEquals = 'dne',
+  Contains = 'c',
+  NotContains = 'dnc',
+
+  IsSet = 'is',
+  IsNotSet = 'ins',
+
+  IsTrue = 'it',
+  IsFalse = 'if',
+
+  In = 'in',
+  NotIn = 'nin',
+
+  NumberGt = 'numberigt',
+  NumberLt = 'numberilt',
+
+  DateGte = 'dateigt',
+  DateLte = 'dateilt',
+
+  MinutesFromNow = 'wobm',
+  MinutesAgo = 'woam',
+  DaysFromNow = 'wobd',
+  DaysAgo = 'woad',
+
+  AnniversaryToday = 'annt',
+  AnniversaryFromNow = 'annfn',
+  AnniversaryAgo = 'annago',
+
+  DateIsSet = 'dateis',
+  DateIsNotSet = 'dateins',
+  NumberEquals = 'numbere',
+  NumberNotEquals = 'numberdne',
+  DateThisMonth = 'dtm',
+  DateThisQuarter = 'dtq',
+  DateThisYear = 'dty',
+
+  DateRelativeLt = 'drlt',
+  DateRelativeGt = 'drgt',
+}
+
+export type SegmentOperatorInput = 'none' | 'field' | 'number';
+
+export type SegmentOperatorSpec = {
+  value: SegmentOperator;
+  label: string;
+  input: SegmentOperatorInput;
+  hint?: string;
+  deprecated?: SegmentOperator;
+};
+
+export const SEGMENT_OPERATOR_SPECS: Record<
+  SegmentOperator,
+  SegmentOperatorSpec
+> = {
+  [SegmentOperator.Equals]: {
+    value: SegmentOperator.Equals,
+    label: 'equals to',
+    input: 'field',
+  },
+  [SegmentOperator.NotEquals]: {
+    value: SegmentOperator.NotEquals,
+    label: 'is not equal to',
+    input: 'field',
+  },
+  [SegmentOperator.Contains]: {
+    value: SegmentOperator.Contains,
+    label: 'contains',
+    input: 'field',
+  },
+  [SegmentOperator.NotContains]: {
+    value: SegmentOperator.NotContains,
+    label: 'does not contain',
+    input: 'field',
+  },
+  [SegmentOperator.IsSet]: {
+    value: SegmentOperator.IsSet,
+    label: 'is set',
+    input: 'none',
+  },
+  [SegmentOperator.IsNotSet]: {
+    value: SegmentOperator.IsNotSet,
+    label: 'is not set',
+    input: 'none',
+  },
+  [SegmentOperator.IsTrue]: {
+    value: SegmentOperator.IsTrue,
+    label: 'is true',
+    input: 'none',
+  },
+  [SegmentOperator.IsFalse]: {
+    value: SegmentOperator.IsFalse,
+    label: 'is false',
+    input: 'none',
+  },
+  [SegmentOperator.In]: {
+    value: SegmentOperator.In,
+    label: 'is any of',
+    input: 'field',
+  },
+  [SegmentOperator.NotIn]: {
+    value: SegmentOperator.NotIn,
+    label: 'is none of',
+    input: 'field',
+  },
+  [SegmentOperator.NumberGt]: {
+    value: SegmentOperator.NumberGt,
+    label: 'is greater than or equal to',
+    input: 'field',
+  },
+  [SegmentOperator.NumberLt]: {
+    value: SegmentOperator.NumberLt,
+    label: 'is less than or equal to',
+    input: 'field',
+  },
+  [SegmentOperator.DateGte]: {
+    value: SegmentOperator.DateGte,
+    label: 'is on or after',
+    input: 'field',
+  },
+  [SegmentOperator.DateLte]: {
+    value: SegmentOperator.DateLte,
+    label: 'is on or before',
+    input: 'field',
+  },
+  [SegmentOperator.MinutesFromNow]: {
+    value: SegmentOperator.MinutesFromNow,
+    label: 'minute(s) before',
+    input: 'number',
+    hint: 'That one minute, not everything between now and then.',
+  },
+  [SegmentOperator.MinutesAgo]: {
+    value: SegmentOperator.MinutesAgo,
+    label: 'minute(s) later',
+    input: 'number',
+    hint: 'That one minute, not everything since.',
+  },
+  [SegmentOperator.DaysFromNow]: {
+    value: SegmentOperator.DaysFromNow,
+    label: 'day(s) before',
+    input: 'number',
+    hint: 'That one day, not everything between now and then.',
+  },
+  [SegmentOperator.DaysAgo]: {
+    value: SegmentOperator.DaysAgo,
+    label: 'day(s) later',
+    input: 'number',
+    hint: 'That one day, not everything since.',
+  },
+
+  [SegmentOperator.AnniversaryToday]: {
+    value: SegmentOperator.AnniversaryToday,
+    label: 'anniversary is today',
+    input: 'none',
+    hint: 'The day and month, in any year - a birthday rather than the date itself. Re-checked every night, so it holds only for the day.',
+  },
+  [SegmentOperator.DateThisMonth]: {
+    value: SegmentOperator.DateThisMonth,
+    label: 'is in this month',
+    input: 'none',
+    hint: 'From the 1st of the current month; moves on by itself each month.',
+  },
+  [SegmentOperator.DateThisQuarter]: {
+    value: SegmentOperator.DateThisQuarter,
+    label: 'is in this quarter',
+    input: 'none',
+    hint: 'From the first day of the current quarter; moves on by itself.',
+  },
+  [SegmentOperator.DateThisYear]: {
+    value: SegmentOperator.DateThisYear,
+    label: 'is in this year',
+    input: 'none',
+    hint: 'From January 1st of the current year; moves on by itself each year.',
+  },
+  [SegmentOperator.AnniversaryFromNow]: {
+    value: SegmentOperator.AnniversaryFromNow,
+    label: 'anniversary in day(s)',
+    input: 'number',
+    hint: 'Counts forward to the next time that day and month come round, in any year.',
+  },
+  [SegmentOperator.AnniversaryAgo]: {
+    value: SegmentOperator.AnniversaryAgo,
+    label: 'day(s) since anniversary',
+    input: 'number',
+    hint: 'Counts back to the last time that day and month came round, in any year.',
+  },
+
+  [SegmentOperator.DateIsSet]: {
+    value: SegmentOperator.DateIsSet,
+    label: 'is set',
+    input: 'none',
+    deprecated: SegmentOperator.IsSet,
+  },
+  [SegmentOperator.DateIsNotSet]: {
+    value: SegmentOperator.DateIsNotSet,
+    label: 'is not set',
+    input: 'none',
+    deprecated: SegmentOperator.IsNotSet,
+  },
+  [SegmentOperator.NumberEquals]: {
+    value: SegmentOperator.NumberEquals,
+    label: 'equals to',
+    input: 'field',
+    deprecated: SegmentOperator.Equals,
+  },
+  [SegmentOperator.NumberNotEquals]: {
+    value: SegmentOperator.NumberNotEquals,
+    label: 'is not equal to',
+    input: 'field',
+    deprecated: SegmentOperator.NotEquals,
+  },
+  [SegmentOperator.DateRelativeLt]: {
+    value: SegmentOperator.DateRelativeLt,
+    label: 'is on or before',
+    input: 'field',
+    deprecated: SegmentOperator.DateLte,
+  },
+  [SegmentOperator.DateRelativeGt]: {
+    value: SegmentOperator.DateRelativeGt,
+    label: 'is on or after',
+    input: 'field',
+    deprecated: SegmentOperator.DateGte,
+  },
+};
+
+export const SEGMENT_IMPLICIT_OPERATORS: SegmentOperator[] = [
+  SegmentOperator.IsSet,
+  SegmentOperator.IsNotSet,
+];
+
+export const normalizeSegmentOperator = (
+  operator: SegmentOperator,
+): SegmentOperator => SEGMENT_OPERATOR_SPECS[operator]?.deprecated || operator;
+
+// Calendar periods holding "now", moving on by themselves.
+export const PERIOD_OPERATORS: Partial<Record<SegmentOperator, ZonedPeriod>> = {
+  [SegmentOperator.DateThisMonth]: 'month',
+  [SegmentOperator.DateThisQuarter]: 'quarter',
+  [SegmentOperator.DateThisYear]: 'year',
+};

package/src/core-modules/segments/ownedSource.ts

@@ -0,0 +1,16 @@
+export type SegmentOwnedSource = {
+  find: (
+    query: Record<string, unknown>,
+    projection: Record<string, 1>,
+  ) => Promise<Record<string, unknown>[]>;
+
+  aggregate?: (
+    pipeline: Record<string, unknown>[],
+  ) => Promise<Record<string, unknown>[]>;
+
+  baseQuery?: Record<string, unknown>;
+};
+
+export type SegmentSourceResolver = (
+  contentType: string,
+) => SegmentOwnedSource | null;