@erxes/api-shared 0.0.0-stage → 3.2.19-1

package/src/core-modules/segments/events.ts

@@ -0,0 +1,142 @@
+import { sendWorkerQueue } from '../../utils/mq-worker';
+
+export const SEGMENT_QUEUES = {
+  service: 'logs',
+  changed: 'segment-changed',
+  forget: 'segment-forget',
+  rebuild: 'segment-rebuild',
+  reconcile: 'segment-reconcile',
+} as const;
+
+export type SegmentQueueName = (typeof SEGMENT_QUEUES)[Exclude<
+  keyof typeof SEGMENT_QUEUES,
+  'service'
+>];
+
+export const segmentQueueFor = (job: SegmentJob): SegmentQueueName =>
+  SEGMENT_QUEUES[job.kind || 'changed'];
+
+export type SegmentChangedEvent = {
+  kind?: 'changed';
+  subdomain: string;
+  contentType: string;
+  docIds: string[];
+  segmentIds?: string[];
+  changed?: Record<string, { prev?: unknown; next?: unknown }>;
+};
+
+export type SegmentRebuildEvent = {
+  kind: 'rebuild';
+  subdomain: string;
+  segmentId: string;
+  // Only the nightly pass says so: a create, edit or manual rebuild is not
+  // the day's materialization.
+  source?: 'reconcile';
+};
+
+export type SegmentReconcileEvent = {
+  kind: 'reconcile';
+  subdomain: string;
+  before?: string;
+  step?: number;
+};
+
+export type SegmentForgetEvent = {
+  kind: 'forget';
+  subdomain: string;
+  contentType: string;
+  segmentIds: string[];
+};
+
+export type SegmentJob =
+  | SegmentChangedEvent
+  | SegmentRebuildEvent
+  | SegmentForgetEvent
+  | SegmentReconcileEvent;
+
+const RECONCILE_CRON = process.env.SEGMENT_RECONCILE_CRON || '0 3 * * *';
+const RECONCILE_TZ = process.env.SEGMENT_RECONCILE_TZ;
+
+const reconcileScheduled = new Set<string>();
+
+export const scheduleSegmentReconcile = (subdomain: string): void => {
+  if (reconcileScheduled.has(subdomain)) {
+    return;
+  }
+
+  reconcileScheduled.add(subdomain);
+
+  try {
+    sendWorkerQueue(SEGMENT_QUEUES.service, SEGMENT_QUEUES.reconcile)
+      .add(
+        SEGMENT_QUEUES.reconcile,
+        { kind: 'reconcile', subdomain } as SegmentReconcileEvent,
+        {
+          repeat: {
+            pattern: RECONCILE_CRON,
+            ...(RECONCILE_TZ ? { tz: RECONCILE_TZ } : {}),
+          },
+          jobId: `segment-reconcile-${subdomain}`,
+          removeOnComplete: true,
+          removeOnFail: true,
+        },
+      )
+      .catch(() => {
+        reconcileScheduled.delete(subdomain);
+      });
+  } catch {
+    reconcileScheduled.delete(subdomain);
+  }
+};
+
+const attemptsFor = (job: SegmentJob): number =>
+  job.kind === 'rebuild' || job.kind === 'reconcile' ? 1 : 3;
+
+const enqueue = (job: SegmentJob): void => {
+  scheduleSegmentReconcile(job.subdomain);
+
+  try {
+    const name = segmentQueueFor(job);
+
+    sendWorkerQueue(SEGMENT_QUEUES.service, name)
+      .add(name, job, {
+        removeOnComplete: true,
+        removeOnFail: true,
+        attempts: attemptsFor(job),
+      })
+      .catch(() => {
+        // Best effort: a write is never rolled back because its follow-up
+        // could not be queued. Reconciliation catches what drifts.
+      });
+  } catch {
+    // As above - the queue being unreachable must not fail the write.
+  }
+};
+
+export const sendSegmentChanged = (
+  event: Omit<SegmentChangedEvent, 'kind'>,
+): void => {
+  if (!event.docIds.length) {
+    return;
+  }
+
+  enqueue({ ...event, kind: 'changed' });
+};
+
+export const sendSegmentReconcile = (
+  event: Omit<SegmentReconcileEvent, 'kind'>,
+): void => enqueue({ ...event, kind: 'reconcile' });
+
+export const sendSegmentRebuild = (
+  event: Omit<SegmentRebuildEvent, 'kind'>,
+): void => enqueue({ ...event, kind: 'rebuild' });
+
+export const sendSegmentForget = (
+  event: Omit<SegmentForgetEvent, 'kind'>,
+): void => {
+  if (!event.segmentIds.length) {
+    return;
+  }
+
+  enqueue({ ...event, kind: 'forget' });
+};

package/src/core-modules/segments/fieldBuilders.ts

@@ -0,0 +1,152 @@
+import { SegmentOperator } from './operators';
+import {
+  SegmentFieldMeta,
+  SegmentFieldOption,
+  SegmentFieldQuery,
+} from './fieldMeta';
+
+export const SEGMENT_TEXT_OPERATORS = [
+  SegmentOperator.Equals,
+  SegmentOperator.NotEquals,
+  SegmentOperator.Contains,
+  SegmentOperator.NotContains,
+];
+
+export const SEGMENT_ID_OPERATORS = [
+  SegmentOperator.Equals,
+  SegmentOperator.NotEquals,
+];
+
+export const SEGMENT_NUMBER_OPERATORS = [
+  SegmentOperator.Equals,
+  SegmentOperator.NumberGt,
+  SegmentOperator.NumberLt,
+];
+
+export const SEGMENT_DATE_OPERATORS = [
+  SegmentOperator.DateGte,
+  SegmentOperator.DateLte,
+  SegmentOperator.DaysAgo,
+  SegmentOperator.DaysFromNow,
+  SegmentOperator.DateThisMonth,
+  SegmentOperator.DateThisQuarter,
+  SegmentOperator.DateThisYear,
+  SegmentOperator.AnniversaryToday,
+  SegmentOperator.AnniversaryFromNow,
+  SegmentOperator.AnniversaryAgo,
+];
+
+export const SEGMENT_BOOLEAN_OPERATORS = [
+  SegmentOperator.IsTrue,
+  SegmentOperator.IsFalse,
+];
+
+type Base = {
+  key: string;
+  label: string;
+  path?: string;
+};
+
+const text = ({ key, label, path }: Base): SegmentFieldMeta => ({
+  key,
+  label,
+  operators: SEGMENT_TEXT_OPERATORS,
+  kind: 'projected',
+  path: path || key,
+  input: 'text',
+});
+
+const number = ({ key, label, path }: Base): SegmentFieldMeta => ({
+  key,
+  label,
+  operators: SEGMENT_NUMBER_OPERATORS,
+  kind: 'projected',
+  path: path || key,
+  input: 'number',
+});
+
+const date = ({ key, label, path }: Base): SegmentFieldMeta => ({
+  key,
+  label,
+  operators: SEGMENT_DATE_OPERATORS,
+  kind: 'projected',
+  path: path || key,
+  input: 'date',
+});
+
+const boolean = ({ key, label, path }: Base): SegmentFieldMeta => ({
+  key,
+  label,
+  operators: SEGMENT_BOOLEAN_OPERATORS,
+  kind: 'projected',
+  path: path || key,
+  input: 'boolean',
+});
+
+const staticOptions = ({
+  key,
+  label,
+  path,
+  options,
+}: Base & {
+  options: readonly string[] | SegmentFieldOption[];
+}): SegmentFieldMeta => ({
+  key,
+  label,
+  operators: SEGMENT_ID_OPERATORS,
+  kind: 'projected',
+  path: path || key,
+  input: 'select',
+  source: 'static',
+  options: options.map((option) =>
+    typeof option === 'string' ? { value: option, label: option } : option,
+  ),
+});
+
+const lookup = ({
+  key,
+  label,
+  path,
+  query,
+}: Base & { query: SegmentFieldQuery }): SegmentFieldMeta => ({
+  key,
+  label,
+  operators: SEGMENT_ID_OPERATORS,
+  kind: 'projected',
+  path: path || key,
+  input: 'select',
+  source: 'query',
+  query,
+});
+
+const component = ({
+  key,
+  label,
+  path,
+  component,
+}: Base & { component: string }): SegmentFieldMeta => ({
+  key,
+  label,
+  operators: SEGMENT_ID_OPERATORS,
+  kind: 'projected',
+  path: path || key,
+  input: 'select',
+  source: 'component',
+  component,
+});
+
+/**
+ * Builders for a content type's filterable fields.
+ *
+ * Namespaced because every call site is a list of one-liners, where a bare
+ * `textField(...)` says nothing about what it is building.
+ */
+export const SegmentField = {
+  text,
+  number,
+  date,
+  boolean,
+  static: staticOptions,
+  lookup,
+  component,
+} as const;

package/src/core-modules/segments/fieldMeta.ts

@@ -0,0 +1,96 @@
+import {
+  SegmentOperator,
+  SegmentOperatorSpec,
+  SEGMENT_IMPLICIT_OPERATORS,
+  SEGMENT_OPERATOR_SPECS,
+} from './operators';
+
+export type SegmentFieldOption = {
+  value: string;
+  label: string;
+};
+
+export type SegmentFieldQuery = {
+  name: string;
+  labelField: string;
+  valueField?: string;
+  // Narrows the list, e.g. a tags lookup to the tags of one content type.
+  variables?: Record<string, string | boolean>;
+};
+
+export const DEFAULT_SEGMENT_VALUE_FIELD = '_id';
+
+export type SegmentFieldDependency = {
+  contentType?: string;
+  fields: string[];
+  via?: string;
+};
+
+type SegmentFieldSource =
+  | {
+      kind: 'projected';
+      path: string;
+      dependsOn?: SegmentFieldDependency[];
+    }
+  | {
+      kind: 'derived';
+      dependsOn: SegmentFieldDependency[];
+    };
+
+type SegmentFieldIdentity = {
+  key: string;
+  label: string;
+  operators: SegmentOperator[];
+};
+
+type SegmentFieldPresentation =
+  | { input: 'text' | 'number' | 'date' | 'boolean' }
+  | { input: 'select'; source: 'static'; options: SegmentFieldOption[] }
+  | { input: 'select'; source: 'query'; query: SegmentFieldQuery }
+  | { input: 'select'; source: 'component'; component: string };
+
+export type SegmentFieldMeta = SegmentFieldIdentity &
+  SegmentFieldSource &
+  SegmentFieldPresentation;
+
+/**
+ * A group of fields nobody can enumerate at build time - a tenant's own
+ * custom properties. The values live in one object on the record, keyed by
+ * field id, so `prefix.<id>` reads and filters as the dotted path it is.
+ */
+export type SegmentFieldNamespace = {
+  prefix: string;
+  label: string;
+  path: string;
+  // properties `contentType` the fields are listed from, e.g. `sales:deal`
+  propertyType?: string;
+};
+
+export const resolveSegmentFieldOperators = (
+  field: SegmentFieldMeta,
+): SegmentOperatorSpec[] => {
+  if (field.kind === 'derived') {
+    return field.operators.map((operator) => SEGMENT_OPERATOR_SPECS[operator]);
+  }
+
+  const declared = field.operators.filter(
+    (operator) => !SEGMENT_IMPLICIT_OPERATORS.includes(operator),
+  );
+
+  return [...declared, ...SEGMENT_IMPLICIT_OPERATORS].map(
+    (operator) => SEGMENT_OPERATOR_SPECS[operator],
+  );
+};
+
+export const resolveSegmentFieldDependencies = (
+  field: SegmentFieldMeta,
+): SegmentFieldDependency[] => {
+  if (field.dependsOn?.length) {
+    return field.dependsOn;
+  }
+
+  return field.kind === 'projected' ? [{ fields: [field.path] }] : [];
+};
+
+export const resolveSegmentValueField = (query: SegmentFieldQuery): string =>
+  query.valueField || DEFAULT_SEGMENT_VALUE_FIELD;

package/src/core-modules/segments/fieldRequests.test.ts

@@ -0,0 +1,302 @@
+import { SegmentOperator } from './operators';
+import { SegmentFieldMeta, SegmentFieldNamespace } from './fieldMeta';
+import {
+  namespacePaths,
+  projectionForRequests,
+  readNamespacedValue,
+  splitSegmentFieldRequests,
+} from './fieldRequests';
+import { SegmentValueRequest } from './plan';
+
+const fields: Record<string, SegmentFieldMeta[]> = {
+  'sales:deal': [
+    {
+      key: 'stageId',
+      label: 'Stage',
+      operators: [SegmentOperator.Equals],
+      kind: 'projected',
+      path: 'stageId',
+      input: 'select',
+      source: 'component',
+      component: 'dealStage',
+    },
+    {
+      key: 'productId',
+      label: 'Product',
+      operators: [SegmentOperator.Equals],
+      kind: 'projected',
+      path: 'productsData.productId',
+      input: 'text',
+    },
+    {
+      key: 'totalAmount',
+      label: 'Total amount',
+      operators: [SegmentOperator.NumberGt],
+      kind: 'derived',
+      dependsOn: [{ fields: ['productsData'] }],
+      input: 'number',
+    },
+  ],
+};
+
+const fieldRequest = (fieldKey: string): SegmentValueRequest => ({
+  kind: 'field',
+  ref: `sales:deal.${fieldKey}`,
+  contentType: 'sales:deal',
+  fieldKey,
+});
+
+describe('splitSegmentFieldRequests', () => {
+  it('separates fields read off the document from ones the plugin computes', () => {
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('stageId'), fieldRequest('totalAmount')],
+      fields,
+    );
+
+    expect(split.projected).toEqual([
+      {
+        ref: 'sales:deal.stageId',
+        contentType: 'sales:deal',
+        fieldKey: 'stageId',
+        path: 'stageId',
+      },
+    ]);
+    expect(split.derived).toEqual([
+      {
+        ref: 'sales:deal.totalAmount',
+        contentType: 'sales:deal',
+        fieldKey: 'totalAmount',
+      },
+    ]);
+    expect(split.undeclared).toEqual([]);
+  });
+
+  it('carries the declared mongo path, which may differ from the key', () => {
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('productId')],
+      fields,
+    );
+
+    expect(split.projected[0].path).toBe('productsData.productId');
+  });
+
+  it('reports a ref the plugin no longer declares', () => {
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('removedField')],
+      fields,
+    );
+
+    expect(split.undeclared).toEqual(['sales:deal.removedField']);
+    expect(split.projected).toEqual([]);
+    expect(split.derived).toEqual([]);
+  });
+
+  it('reports a ref for a content type the plugin declares nothing for', () => {
+    const split = splitSegmentFieldRequests(
+      [
+        {
+          kind: 'field',
+          ref: 'sales:order.total',
+          contentType: 'sales:order',
+          fieldKey: 'total',
+        },
+      ],
+      fields,
+    );
+
+    expect(split.undeclared).toEqual(['sales:order.total']);
+  });
+
+  it('keeps relation requests aside for their own resolver', () => {
+    const relation: SegmentValueRequest = {
+      kind: 'relation',
+      ref: 'customer.deals#abcd1234',
+      relationKey: 'customer.deals',
+      measure: { op: 'exists' },
+      child: {
+        kind: 'field',
+        contentType: 'sales:deal',
+        fieldKey: 'status',
+        operator: SegmentOperator.Equals,
+        value: 'won',
+      },
+    };
+
+    const split = splitSegmentFieldRequests(
+      [relation, fieldRequest('stageId')],
+      fields,
+    );
+
+    expect(split.relations).toEqual([relation]);
+    expect(split.projected).toHaveLength(1);
+  });
+
+  it('treats a plugin with no declaration as declaring nothing', () => {
+    expect(
+      splitSegmentFieldRequests([fieldRequest('stageId')]).undeclared,
+    ).toEqual(['sales:deal.stageId']);
+  });
+});
+
+describe('projectionForRequests', () => {
+  it('reads every projected path in one query, always with _id', () => {
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('stageId'), fieldRequest('productId')],
+      fields,
+    );
+
+    expect(projectionForRequests(split.projected)).toEqual({
+      _id: 1,
+      stageId: 1,
+      'productsData.productId': 1,
+    });
+  });
+
+  it('still asks for _id when nothing is projected', () => {
+    expect(projectionForRequests([])).toEqual({ _id: 1 });
+  });
+});
+
+describe('splitSegmentFieldRequests · namespaces', () => {
+  const namespaces: Record<string, SegmentFieldNamespace[]> = {
+    'sales:deal': [
+      {
+        prefix: 'propertiesData',
+        label: 'Custom properties',
+        path: 'propertiesData',
+      },
+    ],
+  };
+
+  it('routes a tenant-keyed entry to its namespace', () => {
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('propertiesData.Gm3K8ni3r2i4eNeXC')],
+      fields,
+      namespaces,
+    );
+
+    expect(split.namespaced).toEqual([
+      {
+        ref: 'sales:deal.propertiesData.Gm3K8ni3r2i4eNeXC',
+        contentType: 'sales:deal',
+        fieldKey: 'propertiesData.Gm3K8ni3r2i4eNeXC',
+        namespace: namespaces['sales:deal'][0],
+        entryKey: 'Gm3K8ni3r2i4eNeXC',
+      },
+    ]);
+    expect(split.undeclared).toEqual([]);
+  });
+
+  it('leaves a bare prefix with no key undeclared', () => {
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('propertiesData')],
+      fields,
+      namespaces,
+    );
+
+    expect(split.namespaced).toEqual([]);
+    expect(split.undeclared).toEqual(['sales:deal.propertiesData']);
+  });
+
+  it('lets a declared field win over a namespace prefix', () => {
+    const withPrefixField = {
+      'sales:deal': [
+        ...fields['sales:deal'],
+        {
+          key: 'propertiesData.pinned',
+          label: 'Pinned',
+          operators: [SegmentOperator.Equals],
+          kind: 'projected' as const,
+          path: 'pinned',
+          input: 'text' as const,
+        },
+      ],
+    };
+
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('propertiesData.pinned')],
+      withPrefixField,
+      namespaces,
+    );
+
+    expect(split.projected).toHaveLength(1);
+    expect(split.namespaced).toEqual([]);
+  });
+
+  it('collects the leaf value of every row in a repeating group', () => {
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('propertiesData.g:edu/school')],
+      fields,
+      namespaces,
+    );
+
+    const document = {
+      propertiesData: {
+        plan: 'enterprise',
+        'g:edu': [
+          { _id: 'r1', school: 'MUIS', year: 2018 },
+          { _id: 'r2', school: 'SUIS' },
+          { _id: 'r3', year: 2022 },
+        ],
+      },
+    };
+
+    expect(readNamespacedValue(document, split.namespaced[0])).toEqual([
+      'MUIS',
+      'SUIS',
+    ]);
+  });
+
+  it('reads nothing when a repeating group holds no rows', () => {
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('propertiesData.g:edu/school')],
+      fields,
+      namespaces,
+    );
+
+    expect(
+      readNamespacedValue({ propertiesData: { plan: 'x' } }, split.namespaced[0]),
+    ).toBeUndefined();
+  });
+
+  it('reads the value the key holds', () => {
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('propertiesData.plan')],
+      fields,
+      namespaces,
+    );
+
+    const document = {
+      propertiesData: { other: 'no', plan: 'enterprise' },
+    };
+
+    expect(readNamespacedValue(document, split.namespaced[0])).toBe(
+      'enterprise',
+    );
+    expect(readNamespacedValue({}, split.namespaced[0])).toBeUndefined();
+  });
+
+  it('reads nothing out of the retired array shape', () => {
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('propertiesData.plan')],
+      fields,
+      namespaces,
+    );
+
+    const document = {
+      propertiesData: [{ field: 'plan', value: 'enterprise' }],
+    };
+
+    expect(readNamespacedValue(document, split.namespaced[0])).toBeUndefined();
+  });
+
+  it('lists each namespace path once for the projection', () => {
+    const split = splitSegmentFieldRequests(
+      [fieldRequest('propertiesData.a'), fieldRequest('propertiesData.b')],
+      fields,
+      namespaces,
+    );
+
+    expect(namespacePaths(split.namespaced)).toEqual(['propertiesData']);
+  });
+});