@erxes/api-shared 0.0.0-stage → 3.2.19-1

package/src/core-modules/segments/segmentReference.test.ts

@@ -0,0 +1,142 @@
+import { segmentReferenceRef } from './nodeRefs';
+import { collectSegmentReferences } from './walkNodes';
+import { segmentDependencies, segmentDependencyKey } from './dependencies';
+import { decideSegmentNode } from './evaluate';
+import { splitSegmentFieldRequests } from './fieldRequests';
+import { compileSegmentMongoFilter } from './mongoFilter';
+import { SEGMENT_MEMBERSHIP_FIELD, SegmentNode } from './nodes';
+import { buildSegmentEvaluationPlan } from './plan';
+
+const SUBJECT = 'core:contacts.customers';
+const REF = segmentReferenceRef(SUBJECT);
+
+const reference = (segmentId: string, exclude?: boolean): SegmentNode => ({
+  kind: 'segment',
+  segmentId,
+  ...(exclude ? { exclude } : {}),
+});
+
+const decide = (node: SegmentNode, memberships: string[] | undefined) =>
+  decideSegmentNode(node, {
+    subjectType: SUBJECT,
+    values: new Map(memberships === undefined ? [] : [[REF, memberships]]),
+  });
+
+describe('a segment used as a condition', () => {
+  it('matches a record carrying the referenced id', () => {
+    expect(decide(reference('vip'), ['vip', 'other'])).toBe('matched');
+    expect(decide(reference('vip'), ['other'])).toBe('notMatched');
+  });
+
+  it('reads a record with no membership at all as outside it', () => {
+    expect(decide(reference('vip'), undefined)).toBe('notMatched');
+    expect(decide(reference('vip', true), undefined)).toBe('matched');
+  });
+
+  it('inverts when excluded', () => {
+    expect(decide(reference('vip', true), ['vip'])).toBe('notMatched');
+    expect(decide(reference('vip', true), ['other'])).toBe('matched');
+  });
+
+  it('stays undecided when the membership could not be read', () => {
+    expect(
+      decideSegmentNode(reference('vip'), {
+        subjectType: SUBJECT,
+        values: new Map(),
+        unavailable: new Set([REF]),
+      }),
+    ).toBe('unknown');
+  });
+
+  it('compiles to the same answer it decides', () => {
+    expect(compileSegmentMongoFilter(reference('vip'), { fields: [] })).toEqual(
+      {
+        filter: { [SEGMENT_MEMBERSHIP_FIELD]: 'vip' },
+        unsupported: [],
+      },
+    );
+
+    expect(
+      compileSegmentMongoFilter(reference('vip', true), { fields: [] }),
+    ).toEqual({
+      filter: { [SEGMENT_MEMBERSHIP_FIELD]: { $ne: 'vip' } },
+      unsupported: [],
+    });
+  });
+
+  it('costs one value however many references a tree makes', () => {
+    const plan = buildSegmentEvaluationPlan({
+      subjectType: SUBJECT,
+      subjectIds: ['c-1'],
+      segments: [
+        {
+          _id: 'seg-1',
+          root: {
+            kind: 'group',
+            conjunction: 'and',
+            children: [reference('vip'), reference('churned', true)],
+          },
+        },
+      ],
+    });
+
+    expect(plan.requestsByPlugin.get('core')).toEqual([
+      {
+        kind: 'field',
+        ref: REF,
+        contentType: SUBJECT,
+        fieldKey: SEGMENT_MEMBERSHIP_FIELD,
+      },
+    ]);
+  });
+
+  it('is answerable without the owning plugin declaring the field', () => {
+    const split = splitSegmentFieldRequests(
+      [
+        {
+          kind: 'field',
+          ref: REF,
+          contentType: SUBJECT,
+          fieldKey: SEGMENT_MEMBERSHIP_FIELD,
+        },
+      ],
+      {},
+    );
+
+    expect(split.undeclared).toEqual([]);
+    expect(split.projected).toEqual([
+      {
+        ref: REF,
+        contentType: SUBJECT,
+        fieldKey: SEGMENT_MEMBERSHIP_FIELD,
+        path: SEGMENT_MEMBERSHIP_FIELD,
+      },
+    ]);
+  });
+
+  it('records what it reads, so a membership move finds it again', () => {
+    const root: SegmentNode = {
+      kind: 'group',
+      conjunction: 'and',
+      children: [reference('vip')],
+    };
+
+    expect(segmentDependencies(SUBJECT, root)).toEqual([
+      SUBJECT,
+      segmentDependencyKey('vip'),
+    ]);
+
+    expect(collectSegmentReferences(root)).toEqual(['vip']);
+  });
+
+  it('finds a reference nested inside a relation predicate', () => {
+    const root: SegmentNode = {
+      kind: 'relation',
+      relationKey: 'customer.deals',
+      measure: { op: 'exists' },
+      child: reference('won'),
+    };
+
+    expect(collectSegmentReferences(root)).toEqual(['won']);
+  });
+});

package/src/core-modules/segments/setupSegmentProducers.ts

@@ -0,0 +1,77 @@
+import { AnyProcedure, initTRPC } from '@trpc/server';
+import * as trpcExpress from '@trpc/server/adapters/express';
+import { Express } from 'express';
+import { nanoid } from 'nanoid';
+import { createTRPCContext, initializePluginConfig } from '../../utils';
+import { SegmentConfigs, TSegmentProducers } from './types';
+import {
+  ApplyMembershipInput,
+  CountSegmentMembersInput,
+  EvaluateFieldsInput,
+  ListSegmentMembersInput,
+} from './zodSchemas';
+
+export const initSegmentProducers = async (
+  app: Express,
+  pluginName: string,
+  config: SegmentConfigs,
+) => {
+  await initializePluginConfig(pluginName, 'segments', config);
+
+  const t = initTRPC
+    .context<{ subdomain: string; processId: string }>()
+    .create();
+
+  const {
+    evaluateFields,
+    listSegmentMembers,
+    countSegmentMembers,
+    applyMembership,
+  } = config || {};
+
+  const segmentProducers: Partial<Record<TSegmentProducers, AnyProcedure>> = {};
+
+  if (evaluateFields) {
+    segmentProducers[TSegmentProducers.EVALUATE_FIELDS] = t.procedure
+      .input(EvaluateFieldsInput)
+      .query(async ({ ctx, input }) => evaluateFields(input, ctx));
+  }
+
+  if (listSegmentMembers) {
+    segmentProducers[TSegmentProducers.LIST_MEMBERS] = t.procedure
+      .input(ListSegmentMembersInput)
+      .query(async ({ ctx, input }) => listSegmentMembers(input, ctx));
+  }
+
+  if (countSegmentMembers) {
+    segmentProducers[TSegmentProducers.COUNT_MEMBERS] = t.procedure
+      .input(CountSegmentMembersInput)
+      .query(async ({ ctx, input }) => countSegmentMembers(input, ctx));
+  }
+
+  if (applyMembership) {
+    segmentProducers[TSegmentProducers.APPLY_MEMBERSHIP] = t.procedure
+      .input(ApplyMembershipInput)
+      .mutation(async ({ ctx, input }) => applyMembership(input, ctx));
+  }
+
+  if (!Object.keys(segmentProducers)?.length) {
+    console.warn(
+      `[Segments] No segment producers found for plugin ${pluginName}`,
+    );
+    return;
+  }
+
+  const segmentsRouter = t.router(segmentProducers);
+
+  const trpcMiddleware = trpcExpress.createExpressMiddleware({
+    router: segmentsRouter,
+    createContext: createTRPCContext(async (_subdomain, context) => {
+      const processId = nanoid(12);
+      context.processId = processId;
+      return context;
+    }),
+  });
+
+  app.use('/segments', trpcMiddleware);
+};

package/src/core-modules/segments/subjectsReachedBy.ts

@@ -0,0 +1,70 @@
+import { sendTRPCMessage } from '../../utils';
+import { SegmentChangedEvent } from './events';
+import { gatherSegmentRelations } from './relationRegistry';
+
+const idsAt = (value: unknown): string[] =>
+  (Array.isArray(value) ? value : [value]).filter(
+    (id): id is string => typeof id === 'string' && id.length > 0,
+  );
+
+/**
+ * Which of a segment's subjects a change to `docIds` can move. Shared by the
+ * membership worker and by automations, whose trigger segments are never
+ * materialized and so must be re-asked when a related record changes.
+ */
+export const segmentSubjectsReachedBy = async (
+  subdomain: string,
+  segment: { _id: string; contentType: string },
+  changedTypes: string[],
+  docIds: string[],
+  changed?: SegmentChangedEvent['changed'],
+  onSkip?: (message: string, data?: Record<string, unknown>) => void,
+): Promise<string[]> => {
+  if (changedTypes.includes(segment.contentType)) {
+    return docIds;
+  }
+
+  const { relations } = await gatherSegmentRelations(segment.contentType);
+
+  const reaching = [...relations.values()].filter((relation) =>
+    changedTypes.includes(relation.relatedType),
+  );
+
+  const subjects = new Set<string>();
+
+  for (const relation of reaching) {
+    if (relation.join.via === 'field') {
+      const moved = changed?.[relation.join.path];
+
+      if (relation.join.on === 'subject') {
+        docIds.forEach((id) => subjects.add(id));
+      } else if (moved) {
+        idsAt(moved.prev).forEach((id) => subjects.add(id));
+        idsAt(moved.next).forEach((id) => subjects.add(id));
+      } else {
+        onSkip?.(`${segment._id}: ${relation.key} moved without a diff`, {
+          relatedType: relation.relatedType,
+        });
+      }
+
+      continue;
+    }
+
+    const found: string[] = await sendTRPCMessage({
+      subdomain,
+      pluginName: 'core',
+      module: 'segment',
+      action: 'relationSubjects',
+      input: {
+        subjectType: relation.join.subjectRecordType,
+        relatedType: relation.join.relatedRecordType,
+        relatedIds: docIds,
+      },
+      defaultValue: [],
+    });
+
+    found.forEach((id) => subjects.add(id));
+  }
+
+  return [...subjects];
+};

package/src/core-modules/segments/timeZone.ts

@@ -0,0 +1,56 @@
+import { getEnv, sendTRPCMessage } from '../../utils';
+import { DEFAULT_SEGMENT_TIME_ZONE } from './zonedTime';
+
+const TTL_MS = 5 * 60_000;
+const CONFIG_CODE = 'TIMEZONE';
+
+const cache = new Map<string, { value: string; until: number }>();
+
+export const segmentTimeZone = async (subdomain: string): Promise<string> => {
+  const cached = cache.get(subdomain);
+
+  if (cached && cached.until > Date.now()) {
+    return cached.value;
+  }
+
+  const configs: Record<string, unknown> = await sendTRPCMessage({
+    subdomain,
+    pluginName: 'core',
+    module: 'configs',
+    action: 'getConfigs',
+    method: 'query',
+    input: { codes: [CONFIG_CODE] },
+    defaultValue: {},
+  });
+
+  const configured = configs?.[CONFIG_CODE];
+
+  const value =
+    (typeof configured === 'string' && configured.trim()) ||
+    getEnv({ name: 'SEGMENT_TIME_ZONE', defaultValue: '' }) ||
+    DEFAULT_SEGMENT_TIME_ZONE;
+
+  if (!isResolvable(value)) {
+    console.error(`segments: unknown time zone "${value}", using UTC`);
+
+    cache.set(subdomain, {
+      value: DEFAULT_SEGMENT_TIME_ZONE,
+      until: Date.now() + TTL_MS,
+    });
+
+    return DEFAULT_SEGMENT_TIME_ZONE;
+  }
+
+  cache.set(subdomain, { value, until: Date.now() + TTL_MS });
+
+  return value;
+};
+
+const isResolvable = (timeZone: string): boolean => {
+  try {
+    new Intl.DateTimeFormat('en-US', { timeZone });
+    return true;
+  } catch {
+    return false;
+  }
+};

package/src/core-modules/segments/types.ts

@@ -0,0 +1,97 @@
+import { SegmentRelationMeta } from './relationRegistry';
+import { z } from 'zod';
+import { SegmentFieldMeta, SegmentFieldNamespace } from './fieldMeta';
+import {
+  ApplyMembershipInput,
+  CountSegmentMembersInput,
+  EvaluateFieldsInput,
+  ListSegmentMembersInput,
+} from './zodSchemas';
+
+type IContext = {
+  subdomain: string;
+  processId?: string;
+};
+
+export type ISegmentDependentModule = {
+  name: string;
+  types?: string[];
+  twoWay?: boolean;
+  associated?: boolean;
+};
+
+export type ISegmentContentType = {
+  contentType?: string;
+  eventTypes?: string[];
+  moduleName: string;
+  type: string;
+  description: string;
+  hideInSidebar?: boolean;
+  notAssociated?: boolean;
+};
+
+export type SegmentEvaluateFieldsResult = {
+  values: Record<string, Record<string, unknown>>;
+  unavailable?: string[];
+};
+
+export type SegmentMemberPage = {
+  ids: string[];
+  nextCursor?: string;
+  unsupported?: string[];
+};
+
+export type SegmentMemberCount = {
+  count: number;
+  unsupported?: string[];
+  exceeded?: boolean;
+};
+
+export type SegmentApplyMembershipResult = {
+  counts: Record<string, number>;
+  transitions?: SegmentMembershipTransition[];
+  unsupported?: string[];
+  countedAt?: string;
+};
+
+export type SegmentMembershipTransition = {
+  segmentId: string;
+  joined: string[];
+  left: string[];
+};
+
+export interface SegmentConfigs {
+  contentTypes: ISegmentContentType[];
+  dependentModules?: ISegmentDependentModule[];
+
+  segmentFields?: Record<string, SegmentFieldMeta[]>;
+  segmentFieldNamespaces?: Record<string, SegmentFieldNamespace[]>;
+  segmentRelations?: SegmentRelationMeta[];
+
+  evaluateFields?: (
+    args: z.infer<typeof EvaluateFieldsInput>,
+    context: IContext,
+  ) => Promise<SegmentEvaluateFieldsResult>;
+
+  listSegmentMembers?: (
+    args: z.infer<typeof ListSegmentMembersInput>,
+    context: IContext,
+  ) => Promise<SegmentMemberPage>;
+
+  countSegmentMembers?: (
+    args: z.infer<typeof CountSegmentMembersInput>,
+    context: IContext,
+  ) => Promise<SegmentMemberCount>;
+
+  applyMembership?: (
+    args: z.infer<typeof ApplyMembershipInput>,
+    context: IContext,
+  ) => Promise<SegmentApplyMembershipResult>;
+}
+
+export enum TSegmentProducers {
+  EVALUATE_FIELDS = 'evaluateFields',
+  LIST_MEMBERS = 'listSegmentMembers',
+  COUNT_MEMBERS = 'countSegmentMembers',
+  APPLY_MEMBERSHIP = 'applyMembership',
+}

package/src/core-modules/segments/utils.ts

@@ -0,0 +1,122 @@
+import { getPlugin, getPlugins } from '../../utils';
+import {
+  ISegmentDependentModule,
+  ISegmentContentType,
+  SegmentConfigs,
+} from './types';
+
+export const gatherDependentServicesType = async (
+  pluginName: string,
+  gatherTypes: (
+    contentType: ISegmentContentType,
+    pName: string,
+    depService: ISegmentDependentModule,
+  ) => void,
+) => {
+  const pluginNames = await getPlugins();
+
+  for (const pName of pluginNames) {
+    const plugin = await getPlugin(pName);
+    const segmentMeta = plugin.config?.meta?.segments;
+
+    if (!segmentMeta) {
+      continue;
+    }
+
+    const dependentServices = segmentMeta.dependentServices || [];
+    const contentTypes = segmentMeta.contentTypes || [];
+
+    for (const dService of dependentServices) {
+      if (dService.name !== pluginName || !dService.twoWay) {
+        continue;
+      }
+
+      for (const contentType of contentTypes) {
+        if (
+          !!dService?.types?.length &&
+          !(dService?.types || []).includes(contentType?.type)
+        ) {
+          continue;
+        }
+
+        gatherTypes(contentType, pName, dService);
+      }
+    }
+  }
+};
+export const gatherAssociatedTypes = async (contentType: string) => {
+  const [pluginName, currentType] = contentType.split(':');
+  const plugin = await getPlugin(pluginName);
+  const segmentMeta = plugin.config.meta?.segments as SegmentConfigs;
+
+  const associatedTypes: string[] = [];
+
+  if (segmentMeta) {
+    const { contentTypes } = segmentMeta;
+
+    // gather current services associatedTypes
+    for (const ct of contentTypes) {
+      if (ct.type !== currentType && !ct.notAssociated) {
+        associatedTypes.push(`${pluginName}:${ct.moduleName}.${ct.type}`);
+      }
+    }
+
+    // gather current services dependentServices associatedTypes
+    if (segmentMeta.dependentModules) {
+      const dependentModules = segmentMeta.dependentModules || [];
+
+      for (const dependentModule of dependentModules) {
+        if (!dependentModule.associated) {
+          continue;
+        }
+
+        await gatherServicesAssociatedTypes(
+          dependentModule.name,
+          associatedTypes,
+        );
+      }
+    }
+  }
+
+  // gather current services dependentServices associatedTypes
+  await gatherDependentServicesType(
+    pluginName,
+    (
+      ct: ISegmentContentType,
+      pName: string,
+      depService: ISegmentDependentModule,
+    ) => {
+      if (!depService.associated || ct.notAssociated) {
+        return;
+      }
+
+      associatedTypes.push(`${pName}:${ct.moduleName}.${ct.type}`);
+    },
+  );
+
+  return associatedTypes;
+};
+
+const gatherServicesAssociatedTypes = async (
+  pluginName: string,
+  associatedTypes: string[],
+) => {
+  const plugin = await getPlugin(pluginName);
+  const segmentMeta = plugin.config?.meta?.segments;
+
+  if (!segmentMeta) {
+    return;
+  }
+
+  const contentTypes = segmentMeta.contentTypes || [];
+
+  for (const contentType of contentTypes) {
+    if (contentType.notAssociated) {
+      continue;
+    }
+
+    associatedTypes.push(
+      `${pluginName}:${contentType.moduleName}.${contentType.type}`,
+    );
+  }
+};

package/src/core-modules/segments/walkNodes.ts

@@ -0,0 +1,76 @@
+import { SegmentNode } from './nodes';
+import {
+  segmentFieldRef,
+  segmentRelationRef,
+  segmentReferenceRef,
+} from './nodeRefs';
+
+export type SegmentWalkOptions = {
+  intoRelationChild?: boolean;
+};
+
+export function* walkSegmentNodes(
+  node: SegmentNode,
+  options: SegmentWalkOptions = {},
+): Generator<SegmentNode> {
+  yield node;
+
+  if (node.kind === 'group') {
+    for (const child of node.children) {
+      yield* walkSegmentNodes(child, options);
+    }
+  }
+
+  if (node.kind === 'relation' && options.intoRelationChild && node.child) {
+    yield* walkSegmentNodes(node.child, options);
+  }
+}
+
+export const collectSegmentValueRefs = (
+  node: SegmentNode,
+  subjectType?: string,
+): string[] => {
+  const refs = new Set<string>();
+
+  for (const current of walkSegmentNodes(node)) {
+    if (current.kind === 'field') {
+      refs.add(segmentFieldRef(current));
+    }
+
+    if (current.kind === 'relation') {
+      refs.add(segmentRelationRef(current));
+    }
+
+    if (current.kind === 'segment' && subjectType) {
+      refs.add(segmentReferenceRef(subjectType));
+    }
+  }
+
+  return [...refs];
+};
+
+export const collectSegmentReferences = (node: SegmentNode): string[] => {
+  const ids = new Set<string>();
+
+  for (const current of walkSegmentNodes(node, { intoRelationChild: true })) {
+    if (current.kind === 'segment') {
+      ids.add(current.segmentId);
+    }
+  }
+
+  return [...ids];
+};
+
+export const hasSegmentReferenceInRelation = (node: SegmentNode): boolean => {
+  for (const current of walkSegmentNodes(node)) {
+    if (
+      current.kind === 'relation' &&
+      current.child &&
+      collectSegmentReferences(current.child).length
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+};