@erxes/api-shared 0.0.0-stage → 3.2.19-1

package/src/core-modules/segments/fieldRequests.ts

@@ -0,0 +1,172 @@
+import { parsePropertyDataKey, toPropertyGroupKey } from '../properties/keys';
+import { SegmentFieldMeta, SegmentFieldNamespace } from './fieldMeta';
+import { SEGMENT_MEMBERSHIP_FIELD } from './nodes';
+import { SegmentRelationRequest, SegmentValueRequest } from './plan';
+
+export type SegmentProjectedRequest = {
+  ref: string;
+  contentType: string;
+  fieldKey: string;
+  path: string;
+};
+
+export type SegmentDerivedRequest = {
+  ref: string;
+  contentType: string;
+  fieldKey: string;
+};
+
+export type SegmentNamespacedRequest = {
+  ref: string;
+  contentType: string;
+  fieldKey: string;
+  namespace: SegmentFieldNamespace;
+  entryKey: string;
+};
+
+export type SegmentRequestSplit = {
+  projected: SegmentProjectedRequest[];
+  derived: SegmentDerivedRequest[];
+  namespaced: SegmentNamespacedRequest[];
+  relations: SegmentRelationRequest[];
+  undeclared: string[];
+};
+
+export const splitSegmentFieldRequests = (
+  requests: SegmentValueRequest[],
+  segmentFields: Record<string, SegmentFieldMeta[]> = {},
+  segmentFieldNamespaces: Record<string, SegmentFieldNamespace[]> = {},
+): SegmentRequestSplit => {
+  const split: SegmentRequestSplit = {
+    projected: [],
+    derived: [],
+    namespaced: [],
+    relations: [],
+    undeclared: [],
+  };
+
+  for (const request of requests) {
+    if (request.kind === 'relation') {
+      split.relations.push(request);
+      continue;
+    }
+
+    if (request.fieldKey === SEGMENT_MEMBERSHIP_FIELD) {
+      split.projected.push({
+        ref: request.ref,
+        contentType: request.contentType,
+        fieldKey: request.fieldKey,
+        path: SEGMENT_MEMBERSHIP_FIELD,
+      });
+      continue;
+    }
+
+    const field = (segmentFields[request.contentType] || []).find(
+      (candidate) => candidate.key === request.fieldKey,
+    );
+
+    if (!field) {
+      const [prefix, ...rest] = request.fieldKey.split('.');
+      const namespace = (
+        segmentFieldNamespaces[request.contentType] || []
+      ).find((candidate) => candidate.prefix === prefix);
+
+      if (namespace && rest.length) {
+        split.namespaced.push({
+          ref: request.ref,
+          contentType: request.contentType,
+          fieldKey: request.fieldKey,
+          namespace,
+          entryKey: rest.join('.'),
+        });
+        continue;
+      }
+
+      split.undeclared.push(request.ref);
+      continue;
+    }
+
+    if (field.kind === 'derived') {
+      split.derived.push({
+        ref: request.ref,
+        contentType: request.contentType,
+        fieldKey: request.fieldKey,
+      });
+      continue;
+    }
+
+    split.projected.push({
+      ref: request.ref,
+      contentType: request.contentType,
+      fieldKey: request.fieldKey,
+      path: field.path,
+    });
+  }
+
+  return split;
+};
+
+export const readSegmentPath = (source: unknown, path: string): unknown =>
+  path.split('.').reduce<unknown>((current, segment) => {
+    if (Array.isArray(current)) {
+      const collected = current
+        .map((item) => (item as Record<string, unknown>)?.[segment])
+        .filter((item) => item !== undefined && item !== null);
+
+      return collected.length ? collected : undefined;
+    }
+
+    if (current && typeof current === 'object') {
+      return (current as Record<string, unknown>)[segment];
+    }
+
+    return undefined;
+  }, source);
+
+export const projectionForRequests = (
+  projected: SegmentProjectedRequest[],
+): Record<string, 1> =>
+  projected.reduce<Record<string, 1>>(
+    (projection, request) => {
+      projection[request.path] = 1;
+      return projection;
+    },
+    { _id: 1 },
+  );
+
+export const readNamespacedValue = (
+  document: Record<string, unknown>,
+  request: SegmentNamespacedRequest,
+): unknown => {
+  const values = document[request.namespace.path];
+
+  if (!values || typeof values !== 'object' || Array.isArray(values)) {
+    return undefined;
+  }
+
+  const bag = values as Record<string, unknown>;
+  const key = parsePropertyDataKey(request.entryKey);
+
+  if (key.kind === 'row') {
+    const rows = bag[toPropertyGroupKey(key.groupId)];
+
+    if (!Array.isArray(rows)) {
+      return undefined;
+    }
+
+    // same shape a path walk through an array produces
+    const collected = rows
+      .map((row) => (row as Record<string, unknown>)?.[key.fieldId])
+      .filter((item) => item !== undefined && item !== null);
+
+    return collected.length ? collected : undefined;
+  }
+
+  return bag[request.entryKey];
+};
+
+export const namespacePaths = (
+  namespaced: SegmentNamespacedRequest[],
+): string[] => [
+  ...new Set(namespaced.map((request) => request.namespace.path)),
+];

package/src/core-modules/segments/fieldSources.test.ts

@@ -0,0 +1,99 @@
+import { SegmentFieldMeta } from './fieldMeta';
+
+const fields: Record<string, Record<string, SegmentFieldMeta[]>> = {
+  sales: {
+    'sales:sales.deals': [
+      {
+        key: 'name',
+        label: 'Name',
+        kind: 'projected',
+        path: 'name',
+        operators: [],
+        input: 'text',
+      },
+      {
+        key: 'stageProbability',
+        label: 'Stage probability',
+        kind: 'derived',
+        operators: [],
+        input: 'text',
+        dependsOn: [
+          { fields: ['stageId'] },
+          {
+            contentType: 'sales:sales.stages',
+            fields: ['probability'],
+            via: 'stageId',
+          },
+        ],
+      },
+      {
+        key: 'pipelineId',
+        label: 'Pipeline',
+        kind: 'derived',
+        operators: [],
+        input: 'text',
+        dependsOn: [
+          { fields: ['stageId'] },
+          {
+            contentType: 'sales:sales.stages',
+            fields: ['pipelineId'],
+            via: 'stageId',
+          },
+        ],
+      },
+      {
+        key: 'boardName',
+        label: 'Board name',
+        kind: 'derived',
+        operators: [],
+        input: 'text',
+        dependsOn: [{ contentType: 'sales:sales.boards', fields: ['name'] }],
+      },
+    ],
+  },
+};
+
+jest.mock('../../utils', () => ({
+  getPlugins: async () => Object.keys(fields),
+  getPlugin: async (name: string) => ({
+    config: { meta: { segments: { segmentFields: fields[name] } } },
+  }),
+}));
+
+const { gatherSegmentFieldSources } = require('./relationRegistry');
+
+describe('gatherSegmentFieldSources', () => {
+  it('files a field under every collection its value is read from', async () => {
+    const { byField } = await gatherSegmentFieldSources();
+
+    expect(byField.get('sales:sales.deals:stageProbability')).toEqual([
+      'sales:sales.stages',
+    ]);
+    expect(byField.get('sales:sales.deals:boardName')).toEqual([
+      'sales:sales.boards',
+    ]);
+  });
+
+  it("leaves the subject's own fields out - their writes already announce them", async () => {
+    const { byField, bySource } = await gatherSegmentFieldSources();
+
+    expect(byField.has('sales:sales.deals:name')).toBe(false);
+    expect(bySource.has('sales:sales.deals')).toBe(false);
+  });
+
+  it('collapses fields that arrive through the same path', async () => {
+    const { bySource } = await gatherSegmentFieldSources();
+
+    expect(bySource.get('sales:sales.stages')).toHaveLength(1);
+  });
+
+  it('names the path back only where the declaration gives one', async () => {
+    const { bySource } = await gatherSegmentFieldSources();
+
+    expect(bySource.get('sales:sales.stages')).toEqual([
+      { subjectType: 'sales:sales.deals', via: 'stageId' },
+    ]);
+
+    expect(bySource.has('sales:sales.boards')).toBe(false);
+  });
+});

package/src/core-modules/segments/fingerprint.test.ts

@@ -0,0 +1,111 @@
+import { SegmentOperator } from './operators';
+
+import {
+  canonicalSegmentText,
+  sameSegmentDefinition,
+  segmentFingerprint,
+} from './fingerprint';
+import { SegmentNode } from './nodes';
+
+const field = (fieldKey: string, value: string): SegmentNode => ({
+  kind: 'field',
+  contentType: 'core:contacts.customers',
+  fieldKey,
+  operator: SegmentOperator.Equals,
+  value,
+});
+
+const group = (children: SegmentNode[]): SegmentNode => ({
+  kind: 'group',
+  conjunction: 'and',
+  children,
+});
+
+const print = (root: SegmentNode) =>
+  segmentFingerprint('core:contacts.customers', root);
+
+describe('segmentFingerprint', () => {
+  it('ignores the order conditions were typed in', () => {
+    const one = group([field('state', 'customer'), field('tagIds', 't-1')]);
+    const other = group([field('tagIds', 't-1'), field('state', 'customer')]);
+
+    expect(print(one)).toBe(print(other));
+  });
+
+  it('sees through a group wrapped around a single condition', () => {
+    const bare = group([field('state', 'customer')]);
+    const wrapped = group([group([field('state', 'customer')])]);
+
+    expect(print(bare)).toBe(print(wrapped));
+  });
+
+  it('ignores a group left empty by the builder', () => {
+    const withEmpty = group([field('state', 'customer'), group([])]);
+
+    expect(print(withEmpty)).toBe(print(group([field('state', 'customer')])));
+  });
+
+  it('separates definitions that differ', () => {
+    expect(print(group([field('state', 'customer')]))).not.toBe(
+      print(group([field('state', 'lead')])),
+    );
+  });
+
+  it('separates the same tree on a different content type', () => {
+    const root = group([field('state', 'customer')]);
+
+    expect(segmentFingerprint('core:contacts.customers', root)).not.toBe(
+      segmentFingerprint('core:contacts.leads', root),
+    );
+  });
+
+  it('keeps the exact text, so a hash collision cannot decide anything', () => {
+    expect(
+      canonicalSegmentText('core:contacts.customers', group([field('a', 'x')])),
+    ).not.toBe(
+      canonicalSegmentText('core:contacts.customers', group([field('a', 'y')])),
+    );
+  });
+});
+
+describe('sameSegmentDefinition', () => {
+  const TYPE = 'core:contacts.customers';
+  const state = field('state', 'customer');
+  const tag = field('tagIds', 't-1');
+
+  it('sees a reordered tree as the same question', () => {
+    expect(
+      sameSegmentDefinition(TYPE, group([state, tag]), group([tag, state])),
+    ).toBe(true);
+  });
+
+  it('sees through a group wrapped around one condition', () => {
+    expect(
+      sameSegmentDefinition(
+        TYPE,
+        group([state, tag]),
+        group([group([state]), tag]),
+      ),
+    ).toBe(true);
+  });
+
+  it('does not confuse a changed value for a reorder', () => {
+    expect(
+      sameSegmentDefinition(
+        TYPE,
+        group([state, tag]),
+        group([tag, field('state', 'lead')]),
+      ),
+    ).toBe(false);
+  });
+
+  it('separates trees about different records', () => {
+    expect(sameSegmentDefinition(TYPE, group([state]), group([state]))).toBe(
+      true,
+    );
+    expect(
+      canonicalSegmentText(TYPE, group([state])) ===
+        canonicalSegmentText('sales:sales.deals', group([state])),
+    ).toBe(false);
+  });
+});

package/src/core-modules/segments/fingerprint.ts

@@ -0,0 +1,70 @@
+import { SegmentNode } from './nodes';
+
+const stable = (value: unknown): string => {
+  if (Array.isArray(value)) {
+    return `[${value.map(stable).join(',')}]`;
+  }
+
+  if (value && typeof value === 'object') {
+    const entries = Object.entries(value as Record<string, unknown>)
+      .filter(([, entry]) => entry !== undefined)
+      .sort(([left], [right]) => left.localeCompare(right));
+
+    return `{${entries
+      .map(([key, entry]) => `${JSON.stringify(key)}:${stable(entry)}`)
+      .join(',')}}`;
+  }
+
+  return JSON.stringify(value ?? null);
+};
+
+export const canonicalSegmentNode = (node: SegmentNode): SegmentNode => {
+  if (node.kind === 'field' && node.meta) {
+    const { meta: _picker, ...asked } = node;
+
+    return asked;
+  }
+
+  if (node.kind !== 'group') {
+    return node;
+  }
+
+  const children = node.children
+    .map(canonicalSegmentNode)
+    .filter((child) => child.kind !== 'group' || child.children.length > 0)
+    .sort((left, right) => stable(left).localeCompare(stable(right)));
+
+  if (children.length === 1) {
+    return children[0];
+  }
+
+  return { ...node, children };
+};
+
+export const canonicalSegmentText = (
+  contentType: string,
+  root: SegmentNode,
+): string => `${contentType}|${stable(canonicalSegmentNode(root))}`;
+
+export const segmentFingerprint = (
+  contentType: string,
+  root: SegmentNode,
+): string => {
+  const source = canonicalSegmentText(contentType, root);
+  let hash = 0x811c9dc5;
+
+  for (let index = 0; index < source.length; index++) {
+    hash ^= source.charCodeAt(index);
+    hash = Math.imul(hash, 0x01000193) >>> 0;
+  }
+
+  return hash.toString(16).padStart(8, '0');
+};
+
+export const sameSegmentDefinition = (
+  contentType: string,
+  left: SegmentNode,
+  right: SegmentNode,
+): boolean =>
+  canonicalSegmentText(contentType, left) ===
+  canonicalSegmentText(contentType, right);

package/src/core-modules/segments/index.ts

@@ -0,0 +1,30 @@
+export * from './memberQuery';
+export * from './membership';
+export * from './mongoFilter';
+export * from './fieldBuilders';
+export * from './fieldRequests';
+export * from './plan';
+export * from './dependencies';
+export * from './evaluate';
+export * from './fingerprint';
+export * from './evaluateBatch';
+export * from './events';
+export * from './materialized';
+export * from './evaluateFieldsRouter';
+export * from './evaluateOwned';
+export * from './joinChanges';
+export * from './measureRelations';
+export * from './ownedSource';
+export * from './relationRegistry';
+export * from './timeZone';
+export * from './zonedTime';
+export * from './nodes';
+export * from './nodeRefs';
+export * from './walkNodes';
+export * from './fieldMeta';
+export * from './operators';
+export * from './setupSegmentProducers';
+export * from './subjectsReachedBy';
+export * from './types';
+export * from './utils';
+export * from './zodSchemas';

package/src/core-modules/segments/joinChanges.test.ts

@@ -0,0 +1,96 @@
+import { SegmentRelationMeta } from './relationRegistry';
+
+const plugins: Record<string, SegmentRelationMeta[]> = {
+  core: [
+    {
+      key: 'user.customers',
+      label: 'Owned customers',
+      subjectType: 'core:organization.users',
+      relatedType: 'core:contacts.customers',
+      join: { via: 'field', on: 'related', path: 'ownerId' },
+    },
+    {
+      key: 'customer.deals',
+      label: 'Deals',
+      subjectType: 'core:contacts.customers',
+      relatedType: 'sales:sales.deals',
+      join: {
+        via: 'relation',
+        subjectRecordType: 'core:customer',
+        relatedRecordType: 'sales:deal',
+      },
+    },
+  ],
+};
+
+jest.mock('../../utils', () => ({
+  getPlugins: async () => Object.keys(plugins),
+  getPlugin: async (name: string) => ({
+    config: { meta: { segments: { segmentRelations: plugins[name] } } },
+  }),
+}));
+
+const { segmentJoinChanges } = require('./joinChanges');
+const { gatherSegmentJoinPaths } = require('./relationRegistry');
+
+describe('gatherSegmentJoinPaths', () => {
+  it('files the path under whichever document carries it', async () => {
+    const paths = await gatherSegmentJoinPaths();
+
+    expect(paths.get('core:contacts.customers')).toEqual(['ownerId']);
+  });
+
+  it('leaves relation-record joins out - they have a row of their own', async () => {
+    const paths = await gatherSegmentJoinPaths();
+
+    expect(paths.has('sales:sales.deals')).toBe(false);
+  });
+});
+
+describe('segmentJoinChanges', () => {
+  it('reports both owners when one replaces the other', async () => {
+    const changes = await segmentJoinChanges('core:contacts.customers', {
+      updated: { ownerId: { prev: 'user_a', current: 'user_b' } },
+    });
+
+    expect(changes).toEqual({ ownerId: { prev: 'user_a', next: 'user_b' } });
+  });
+
+  it('reports the old owner when the field is cleared', async () => {
+    const changes = await segmentJoinChanges('core:contacts.customers', {
+      removed: { ownerId: 'user_a' },
+    });
+
+    expect(changes).toEqual({ ownerId: { prev: 'user_a', next: undefined } });
+  });
+
+  it('reports the new owner on a record that had none', async () => {
+    const changes = await segmentJoinChanges('core:contacts.customers', {
+      added: { ownerId: 'user_b' },
+    });
+
+    expect(changes).toEqual({ ownerId: { prev: undefined, next: 'user_b' } });
+  });
+
+  it('costs nothing when no join path moved', async () => {
+    const changes = await segmentJoinChanges('core:contacts.customers', {
+      updated: { primaryEmail: { prev: 'a@x.mn', current: 'b@x.mn' } },
+    });
+
+    expect(changes).toBeUndefined();
+  });
+
+  it('ignores a collection nothing joins by', async () => {
+    expect(
+      await segmentJoinChanges('sales:sales.deals', {
+        updated: { ownerId: { prev: 'user_a', current: 'user_b' } },
+      }),
+    ).toBeUndefined();
+  });
+
+  it('has nothing to report when the write carried no diff', async () => {
+    expect(
+      await segmentJoinChanges('core:contacts.customers', undefined),
+    ).toBeUndefined();
+  });
+});

package/src/core-modules/segments/joinChanges.ts

@@ -0,0 +1,67 @@
+import { gatherSegmentJoinPaths } from './relationRegistry';
+
+export type SegmentJoinChanges = Record<
+  string,
+  { prev?: unknown; next?: unknown }
+>;
+
+export type SegmentDiff = {
+  added?: Record<string, unknown>;
+  removed?: Record<string, unknown>;
+  updated?: Record<string, unknown>;
+};
+
+const at = (source: unknown, path: string): unknown =>
+  path
+    .split('.')
+    .reduce<unknown>(
+      (current, segment) =>
+        current && typeof current === 'object'
+          ? (current as Record<string, unknown>)[segment]
+          : undefined,
+      source,
+    );
+
+const isBothSides = (
+  value: unknown,
+): value is { prev?: unknown; current?: unknown } =>
+  Boolean(value) &&
+  typeof value === 'object' &&
+  ('prev' in (value as object) || 'current' in (value as object));
+
+export const segmentJoinChanges = async (
+  contentType: string,
+  diff: SegmentDiff | undefined,
+): Promise<SegmentJoinChanges | undefined> => {
+  if (!diff) {
+    return undefined;
+  }
+
+  let paths: string[];
+
+  try {
+    paths = (await gatherSegmentJoinPaths()).get(contentType) || [];
+  } catch {
+    return undefined;
+  }
+
+  const changes: SegmentJoinChanges = {};
+
+  for (const path of paths) {
+    const updated = at(diff.updated, path);
+
+    if (isBothSides(updated)) {
+      changes[path] = { prev: updated.prev, next: updated.current };
+      continue;
+    }
+
+    const added = at(diff.added, path);
+    const removed = at(diff.removed, path);
+
+    if (added !== undefined || removed !== undefined) {
+      changes[path] = { prev: removed, next: added };
+    }
+  }
+
+  return Object.keys(changes).length ? changes : undefined;
+};

package/src/core-modules/segments/materialized.ts

@@ -0,0 +1,23 @@
+import { sendWorkerQueue } from '../../utils/mq-worker';
+
+const BROADCAST_SCHEDULER_QUEUE = 'broadcast_scheduler';
+
+export type SegmentMaterializedPayload = {
+  subdomain: string;
+  kind: 'segmentMaterialized';
+  segmentId: string;
+  materializedAt: number;
+};
+
+/** Tells broadcast a segment's nightly materialization is complete. */
+export const sendSegmentMaterialized = (
+  payload: Omit<SegmentMaterializedPayload, 'kind'>,
+) =>
+  sendWorkerQueue('core', BROADCAST_SCHEDULER_QUEUE)
+    .add(BROADCAST_SCHEDULER_QUEUE, {
+      method: 'schedule',
+      payload: { ...payload, kind: 'segmentMaterialized' },
+    })
+    .catch(() => {
+      // Best effort: a missed day shows on the campaign as no run.
+    });