@erxes/api-shared 0.0.0-stage → 3.2.19-2

package/src/core-modules/segments/measureRelations.test.ts

@@ -0,0 +1,222 @@
+import { SegmentRelationMeta } from './relationRegistry';
+import { SegmentField } from './fieldBuilders';
+
+import { measureSegmentRelations } from './measureRelations';
+import { SegmentOwnedSource } from './ownedSource';
+import { SegmentRelationRequest } from './plan';
+
+const USER = 'core:organization.users';
+const DEAL = 'sales:sales.deals';
+
+type Deal = {
+  _id: string;
+  amount?: number;
+  status?: string;
+  assignedUserIds?: string[];
+};
+
+const DEALS: Deal[] = [
+  { _id: 'd1', amount: 100, status: 'open', assignedUserIds: ['u1'] },
+  { _id: 'd2', amount: 250, status: 'open', assignedUserIds: ['u1', 'u2'] },
+  { _id: 'd3', amount: 40, status: 'lost', assignedUserIds: ['u2'] },
+];
+
+const matches = (deal: Deal, query: Record<string, unknown>): boolean =>
+  Object.entries(query).every(([path, condition]) => {
+    const value = (deal as Record<string, unknown>)[path];
+    const held = Array.isArray(value) ? value : [value];
+
+    if (condition && typeof condition === 'object') {
+      const { $in: any, $eq: exact } = condition as {
+        $in?: unknown[];
+        $eq?: unknown;
+      };
+
+      if (any) {
+        return held.some((item) => any.includes(item));
+      }
+
+      return exact === undefined || held.includes(exact);
+    }
+
+    return held.includes(condition);
+  });
+
+const dealSource = (): SegmentOwnedSource => ({
+  find: async (query) => DEALS.filter((deal) => matches(deal, query)),
+
+  aggregate: async (pipeline) => {
+    const [{ $match: match }, , , { $group: group }] = pipeline as [
+      { $match: Record<string, unknown> },
+      unknown,
+      { $match: Record<string, unknown> },
+      { $group: { _id: string; measured: Record<string, unknown> } },
+    ];
+
+    const path = group._id.replace('$', '');
+    const rows = new Map<string, Deal[]>();
+
+    for (const deal of DEALS.filter((candidate) => matches(candidate, match))) {
+      for (const id of (deal as Record<string, unknown>)[path] as string[]) {
+        rows.set(id, [...(rows.get(id) || []), deal]);
+      }
+    }
+
+    const accumulator = Object.keys(group.measured)[0];
+    const source = String(Object.values(group.measured)[0]).replace('$', '');
+
+    return [...rows].map(([id, deals]) => ({
+      _id: id,
+      measured:
+        accumulator === '$sum' && source === '1'
+          ? deals.length
+          : deals.reduce(
+              (n, deal) =>
+                n + Number((deal as Record<string, unknown>)[source] || 0),
+              0,
+            ),
+    }));
+  },
+});
+
+const context = (relations: SegmentRelationMeta[]) => ({
+  sourceFor: (contentType: string) =>
+    contentType === DEAL ? dealSource() : null,
+  relations,
+  fields: {
+    [DEAL]: [
+      SegmentField.text({ key: 'status', label: 'Status' }),
+      SegmentField.number({ key: 'amount', label: 'Amount' }),
+    ],
+  },
+});
+
+const request = (
+  measure: SegmentRelationRequest['measure'],
+  extra: Partial<SegmentRelationRequest> = {},
+): SegmentRelationRequest => ({
+  kind: 'relation',
+  ref: 'r1',
+  relationKey: 'user.deals',
+  measure,
+  ...extra,
+});
+
+const byField: SegmentRelationMeta[] = [
+  {
+    key: 'user.deals',
+    label: 'Deals',
+    subjectType: USER,
+    relatedType: DEAL,
+    join: { via: 'field', on: 'related', path: 'assignedUserIds' },
+  },
+];
+
+const byRelation: SegmentRelationMeta[] = [
+  {
+    key: 'user.deals',
+    label: 'Deals',
+    subjectType: USER,
+    relatedType: DEAL,
+    join: {
+      via: 'relation',
+      subjectRecordType: 'core:user',
+      relatedRecordType: 'sales:deal',
+    },
+  },
+];
+
+const EDGES = { u1: ['d1', 'd2'], u2: ['d2', 'd3'] };
+
+describe('measureSegmentRelations', () => {
+  it('counts the same either way the records are joined', async () => {
+    const joined = await measureSegmentRelations(
+      context(byField),
+      USER,
+      ['u1', 'u2'],
+      [request({ op: 'count' })],
+    );
+
+    const edged = await measureSegmentRelations(
+      context(byRelation),
+      USER,
+      ['u1', 'u2'],
+      [request({ op: 'count' }, { edges: EDGES })],
+    );
+
+    expect(joined.values).toEqual({ u1: { r1: 2 }, u2: { r1: 2 } });
+    expect(edged.values).toEqual(joined.values);
+  });
+
+  it('narrows both joins by the same predicate', async () => {
+    const child = {
+      kind: 'field' as const,
+      contentType: DEAL,
+      fieldKey: 'status',
+      operator: 'e' as never,
+      value: 'open',
+    };
+
+    const joined = await measureSegmentRelations(
+      context(byField),
+      USER,
+      ['u1', 'u2'],
+      [request({ op: 'count' }, { child })],
+    );
+
+    const edged = await measureSegmentRelations(
+      context(byRelation),
+      USER,
+      ['u1', 'u2'],
+      [request({ op: 'count' }, { child, edges: EDGES })],
+    );
+
+    expect(joined.values).toEqual({ u1: { r1: 2 }, u2: { r1: 1 } });
+    expect(edged.values).toEqual(joined.values);
+  });
+
+  it('reduces a numeric field over the batch', async () => {
+    const { values } = await measureSegmentRelations(
+      context(byField),
+      USER,
+      ['u1', 'u2'],
+      [request({ op: 'sum', fieldKey: 'amount' })],
+    );
+
+    expect(values).toEqual({ u1: { r1: 350 }, u2: { r1: 290 } });
+  });
+
+  it('is undecidable when the edges were never resolved', async () => {
+    const { values, unavailable } = await measureSegmentRelations(
+      context(byRelation),
+      USER,
+      ['u1'],
+      [request({ op: 'count' })],
+    );
+
+    expect(values).toEqual({});
+    expect(unavailable).toEqual(['r1']);
+  });
+
+  it('refuses a measure over a field that is not a stored number', async () => {
+    const { unavailable } = await measureSegmentRelations(
+      context(byField),
+      USER,
+      ['u1'],
+      [request({ op: 'sum', fieldKey: 'status' })],
+    );
+
+    expect(unavailable).toEqual(['r1']);
+  });
+
+  it('reports a relation this service does not own', async () => {
+    const { unavailable } = await measureSegmentRelations(
+      context([]),
+      USER,
+      ['u1'],
+      [request({ op: 'count' })],
+    );
+
+    expect(unavailable).toEqual(['r1']);
+  });
+});

package/src/core-modules/segments/measureRelations.ts

@@ -0,0 +1,322 @@
+import { SegmentRelationMeta } from './relationRegistry';
+import { SegmentFieldMeta } from './fieldMeta';
+import { readSegmentPath } from './fieldRequests';
+import { compileSegmentMongoFilter, SegmentMongoFilter } from './mongoFilter';
+import { SegmentMeasure, SegmentNode } from './nodes';
+import { SegmentOwnedSource, SegmentSourceResolver } from './ownedSource';
+import { SegmentRelationRequest } from './plan';
+
+export type MeasuredRelations = {
+  values: Record<string, Record<string, unknown>>;
+  unavailable: string[];
+};
+
+export type MeasureRelationsContext = {
+  sourceFor: SegmentSourceResolver;
+  relations: SegmentRelationMeta[];
+  fields: Record<string, SegmentFieldMeta[]>;
+  timeZone?: string;
+  rewritePredicate?: (node: SegmentNode) => Promise<SegmentNode>;
+};
+
+type MeasurePlan =
+  | { op: 'exists' }
+  | { op: 'none' }
+  | { op: 'count' }
+  | { op: 'sum' | 'avg' | 'min' | 'max'; path: string };
+
+const planMeasure = (
+  measure: SegmentMeasure,
+  fields: SegmentFieldMeta[],
+): MeasurePlan | null => {
+  if (
+    measure.op === 'exists' ||
+    measure.op === 'none' ||
+    measure.op === 'count'
+  ) {
+    return { op: measure.op };
+  }
+
+  const field = fields.find((candidate) => candidate.key === measure.fieldKey);
+
+  if (!field || field.kind !== 'projected' || field.input !== 'number') {
+    return null;
+  }
+
+  return { op: measure.op, path: field.path };
+};
+
+const total = (numbers: number[]) =>
+  numbers.reduce((sum, number) => sum + number, 0);
+
+const foldMeasure = (plan: MeasurePlan, records: unknown[]): unknown => {
+  if (plan.op === 'exists' || plan.op === 'none') {
+    return records.length > 0;
+  }
+
+  if (plan.op === 'count') {
+    return records.length;
+  }
+
+  const numbers = records
+    .map((record) => Number(readSegmentPath(record, plan.path)))
+    .filter((number) => !Number.isNaN(number));
+
+  if (plan.op === 'sum') {
+    return total(numbers);
+  }
+
+  if (!numbers.length) {
+    return undefined;
+  }
+
+  if (plan.op === 'avg') {
+    return total(numbers) / numbers.length;
+  }
+
+  return plan.op === 'min' ? Math.min(...numbers) : Math.max(...numbers);
+};
+
+const projectionFor = (plan: MeasurePlan): Record<string, 1> =>
+  plan.op === 'exists' || plan.op === 'none' || plan.op === 'count'
+    ? { _id: 1 }
+    : { _id: 1, [plan.path]: 1 };
+
+type Accumulator =
+  | { $sum: 1 | string }
+  | { $avg: string }
+  | { $min: string }
+  | { $max: string };
+
+const accumulatorFor = (plan: MeasurePlan): Accumulator => {
+  if (plan.op === 'exists' || plan.op === 'none' || plan.op === 'count') {
+    return { $sum: 1 };
+  }
+
+  const path = `$${plan.path}`;
+
+  return plan.op === 'sum'
+    ? { $sum: path }
+    : plan.op === 'avg'
+      ? { $avg: path }
+      : plan.op === 'min'
+        ? { $min: path }
+        : { $max: path };
+};
+
+const measureOverEdges = async (
+  source: SegmentOwnedSource,
+  subjectIds: string[],
+  edges: Record<string, string[]> | undefined,
+  plan: MeasurePlan,
+  filter: SegmentMongoFilter,
+): Promise<Map<string, unknown> | null> => {
+  if (!edges) {
+    return null;
+  }
+
+  const relatedIds = [
+    ...new Set(subjectIds.flatMap((subjectId) => edges[subjectId] || [])),
+  ];
+
+  const records = relatedIds.length
+    ? await source.find(
+        { ...(source.baseQuery || {}), ...filter, _id: { $in: relatedIds } },
+        projectionFor(plan),
+      )
+    : [];
+
+  const byId = new Map<string, unknown>(
+    records.map((record) => [String(record._id), record]),
+  );
+
+  const measured = new Map<string, unknown>();
+
+  for (const subjectId of subjectIds) {
+    const passing = (edges[subjectId] || [])
+      .map((relatedId) => byId.get(relatedId))
+      .filter((record) => record !== undefined);
+
+    measured.set(subjectId, foldMeasure(plan, passing));
+  }
+
+  return measured;
+};
+
+const measureOverJoin = async (
+  source: SegmentOwnedSource,
+  subjectIds: string[],
+  joinPath: string,
+  plan: MeasurePlan,
+  filter: SegmentMongoFilter,
+): Promise<Map<string, unknown> | null> => {
+  if (!source.aggregate) {
+    return null;
+  }
+
+  const rows = await source.aggregate([
+    {
+      $match: {
+        ...(source.baseQuery || {}),
+        ...filter,
+        [joinPath]: { $in: subjectIds },
+      },
+    },
+    { $unwind: `$${joinPath}` },
+    { $match: { [joinPath]: { $in: subjectIds } } },
+    { $group: { _id: `$${joinPath}`, measured: accumulatorFor(plan) } },
+  ]);
+
+  const aggregated = new Map(
+    rows.map((row) => [String(row._id), row.measured as number]),
+  );
+
+  const measured = new Map<string, unknown>();
+  const empty = foldMeasure(plan, []);
+
+  for (const subjectId of subjectIds) {
+    const row = aggregated.get(subjectId);
+
+    if (row === undefined) {
+      measured.set(subjectId, empty);
+      continue;
+    }
+
+    measured.set(
+      subjectId,
+      plan.op === 'exists' || plan.op === 'none' ? Boolean(row) : row,
+    );
+  }
+
+  return measured;
+};
+
+const edgesFromSubjects = async (
+  subjects: SegmentOwnedSource,
+  subjectIds: string[],
+  path: string,
+): Promise<Record<string, string[]>> => {
+  const records = await subjects.find(
+    { ...(subjects.baseQuery || {}), _id: { $in: subjectIds } },
+    { _id: 1, [path.split('.')[0]]: 1 },
+  );
+
+  const edges: Record<string, string[]> = {};
+
+  for (const record of records) {
+    const value = readSegmentPath(record, path);
+
+    edges[String(record._id)] = (Array.isArray(value) ? value : [value]).filter(
+      (id): id is string => typeof id === 'string' && id.length > 0,
+    );
+  }
+
+  return edges;
+};
+
+const predicateFilter = async (
+  context: MeasureRelationsContext,
+  relatedType: string,
+  child?: SegmentNode,
+): Promise<SegmentMongoFilter | null> => {
+  if (!child) {
+    return {};
+  }
+
+  const node = context.rewritePredicate
+    ? await context.rewritePredicate(child)
+    : child;
+
+  const { filter, unsupported } = compileSegmentMongoFilter(node, {
+    fields: context.fields[relatedType] || [],
+    timeZone: context.timeZone,
+  });
+
+  return unsupported.length ? null : filter;
+};
+
+export const measureSegmentRelations = async (
+  context: MeasureRelationsContext,
+  subjectType: string,
+  subjectIds: string[],
+  requests: SegmentRelationRequest[],
+): Promise<MeasuredRelations> => {
+  const values: Record<string, Record<string, unknown>> = {};
+  const unavailable: string[] = [];
+
+  for (const request of requests) {
+    const relation = context.relations.find(
+      (candidate) => candidate.key === request.relationKey,
+    );
+
+    const related = relation ? context.sourceFor(relation.relatedType) : null;
+
+    const plan =
+      relation && related
+        ? planMeasure(
+            request.measure,
+            context.fields[relation.relatedType] || [],
+          )
+        : null;
+
+    const filter =
+      relation && plan
+        ? await predicateFilter(context, relation.relatedType, request.child)
+        : null;
+
+    if (!relation || !related || !plan || !filter) {
+      unavailable.push(request.ref);
+      continue;
+    }
+
+    let measured: Map<string, unknown> | null;
+
+    if (relation.join.via === 'relation') {
+      measured = await measureOverEdges(
+        related,
+        subjectIds,
+        request.edges,
+        plan,
+        filter,
+      );
+    } else if (relation.join.on === 'related') {
+      measured = await measureOverJoin(
+        related,
+        subjectIds,
+        relation.join.path,
+        plan,
+        filter,
+      );
+    } else {
+      const subjects = context.sourceFor(subjectType);
+
+      measured = subjects
+        ? await measureOverEdges(
+            related,
+            subjectIds,
+            await edgesFromSubjects(subjects, subjectIds, relation.join.path),
+            plan,
+            filter,
+          )
+        : null;
+    }
+
+    if (!measured) {
+      unavailable.push(request.ref);
+      continue;
+    }
+
+    for (const [subjectId, value] of measured) {
+      if (value === undefined) {
+        continue;
+      }
+
+      values[subjectId] = {
+        ...(values[subjectId] || {}),
+        [request.ref]: value,
+      };
+    }
+  }
+
+  return { values, unavailable };
+};

package/src/core-modules/segments/memberQuery.test.ts

@@ -0,0 +1,60 @@
+import {
+  MAX_SEGMENT_PAGE_SIZE,
+  segmentPage,
+  segmentPageFilter,
+  segmentPageSize,
+} from './memberQuery';
+
+describe('segmentPageFilter', () => {
+  it('leaves the first page untouched', () => {
+    expect(segmentPageFilter({ status: 'won' })).toEqual({ status: 'won' });
+  });
+
+  it('pages on _id rather than skipping', () => {
+    expect(segmentPageFilter({ status: 'won' }, { cursor: 'd-9' })).toEqual({
+      $and: [{ status: 'won' }, { _id: { $gt: 'd-9' } }],
+    });
+  });
+
+  it('does not wrap an empty filter in $and', () => {
+    expect(segmentPageFilter({}, { cursor: 'd-9' })).toEqual({
+      _id: { $gt: 'd-9' },
+    });
+  });
+
+  it('narrows to a known set, which is how one record is checked', () => {
+    expect(segmentPageFilter({ status: 'won' }, { ids: ['d-1'] })).toEqual({
+      $and: [{ status: 'won' }, { _id: { $in: ['d-1'] } }],
+    });
+  });
+
+  it('combines a known set with a cursor', () => {
+    expect(
+      segmentPageFilter({}, { ids: ['d-1', 'd-2'], cursor: 'd-1' }),
+    ).toEqual({
+      $and: [{ _id: { $in: ['d-1', 'd-2'] } }, { _id: { $gt: 'd-1' } }],
+    });
+  });
+});
+
+describe('segmentPageSize', () => {
+  it('defaults and caps', () => {
+    expect(segmentPageSize()).toBe(1000);
+    expect(segmentPageSize(50)).toBe(50);
+    expect(segmentPageSize(999999)).toBe(MAX_SEGMENT_PAGE_SIZE);
+  });
+});
+
+describe('segmentPage', () => {
+  it('reports no next page when the over-fetch came back short', () => {
+    expect(segmentPage(['a', 'b'], 3)).toEqual({ ids: ['a', 'b'] });
+    expect(segmentPage(['a', 'b', 'c'], 3)).toEqual({ ids: ['a', 'b', 'c'] });
+  });
+
+  it('trims the extra row and points the cursor at the last kept id', () => {
+    expect(segmentPage(['a', 'b', 'c', 'd'], 3)).toEqual({
+      ids: ['a', 'b', 'c'],
+      nextCursor: 'c',
+    });
+  });
+});

package/src/core-modules/segments/memberQuery.ts

@@ -0,0 +1,45 @@
+import { SegmentMongoFilter } from './mongoFilter';
+
+export const DEFAULT_SEGMENT_PAGE_SIZE = 1000;
+export const MAX_SEGMENT_PAGE_SIZE = 10000;
+
+export const segmentPageSize = (limit?: number): number =>
+  Math.min(limit || DEFAULT_SEGMENT_PAGE_SIZE, MAX_SEGMENT_PAGE_SIZE);
+
+export const segmentPageFilter = (
+  filter: SegmentMongoFilter,
+  { cursor, ids }: { cursor?: string; ids?: string[] } = {},
+): SegmentMongoFilter => {
+  const narrowing: SegmentMongoFilter[] = [];
+
+  if (ids) {
+    narrowing.push({ _id: { $in: ids } });
+  }
+
+  if (cursor) {
+    narrowing.push({ _id: { $gt: cursor } });
+  }
+
+  if (!narrowing.length) {
+    return filter;
+  }
+
+  return Object.keys(filter).length
+    ? { $and: [filter, ...narrowing] }
+    : narrowing.length === 1
+      ? narrowing[0]
+      : { $and: narrowing };
+};
+
+export const segmentPage = (
+  ids: string[],
+  limit: number,
+): { ids: string[]; nextCursor?: string } => {
+  if (ids.length <= limit) {
+    return { ids };
+  }
+
+  const page = ids.slice(0, limit);
+
+  return { ids: page, nextCursor: page[page.length - 1] };
+};