@erxes/api-shared 0.0.0-stage → 3.2.19-1

package/src/utils/mongo/generate-models.ts

@@ -0,0 +1,97 @@
+import { createScopedEventHandlers } from '../../core-modules/common/eventHandlers/generateEventHandlers';
+import mongoose, { Connection } from 'mongoose';
+import {
+  coreModelOrganizations,
+  getSaasCoreConnection,
+} from '../saas/saas-mongo-connection';
+import { getEnv, getSubdomain } from '../utils';
+import { connect } from './mongo-connection';
+import type { ScopedEventHandlers } from '../../core-modules/common/eventHandlers/types';
+
+const returnGeneratedModels = async <IModels>(
+  connection: Connection,
+  loadClasses: (
+    db: mongoose.Connection,
+    subdomain: string,
+    eventHandlers: ScopedEventHandlers,
+  ) => IModels | Promise<IModels>,
+  subdomain: string,
+  context?: Record<string, any>,
+) => {
+  const eventHandlers = createScopedEventHandlers(subdomain, context);
+  return await loadClasses(connection, subdomain, eventHandlers);
+};
+
+export const createGenerateModels = <IModels>(
+  loadClasses: (
+    db: mongoose.Connection,
+    subdomain: string,
+    eventHandlers: ScopedEventHandlers,
+  ) => IModels | Promise<IModels>,
+): ((
+  hostnameOrSubdomain: string,
+  context?: Record<string, any>,
+) => Promise<IModels>) => {
+  const VERSION = getEnv({ name: 'VERSION', defaultValue: 'os' });
+
+  connect();
+
+  if (VERSION && VERSION !== 'saas') {
+    return async function genereteModels(
+      hostnameOrSubdomain: string,
+      context?: Record<string, any>,
+    ): Promise<IModels> {
+      return await returnGeneratedModels(
+        mongoose.connection,
+        loadClasses,
+        hostnameOrSubdomain,
+        context,
+      );
+    };
+  } else {
+    return async function genereteModels(
+      hostnameOrSubdomain = '',
+      context?: Record<string, any>,
+    ): Promise<IModels> {
+      let subdomain: string = hostnameOrSubdomain;
+
+      if (!subdomain) {
+        throw new Error(`Subdomain is \`${subdomain}\``);
+      }
+
+      // means hostname
+      if (subdomain?.includes('.')) {
+        subdomain = getSubdomain(hostnameOrSubdomain);
+      }
+
+      await getSaasCoreConnection();
+
+      const organization = await coreModelOrganizations.findOne({ subdomain });
+
+      if (!organization) {
+        throw new Error(
+          `Organization with subdomain = ${subdomain} is not found`,
+        );
+      }
+
+      const DB_NAME = getEnv({ name: 'DB_NAME' });
+      const GE_MONGO_URL = (DB_NAME || 'erxes_<organizationId>').replace(
+        '<organizationId>',
+        organization._id,
+      );
+
+      const tenantCon = mongoose.connection.useDb(GE_MONGO_URL, {
+        // so that conn.model method can use cached connection
+        useCache: true,
+        noListener: true,
+      });
+
+      return await returnGeneratedModels(
+        tenantCon,
+        loadClasses,
+        subdomain,
+        context,
+      );
+    };
+  }
+};

package/src/utils/mongo/index.ts

@@ -0,0 +1,5 @@
+export * from './mongoose-types';
+export * from './mongoose-utils';
+export * from './mongo-connection';
+export * from './generate-models';
+export * from './search-tokens';

package/src/utils/mongo/mongo-connection.ts

@@ -0,0 +1,42 @@
+import mongoose from 'mongoose';
+
+const { MONGO_URL = 'mongodb://127.0.0.1:27017/erxes?directConnection=true' } =
+  process.env;
+
+export const mongooseConnectionOptions: mongoose.ConnectOptions = {
+  family: 4,
+};
+
+mongoose.connection
+  .on('connected', () => {
+    console.log(`Connected to the database: ${MONGO_URL}`);
+  })
+  .on('disconnected', (...props) => {
+    console.log(`Disconnected from the database: ${MONGO_URL}`);
+
+    process.exit(1);
+  })
+  .on('error', (error) => {
+    console.error(`Database connection error: ${MONGO_URL} ${error}`);
+
+    process.exit(1);
+  })
+  .on('close', () => {});
+
+export async function connect(): Promise<mongoose.Connection> {
+  if (!MONGO_URL) {
+    throw new Error('MONGO_URL is not defined');
+  }
+
+  await mongoose.connect(MONGO_URL, mongooseConnectionOptions);
+  return mongoose.connection;
+}
+
+export async function closeMongooose() {
+  try {
+    await mongoose.connection.close();
+    console.log('Mongoose connection disconnected ');
+  } catch (e) {
+    console.error(e);
+  }
+}

package/src/utils/mongo/mongoose-types.ts

@@ -0,0 +1,26 @@
+import { nanoid } from 'nanoid';
+
+export const mongooseStringRandomId = {
+  type: String,
+  default: () => nanoid(),
+} as const;
+
+/**
+ * Allows `null | undefined`.
+ * But if it's a `string`, it must contain atleast one non whitespace character.
+ */
+export const mongooseStringNonBlank = {
+  type: String,
+  validate: /\S+?/,
+} as const;
+
+export const mongoStringRequired = {
+  type: String,
+  required: true,
+} as const;
+
+export const mongooseStringRequiredNonBlank = {
+  type: String,
+  validate: /\S+?/,
+  required: true,
+} as const;

package/src/utils/mongo/mongoose-utils.ts

@@ -0,0 +1,329 @@
+import mongoose, {
+  Document,
+  FilterQuery,
+  PipelineStage,
+  Schema,
+} from 'mongoose';
+import { nanoid } from 'nanoid';
+
+import {
+  buildCursorQuery,
+  CursorPaginateAggregationParams,
+  CursorPaginateParams,
+  CursorResult,
+  encodeCursor,
+  PageInfo,
+} from './cursor-util';
+import { mongooseStringRandomId } from './mongoose-types';
+import {
+  configureSchemaSearchTokens,
+  ISchemaWrapperOptions,
+} from './search-tokens';
+
+export interface IOrderInput {
+  _id: string;
+  order: number;
+}
+
+export const mongooseField = (options: any) => {
+  const { pkey, type, optional } = options;
+
+  if (type === String && !pkey && !optional) {
+    options.validate = /\S+/;
+  }
+
+  if (pkey) {
+    options.type = String;
+    options.default = () => nanoid();
+  }
+
+  return options;
+};
+
+export const updateMongoDocumentOrder = async (
+  collection: any,
+  orders: IOrderInput[],
+) => {
+  if (orders.length === 0) {
+    return [];
+  }
+
+  const ids: string[] = [];
+  const bulkOps: Array<{
+    updateOne: {
+      query: { _id: string };
+      update: { order: number };
+    };
+  }> = [];
+
+  for (const { _id, order } of orders) {
+    ids.push(_id);
+
+    const selector: { order: number } = { order };
+
+    bulkOps.push({
+      updateOne: {
+        query: { _id },
+        update: selector,
+      },
+    });
+  }
+
+  await collection.bulkWrite(bulkOps);
+
+  return collection.find({ _id: { $in: ids } }).sort({ order: 1 });
+};
+
+export const defaultPaginate = (
+  collection: mongoose.Query<any, any>,
+  params: {
+    ids?: string[];
+    page?: number;
+    perPage?: number;
+    excludeIds?: boolean;
+  },
+) => {
+  const { page = 1, perPage = 20, ids, excludeIds } = params || { ids: null };
+
+  const _page = Number(page || '1');
+  const _limit = Number(perPage || '20');
+
+  if (ids && ids.length > 0) {
+    return excludeIds ? collection.limit(_limit) : collection;
+  }
+
+  return collection.limit(_limit).skip((_page - 1) * _limit);
+};
+
+export const cursorPaginate = async <T extends Document>({
+  model,
+  params,
+  query,
+  formatter,
+}: CursorPaginateParams<T>): Promise<CursorResult<T>> => {
+  const { limit = 20, cursor, direction = 'forward', orderBy = {} } = params;
+
+  if (limit < 1 || limit > 100) {
+    throw new Error('Limit must be between 1 and 100');
+  }
+
+  const baseQuery: FilterQuery<T> = cursor
+    ? {
+        $and: [
+          query || {},
+          buildCursorQuery(cursor, orderBy, direction, formatter),
+        ],
+      }
+    : { ...(query || {}) };
+
+  const sortFields = Object.keys(orderBy);
+  const sortOrder: Record<string, 1 | -1> = {};
+
+  for (const [field, order] of Object.entries(orderBy)) {
+    const normalizedOrder = order === 1 || order === 'asc' ? 1 : -1;
+    sortOrder[field] =
+      direction === 'forward'
+        ? normalizedOrder
+        : normalizedOrder === 1
+          ? -1
+          : 1;
+  }
+
+  sortOrder._id = (direction === 'forward' ? 1 : -1) as 1 | -1;
+
+  const [items, totalCount] = await Promise.all([
+    model
+      .find(baseQuery as FilterQuery<T>)
+      .sort(sortOrder)
+      .limit(limit + 1)
+      .lean(),
+    model.countDocuments(query as FilterQuery<T>),
+  ]);
+
+  const hasMore = items.length > limit;
+  let list = hasMore ? items.slice(0, limit) : items;
+
+  if (direction === 'backward') {
+    list = list.reverse();
+  }
+
+  const startCursor =
+    list.length > 0 ? encodeCursor(list[0], sortFields) : null;
+  const endCursor =
+    list.length > 0 ? encodeCursor(list[list.length - 1], sortFields) : null;
+
+  const pageInfo: PageInfo = {
+    hasNextPage: direction === 'forward' ? hasMore : Boolean(cursor),
+    hasPreviousPage: direction === 'backward' ? hasMore : Boolean(cursor),
+    startCursor,
+    endCursor,
+  };
+
+  return {
+    list: list as T[],
+    totalCount,
+    pageInfo,
+  };
+};
+
+export async function cursorPaginateAggregation<T>({
+  model,
+  pipeline = [],
+  params,
+  formatter,
+  uniqConcatFields,
+}: CursorPaginateAggregationParams<T>): Promise<CursorResult<T>> {
+  const { limit = 20, cursor, direction = 'forward', orderBy = {} } = params;
+
+  if (limit < 1 || limit > 100) {
+    throw new Error('Limit must be between 1 and 100');
+  }
+
+  // --- merge default order ---
+  const sortFields = Object.keys(orderBy);
+
+  // --- cursor match ---
+  const cursorMatch = cursor
+    ? buildCursorQuery(cursor, orderBy, direction, formatter)
+    : null;
+
+  // --- build aggregation pipeline ---
+  const aggPipeline: PipelineStage[] = [...pipeline];
+
+  if (cursorMatch) {
+    aggPipeline.push({ $match: cursorMatch });
+  }
+
+  // --- sorting ---
+  aggPipeline.push({ $sort: orderBy as any });
+
+  // --- unwind if needed ---
+  if (uniqConcatFields) {
+    // add composite field for cursor
+    aggPipeline.push({
+      $addFields: {
+        compositeField: {
+          $concat: uniqConcatFields.map((field) => ({ $toString: field })),
+        },
+      },
+    });
+  }
+
+  // --- limit + 1 for hasNextPage ---
+  aggPipeline.push({ $limit: limit + 1 });
+
+  // --- execute aggregation ---
+  const listRaw = await model.aggregate(aggPipeline);
+
+  // --- filtered totalCount ---
+  const countPipeline: PipelineStage[] = [
+    ...pipeline,
+    { $count: 'totalCount' },
+  ];
+  const countResult = await model.aggregate(countPipeline);
+  const totalCount = countResult[0]?.totalCount ?? 0;
+
+  // --- slice list for hasNextPage ---
+  const hasMore = listRaw.length > limit;
+  let list = hasMore ? listRaw.slice(0, limit) : listRaw;
+
+  if (direction === 'backward') {
+    list = list.reverse();
+  }
+
+  // --- cursors ---
+  const startCursor =
+    list.length > 0 ? encodeCursor(list[0], sortFields) : null;
+  const endCursor =
+    list.length > 0 ? encodeCursor(list[list.length - 1], sortFields) : null;
+
+  const pageInfo: PageInfo = {
+    hasNextPage: direction === 'forward' ? hasMore : Boolean(cursor),
+    hasPreviousPage: direction === 'backward' ? hasMore : Boolean(cursor),
+    startCursor,
+    endCursor,
+  };
+
+  return {
+    list,
+    totalCount,
+    pageInfo,
+  };
+}
+
+export const checkCollectionCodeDuplication = async (
+  collection: any,
+  code: string,
+) => {
+  if (code.includes('/')) {
+    throw new Error('The "/" character is not allowed in the code');
+  }
+
+  const category = await collection.findOne({
+    code,
+  });
+
+  if (category) {
+    throw new Error('Code must be unique');
+  }
+};
+
+export const schemaWrapper = (
+  schema: Schema,
+  options: ISchemaWrapperOptions = {},
+) => {
+  schema.add({ _id: mongooseStringRandomId });
+  schema.add({ processId: { type: String, optional: true } });
+
+  // Segment membership is written here by the segmentation worker, for whatever
+  // content types segments are built against. Declared once for every schema
+  // rather than per model: a field that has to be added and indexed by hand on
+  // each of a hundred schemas is a field that will be missing from some of them.
+  //
+  // `default: undefined` keeps the field absent until membership is actually
+  // written, which is what lets the sparse index stay empty - and cost nothing -
+  // on the collections no segment ever targets.
+  schema.add({
+    segmentIds: {
+      type: [String],
+      optional: true,
+      default: undefined,
+      index: true,
+      sparse: true,
+    },
+  });
+
+  // Provenance for records nobody typed in: written by whatever produced them
+  // — a campaign, an automation — and never by a person. Declared here for the
+  // same reason as `segmentIds` above: a field that has to be remembered on a
+  // hundred schemas is a field that will be missing from some of them.
+  //
+  // `default: undefined` keeps it absent until something writes it, so the
+  // sparse index costs nothing on the collections that are only ever filled in
+  // by hand.
+  schema.add({
+    createdVia: {
+      type: {
+        source: { type: String },
+        sourceId: { type: String },
+        sourceName: { type: String, optional: true },
+        runId: { type: String, optional: true },
+        actorId: { type: String, optional: true },
+      },
+      optional: true,
+      default: undefined,
+      _id: false,
+    },
+  });
+
+  schema.index(
+    { 'createdVia.sourceId': 1 },
+    { sparse: true, name: 'createdVia_sourceId' },
+  );
+
+  if (options.search) {
+    configureSchemaSearchTokens(schema, options.search);
+  }
+
+  return schema;
+};

package/src/utils/mongo/search-tokens.ts

@@ -0,0 +1,193 @@
+import { FilterQuery, Schema } from 'mongoose';
+
+export type SearchTokenMode = 'exact' | 'word' | 'prefix' | 'word-prefix';
+
+export interface ISearchTokenFieldConfig {
+  path: string;
+  mode: SearchTokenMode;
+  minLength?: number;
+}
+
+export interface ISearchTokenConfig {
+  enabled: boolean;
+  fields: ISearchTokenFieldConfig[];
+  tokenField?: string;
+}
+
+export interface ISchemaWrapperOptions {
+  search?: ISearchTokenConfig;
+}
+
+const DEFAULT_MIN_LENGTH = 3;
+const DEFAULT_TOKEN_FIELD = 'searchTokens';
+const TOKEN_MODE_KEYS: Record<SearchTokenMode, string> = {
+  exact: 'e',
+  word: 'w',
+  prefix: 'p',
+  'word-prefix': 'wp',
+};
+
+const searchConfigurations = new WeakMap<Schema, ISearchTokenConfig>();
+
+const normalizeValue = (value: string) =>
+  value.normalize('NFKC').toLocaleLowerCase().trim().replace(/\s+/g, ' ');
+
+export const splitSearchWords = (value: string): string[] =>
+  normalizeValue(value).match(/[\p{L}\p{N}]+/gu) ?? [];
+
+const normalizeExactValue = (value: string) => splitSearchWords(value).join('');
+
+const getValuesAtPath = (source: unknown, path: string): string[] => {
+  const value = path.split('.').reduce<unknown>((current, key) => {
+    if (current === null || typeof current !== 'object') {
+      return undefined;
+    }
+
+    return (current as Record<string, unknown>)[key];
+  }, source);
+
+  const values = Array.isArray(value) ? value : [value];
+
+  return values
+    .filter(
+      (item): item is string | number =>
+        typeof item === 'string' || typeof item === 'number',
+    )
+    .map(String)
+    .filter(Boolean);
+};
+
+const formatToken = (field: ISearchTokenFieldConfig, value: string): string =>
+  `${field.path}:${TOKEN_MODE_KEYS[field.mode]}:${value}`;
+
+const generateFieldTokens = (
+  value: string,
+  field: ISearchTokenFieldConfig,
+): string[] => {
+  const minLength = field.minLength ?? DEFAULT_MIN_LENGTH;
+
+  if (field.mode === 'exact') {
+    const exactValue = normalizeExactValue(value);
+    return exactValue.length >= minLength
+      ? [formatToken(field, exactValue)]
+      : [];
+  }
+
+  const words =
+    field.mode === 'prefix' ? [normalizeValue(value)] : splitSearchWords(value);
+
+  if (field.mode === 'word') {
+    return words
+      .filter((word) => word.length >= minLength)
+      .map((word) => formatToken(field, word));
+  }
+
+  return words.flatMap((word) => {
+    const tokens: string[] = [];
+
+    for (let length = minLength; length <= word.length; length += 1) {
+      tokens.push(formatToken(field, word.slice(0, length)));
+    }
+
+    return tokens;
+  });
+};
+
+export const generateConfiguredSearchTokens = (
+  source: unknown,
+  config: ISearchTokenConfig,
+): string[] => {
+  if (!config.enabled) {
+    return [];
+  }
+
+  const tokens = config.fields.flatMap((field) =>
+    getValuesAtPath(source, field.path).flatMap((value) =>
+      generateFieldTokens(value, field),
+    ),
+  );
+
+  return [...new Set(tokens)];
+};
+
+export const buildSearchTokenFilter = <T>(
+  searchValue: string,
+  config: ISearchTokenConfig,
+): FilterQuery<T> => {
+  const words = splitSearchWords(searchValue);
+
+  if (!config.enabled || words.length === 0) {
+    return { _id: { $in: [] } } as FilterQuery<T>;
+  }
+
+  const wordClauses = words.map((word) => {
+    const candidates = config.fields.flatMap((field) => {
+      if (field.mode === 'exact') {
+        return [];
+      }
+
+      const minLength = field.minLength ?? DEFAULT_MIN_LENGTH;
+
+      if (word.length < minLength) {
+        return [];
+      }
+
+      return [formatToken(field, normalizeValue(word))];
+    });
+
+    return candidates.length > 0
+      ? { [config.tokenField ?? DEFAULT_TOKEN_FIELD]: { $in: candidates } }
+      : { _id: { $in: [] } };
+  });
+
+  const exactValue = normalizeExactValue(searchValue);
+  const exactCandidates = config.fields.flatMap((field) => {
+    const minLength = field.minLength ?? DEFAULT_MIN_LENGTH;
+
+    return field.mode === 'exact' && exactValue.length >= minLength
+      ? [formatToken(field, exactValue)]
+      : [];
+  });
+  const alternatives: FilterQuery<T>[] = [];
+
+  if (wordClauses.length > 0) {
+    alternatives.push({ $and: wordClauses } as FilterQuery<T>);
+  }
+
+  if (exactCandidates.length > 0) {
+    alternatives.push({
+      [config.tokenField ?? DEFAULT_TOKEN_FIELD]: { $in: exactCandidates },
+    } as FilterQuery<T>);
+  }
+
+  return alternatives.length > 0
+    ? ({ $or: alternatives } as FilterQuery<T>)
+    : ({ _id: { $in: [] } } as FilterQuery<T>);
+};
+
+export const configureSchemaSearchTokens = (
+  schema: Schema,
+  config: ISearchTokenConfig,
+) => {
+  if (!config.enabled) {
+    return;
+  }
+
+  const tokenField = config.tokenField ?? DEFAULT_TOKEN_FIELD;
+
+  schema.add({
+    [tokenField]: { type: [String], default: [] },
+  });
+  searchConfigurations.set(schema, config);
+
+  schema.pre('save', function () {
+    this.set(
+      tokenField,
+      generateConfiguredSearchTokens(this.toObject(), config),
+    );
+  });
+};
+
+export const getSchemaSearchTokenConfig = (
+  schema: Schema,
+): ISearchTokenConfig | undefined => searchConfigurations.get(schema);