@erxes/api-shared 0.0.0-stage → 3.2.19-1

package/src/utils/headers/user.ts

@@ -0,0 +1,113 @@
+import { IncomingHttpHeaders } from 'http';
+
+export const userHeaderName = 'user';
+export const clientPortalHeaderName = 'clientportal';
+export const cpUserHeaderName = 'cpuser';
+
+export function extractUserFromHeader(headers: IncomingHttpHeaders): any {
+  const userHeader = headers[userHeaderName];
+  if (!userHeader) {
+    return null;
+  }
+  if (Array.isArray(userHeader)) {
+    throw new Error(`Multiple user headers`);
+  }
+  const userJson = Buffer.from(userHeader, 'base64').toString('utf-8');
+  return JSON.parse(userJson);
+}
+
+export function extractCPUserFromHeader(headers: IncomingHttpHeaders): any {
+  const cpUserHeader = headers[cpUserHeaderName];
+
+  if (!cpUserHeader) {
+    return null;
+  }
+
+  if (Array.isArray(cpUserHeader)) {
+    throw new Error(`Multiple cp user headers`);
+  }
+  const cpUserJson = Buffer.from(cpUserHeader, 'base64').toString('utf-8');
+  return JSON.parse(cpUserJson);
+}
+
+export function extractClientPortalFromHeader(
+  headers: IncomingHttpHeaders,
+): any {
+  const clientPortalHeader = headers[clientPortalHeaderName];
+
+  if (!clientPortalHeader) {
+    return null;
+  }
+
+  if (Array.isArray(clientPortalHeader)) {
+    throw new Error(`Multiple client portal headers`);
+  }
+  const clientPortalJson = Buffer.from(clientPortalHeader, 'base64').toString(
+    'utf-8',
+  );
+  return JSON.parse(clientPortalJson);
+}
+
+export const USER_HEADER_OMITTED_FIELDS = [
+  'emailSignatures',
+  'customFieldsData',
+  'propertiesData',
+  'links',
+  'loginToken',
+] as const;
+
+export const HEADER_SIZE_WARN_BYTES = 32 * 1024;
+
+function encodeHeader(name: string, id: string, value: unknown): string {
+  const encoded = Buffer.from(JSON.stringify(value), 'utf8').toString('base64');
+
+  if (encoded.length > HEADER_SIZE_WARN_BYTES) {
+    console.warn(
+      `${name} header is ${encoded.length} bytes for ${id}; requests fail once it passes the service header budget`,
+    );
+  }
+
+  return encoded;
+}
+
+export function compactUserForHeader<T extends Record<string, unknown>>(
+  user: T,
+): Omit<T, (typeof USER_HEADER_OMITTED_FIELDS)[number]> {
+  const omitted: readonly string[] = USER_HEADER_OMITTED_FIELDS;
+
+  return Object.fromEntries(
+    Object.entries(user).filter(([field]) => !omitted.includes(field)),
+  ) as Omit<T, (typeof USER_HEADER_OMITTED_FIELDS)[number]>;
+}
+
+export function setUserHeader(headers: IncomingHttpHeaders, user: any) {
+  if (!user) return;
+  headers[userHeaderName] = encodeHeader(
+    userHeaderName,
+    user._id,
+    compactUserForHeader(user),
+  );
+  headers['userid'] = user._id || '';
+}
+
+export function setCPUserHeader(headers: IncomingHttpHeaders, cpUser: any) {
+  if (!cpUser) return;
+  headers[cpUserHeaderName] = encodeHeader(
+    cpUserHeaderName,
+    cpUser._id,
+    cpUser,
+  );
+}
+
+export function setClientPortalHeader(
+  headers: IncomingHttpHeaders,
+  clientPortal: any,
+) {
+  if (!clientPortal) return;
+  const clientPortalJson = JSON.stringify(clientPortal);
+  const clientPortalJsonBase64 = Buffer.from(clientPortalJson, 'utf8').toString(
+    'base64',
+  );
+
+  headers[clientPortalHeaderName] = clientPortalJsonBase64;
+}

package/src/utils/index.ts

@@ -0,0 +1,27 @@
+export * from './apollo';
+export * from './agent-tools';
+export * from './constants';
+export * from './graphqlPubSub';
+export * from './headers';
+export * from './logs';
+export * from './knowledge';
+export * from './mongo';
+export * from './mq-worker';
+export * from './random';
+export * from './redis';
+export * from './saas';
+export * from './sanitize';
+export * from './service-discovery';
+export * from './start-plugin';
+export * from './trpc';
+export * from './trpc/sendCoreModuleProducer';
+export * from './utils';
+export * from './file/read';
+export * from './file/upload';
+export * from './mongo/cursor-util';
+export * from './string';
+export * from './bulkUtils';
+export * from './editor';
+export * from './errorClassifier';
+export * from './sentry-init';
+export * from './email';

package/src/utils/knowledge.ts

@@ -0,0 +1,107 @@
+import type { DefaultJobOptions } from 'bullmq';
+import { sendWorkerQueue } from './mq-worker';
+
+export type TKnowledgeVisibility = 'public' | 'internal';
+
+export type TKnowledgeContentFormat = 'html' | 'markdown' | 'text';
+
+export type TKnowledgeSourceReference = {
+  type: string;
+  id: string;
+  version: string;
+  updatedAt: string;
+  url?: string;
+};
+
+export type TKnowledgeDocumentMetadata = {
+  visibility: TKnowledgeVisibility;
+  language?: 'mn' | 'en' | 'mixed' | 'unknown';
+  tags?: string[];
+  relatedEntityIds?: string[];
+  [key: string]: unknown;
+};
+
+export type TKnowledgeDocument = {
+  source: TKnowledgeSourceReference;
+  title: string;
+  content: string;
+  contentFormat?: TKnowledgeContentFormat;
+  metadata: TKnowledgeDocumentMetadata;
+};
+
+export type TAiKnowledgeSourceReference = {
+  pluginName: string;
+  moduleName: string;
+  key: string;
+  sourceId: string;
+  updatedAt: string;
+};
+
+export const buildKnowledgeSourceType = ({
+  pluginName,
+  moduleName,
+  key,
+}: Pick<TAiKnowledgeSourceReference, 'pluginName' | 'moduleName' | 'key'>) =>
+  `${pluginName}:${moduleName}:${key}`;
+
+export type TKnowledgeIndexJob =
+  | {
+      operation: 'upsert';
+      document: TKnowledgeDocument;
+    }
+  | {
+      operation: 'remove';
+      source: TKnowledgeSourceReference;
+    };
+
+export const enqueueKnowledgeIndexJob = async ({
+  subdomain,
+  job,
+  jobOptions,
+}: {
+  subdomain: string;
+  job: TKnowledgeIndexJob;
+  jobOptions?: DefaultJobOptions;
+}) => {
+  const source = job.operation === 'upsert' ? job.document.source : job.source;
+  const queue = sendWorkerQueue('automations', 'aiAgent');
+
+  return queue.add(
+    'indexKnowledgeDocument',
+    {
+      subdomain,
+      data: job,
+    },
+    {
+      attempts: 2,
+      jobId: `knowledge:${source.type}:${source.id}:${job.operation}:${source.version}:${source.updatedAt}`,
+      removeOnComplete: true,
+      removeOnFail: true,
+      ...jobOptions,
+    },
+  );
+};
+
+export const enqueueAiKnowledgeSourceRefreshJob = async ({
+  subdomain,
+  source,
+  jobOptions,
+}: {
+  subdomain: string;
+  source: TAiKnowledgeSourceReference;
+  jobOptions?: DefaultJobOptions;
+}) => {
+  const queue = sendWorkerQueue('automations', 'aiAgent');
+
+  return queue.add(
+    'refreshAiKnowledgeSource',
+    { subdomain, data: { source } },
+    {
+      attempts: 2,
+      jobId: `knowledge-source:${source.pluginName}:${source.moduleName}:${source.key}:${source.sourceId}:${source.updatedAt}`,
+      removeOnComplete: true,
+      removeOnFail: true,
+      ...jobOptions,
+    },
+  );
+};

package/src/utils/logs/index.ts

@@ -0,0 +1,247 @@
+import { checkServiceRunning } from '../utils';
+import { ILogDoc } from '../../core-types';
+import { sendWorkerQueue } from '../mq-worker';
+import { initializePluginConfig } from '../service-discovery';
+import { Express } from 'express';
+import * as trpcExpress from '@trpc/server/adapters/express';
+import { initTRPC } from '@trpc/server';
+import { createTRPCContext } from '../trpc';
+import { nanoid } from 'nanoid';
+import { z } from 'zod';
+
+export const logHandler = async (
+  resolver: () => Promise<any> | any,
+  logDoc: ILogDoc,
+  onSuccess?: any,
+  onError?: any,
+  skipSaveResult?: boolean,
+) => {
+  if (!(await checkServiceRunning('logs'))) {
+    return await resolver();
+  }
+
+  const payload = { ...(logDoc?.payload || {}) };
+  const startDate = new Date();
+  const startTime = performance.now();
+  try {
+    const result = await resolver();
+
+    const endTime = performance.now();
+    const durationMs = endTime - startTime;
+    logDoc.payload = { ...payload, ...onSuccess };
+    if (!skipSaveResult) {
+      logDoc.payload.result = result;
+    }
+    logDoc.executionTime = {
+      startDate,
+      endDate: new Date(),
+      durationMs: durationMs,
+    };
+    logDoc.status = 'success';
+    sendWorkerQueue('logs', 'put_log').add('put_log', logDoc, {
+      removeOnComplete: true,
+    });
+
+    return result;
+  } catch (error) {
+    const errorDetails = {
+      message: error.message || 'Unknown error',
+      stack: error.stack || 'No stack available',
+      name: error.name || 'Error',
+    };
+
+    logDoc.payload = { ...payload, ...onError, error: errorDetails };
+    logDoc.status = 'failed';
+    sendWorkerQueue('logs', 'put_log').add('put_log', logDoc, {
+      removeOnComplete: true,
+    });
+
+    throw error;
+  }
+};
+type IContext = {
+  subdomain: string;
+  processId?: string;
+};
+
+type AfterMutation = {
+  type: 'afterMutation';
+  mutationNames: string[];
+};
+
+type UpdatedDocument = {
+  type: 'updatedDocument';
+  contentTypes: string[];
+  when?: {
+    fieldsUpdated?: string[];
+    fieldsRemoved?: string[];
+  };
+};
+
+type CreateDocument = {
+  type: 'createdDocument';
+  contentTypes: string[];
+  when?: {
+    fieldsWith?: string[];
+  };
+};
+
+type AfterAPIRequest = {
+  type: 'afterAPIRequest';
+  paths: string[];
+};
+
+type AfterAuth = {
+  type: 'afterAuth';
+  types: string[];
+};
+
+export type IAfterProcessRule =
+  | AfterMutation
+  | CreateDocument
+  | UpdatedDocument
+  | AfterAPIRequest
+  | AfterAuth;
+
+export type TAfterProcessRule = {
+  AfterMutation: AfterMutation;
+  CreateDocument: CreateDocument;
+  UpdatedDocument: UpdatedDocument;
+  AfterAPIRequest: AfterAPIRequest;
+  AfterAuth: AfterAuth;
+};
+
+export interface AfterProcessConfigs {
+  rules: IAfterProcessRule[];
+  afterMutation?: (
+    context: IContext,
+    args: {
+      data: {
+        mutationName: string;
+        args: { [key: string]: any };
+        result: any;
+        userId?: string;
+      };
+    },
+  ) => void | Promise<void>;
+  afterAuth?: (
+    context: IContext,
+    args: { userId: string; email: string; result: string },
+  ) => void;
+  afterApiRequest?: (context: IContext, args: any) => void;
+  afterDocumentUpdated?: <TDocument = any>(
+    context: IContext,
+    args: {
+      data: {
+        collectionName: string;
+        docId: string;
+        prevDocument?: any;
+        currentDocument?: any;
+        updateDescription: {
+          added: { [key: string]: any };
+          updated: { [key: string]: any };
+          removed: string[];
+        };
+        userId: string;
+        processId: string;
+        contentType: string;
+      };
+    },
+  ) => void;
+  afterDocumentCreated?: <TDocument = any>(
+    context: IContext,
+    args: {
+      data: {
+        collectionName: string;
+        docId: string;
+        currentDocument?: any;
+        userId: string;
+        processId: string;
+        contentType: string;
+      };
+    },
+  ) => void;
+}
+
+export interface AfterProcessModuleConfig<TModels = any> {
+  rules: IAfterProcessRule[];
+  createdDocument?: Record<
+    string,
+    (models: TModels, data: any) => Promise<void>
+  >;
+  updatedDocument?: Record<
+    string,
+    (subdomain: string, models: TModels, data: any) => Promise<void>
+  >;
+  afterMutation?: Record<
+    string,
+    (subdomain: string, models: TModels, data: any) => Promise<void>
+  >;
+}
+
+export interface AfterProcessModules<TModels = any> {
+  [moduleName: string]: AfterProcessModuleConfig<TModels>;
+}
+
+export const startAfterProcess = async (
+  app: Express,
+  pluginName: string,
+  config: AfterProcessConfigs,
+) => {
+  await initializePluginConfig(pluginName, 'afterProcess', config);
+
+  const t = initTRPC.context<IContext>().create();
+
+  const {
+    afterMutation,
+    afterAuth,
+    afterApiRequest,
+    afterDocumentUpdated,
+    afterDocumentCreated,
+  } = config || {};
+
+  const routes: Record<string, any> = {};
+
+  if (afterMutation) {
+    routes.afterMutation = t.procedure
+      .input(z.any())
+      .mutation(async ({ ctx, input }) => afterMutation(ctx, input));
+  }
+
+  if (afterAuth) {
+    routes.afterAuth = t.procedure
+      .input(z.any())
+      .mutation(async ({ ctx, input }) => afterAuth(ctx, input));
+  }
+
+  if (afterApiRequest) {
+    routes.afterApiRequest = t.procedure
+      .input(z.any())
+      .mutation(async ({ ctx, input }) => afterApiRequest(ctx, input));
+  }
+
+  if (afterDocumentUpdated) {
+    routes.afterDocumentUpdated = t.procedure
+      .input(z.any())
+      .mutation(async ({ ctx, input }) => afterDocumentUpdated(ctx, input));
+  }
+
+  if (afterDocumentCreated) {
+    routes.afterDocumentCreated = t.procedure
+      .input(z.any())
+      .mutation(async ({ ctx, input }) => afterDocumentCreated(ctx, input));
+  }
+
+  const afterProcessRouter = t.router(routes);
+
+  const trpcMiddleware = trpcExpress.createExpressMiddleware({
+    router: afterProcessRouter,
+    createContext: createTRPCContext(async (_subdomain, context) => {
+      const processId = nanoid(12);
+      context.processId = processId;
+      return context;
+    }),
+  });
+
+  app.use('/afterProcess', trpcMiddleware);
+};

package/src/utils/mongo/cursor-util.ts

@@ -0,0 +1,153 @@
+import { SortOrder, FilterQuery, Model, Types, PipelineStage } from 'mongoose';
+import { fixNum } from '../utils';
+
+export interface CursorPaginateParams<T> {
+  model: Model<T>;
+  params: {
+    limit?: number;
+    cursor?: string;
+    direction?: 'forward' | 'backward';
+    orderBy?: Record<string, SortOrder>;
+  };
+  query?: FilterQuery<T>;
+  formatter?: Record<string, CursorFieldType>;
+}
+
+export interface CursorPaginateAggregationParams<T> {
+  model: Model<T>;
+  pipeline?: PipelineStage[];
+  params: {
+    limit?: number;
+    cursor?: string;
+    direction?: 'forward' | 'backward';
+    orderBy?: Record<string, SortOrder>;
+  };
+  formatter?: Record<string, CursorFieldType>;
+  uniqConcatFields?: string[]; // optional: if unwinded then concat field names
+}
+
+export interface PageInfo {
+  hasNextPage: boolean;
+  hasPreviousPage: boolean;
+  startCursor: string | null;
+  endCursor: string | null;
+}
+
+export interface CursorResult<T> {
+  list: T[];
+  totalCount: number;
+  pageInfo: PageInfo;
+}
+
+export const encodeCursor = (item: any, sortFields: string[]): string => {
+  const cursorData: any = {};
+
+  for (const field of sortFields) {
+    if (item[field] !== undefined) {
+      cursorData[field] = item[field];
+    }
+  }
+
+  // Handle ObjectId serialization
+  cursorData._id =
+    item._id instanceof Types.ObjectId ? item._id.toString() : item._id;
+
+  return Buffer.from(JSON.stringify(cursorData)).toString('base64');
+};
+
+export const decodeCursor = (cursor: string): any => {
+  try {
+    const decoded = JSON.parse(Buffer.from(cursor, 'base64').toString());
+
+    // Convert _id back to ObjectId if it's a valid ObjectId string
+    if (decoded._id && Types.ObjectId.isValid(decoded._id)) {
+      decoded._id = new Types.ObjectId(decoded._id);
+    }
+
+    return decoded;
+  } catch {
+    throw new Error('Invalid cursor format');
+  }
+};
+
+type CursorFieldType = 'date' | 'number' | 'boolean';
+const convertValue = (value: any, type: CursorFieldType) => {
+  if (value == null) return value;
+
+  switch (type) {
+    case 'date':
+      return new Date(value);
+    case 'number':
+      return fixNum(value, 10);
+    case 'boolean':
+      return value === 'true' || value === true;
+    default:
+      return value;
+  }
+};
+
+export const buildCursorQuery = (
+  cursor: string,
+  orderBy: Record<string, SortOrder>,
+  direction: 'forward' | 'backward',
+  formatter?: Record<string, CursorFieldType>,
+): Record<string, any> => {
+  const cursorData = decodeCursor(cursor);
+
+  if (formatter) {
+    for (const key of Object.keys(formatter)) {
+      if (cursorData[key] !== undefined) {
+        cursorData[key] = convertValue(cursorData[key], formatter[key]);
+      }
+    }
+  }
+
+  const sortFields = Object.keys(orderBy);
+
+  if (sortFields.length === 0) {
+    const operator = direction === 'forward' ? '$gt' : '$lt';
+    return { _id: { [operator]: cursorData._id } };
+  }
+
+  const orConditions: Record<string, any>[] = [];
+
+  for (let i = 0; i < sortFields.length; i++) {
+    const field = sortFields[i];
+    const sortOrder = orderBy[field];
+    const isAscending = sortOrder === 1 || sortOrder === 'asc';
+
+    const condition: Record<string, any> = {};
+
+    for (let j = 0; j < i; j++) {
+      const prevField = sortFields[j];
+      condition[prevField] = cursorData[prevField];
+    }
+
+    const operator =
+      direction === 'forward'
+        ? isAscending
+          ? '$gt'
+          : '$lt'
+        : isAscending
+        ? '$lt'
+        : '$gt';
+
+    if (cursorData[field] !== undefined) {
+      condition[field] = { [operator]: cursorData[field] };
+      orConditions.push(condition);
+    }
+  }
+
+  const finalCondition: Record<string, any> = {};
+  for (const field of sortFields) {
+    if (cursorData[field] !== undefined) {
+      finalCondition[field] = cursorData[field];
+    }
+  }
+
+  const idOperator = direction === 'forward' ? '$gt' : '$lt';
+  finalCondition._id = { [idOperator]: cursorData._id };
+  orConditions.push(finalCondition);
+
+  return orConditions.length > 0 ? { $or: orConditions } : {};
+};