@erxes/api-shared 0.0.0-stage → 3.2.19-2

package/src/utils/start-plugin.ts

@@ -0,0 +1,482 @@
+import './sentry-instrument';
+import { ApolloServer } from '@apollo/server';
+import { expressMiddleware } from '@apollo/server/express4';
+import { ApolloServerPluginDrainHttpServer } from '@apollo/server/plugin/drainHttpServer';
+import { buildSubgraphSchema } from '@apollo/subgraph';
+import * as trpcExpress from '@trpc/server/adapters/express';
+import cookieParser from 'cookie-parser';
+import cors from 'cors';
+import * as dotenv from 'dotenv';
+import express, {
+  Request as ApiRequest,
+  Response as ApiResponse,
+  Application,
+  Router,
+} from 'express';
+import { DocumentNode, GraphQLScalarType } from 'graphql';
+import * as http from 'http';
+import * as path from 'path';
+import rateLimit from 'express-rate-limit';
+import { startPayments } from '../common-modules/payment/worker';
+import type {
+  IPropertyMeta,
+  LogsConfigs,
+  SegmentConfigs,
+  TApprovalConfig,
+  TRecordReferencesConfig,
+} from '../core-modules';
+import {
+  initApproval,
+  initPropertyProducers,
+  initRecordReferences,
+  initSegmentProducers,
+  startAutomations,
+} from '../core-modules';
+import { AutomationConfigs } from '../core-modules/automations/types';
+import type { ImportExportConfigs } from '../core-modules/import-export/types';
+import { startImportExportWorker } from '../core-modules/import-export/worker';
+import { IMainContext, IPermissionConfig } from '../core-types';
+import {
+  generateApolloContext,
+  startBeforeResolvers,
+  wrapApolloResolvers,
+  expectedErrorPlugin,
+} from './apollo';
+import { BeforeResolversConfig } from './apollo/beforeResolvers';
+import { extractUserFromHeader } from './headers';
+import { AfterProcessConfigs, logHandler, startAfterProcess } from './logs';
+import { closeMongooose } from './mongo';
+import {
+  initializePluginConfig,
+  joinErxesGateway,
+  leaveErxesGateway,
+} from './service-discovery';
+import { createTRPCContext } from './trpc';
+import { mountAgentTools } from './agent-tools';
+import { applyTrustProxy, getSubdomain } from './utils';
+import * as Sentry from '@sentry/node';
+
+export const MAX_HEADER_BYTES = 64 * 1024;
+
+dotenv.config();
+
+enum API_METHODS {
+  GET = 'get',
+  POST = 'post',
+  PUT = 'put',
+  PATCH = 'patch',
+  DELETE = 'delete',
+}
+
+type TAPIMethod = keyof typeof API_METHODS;
+
+type IMeta = {
+  automations?: AutomationConfigs;
+  segments?: SegmentConfigs;
+  logs?: LogsConfigs;
+  afterProcess?: AfterProcessConfigs;
+  payments?: any;
+  notifications?: any;
+  tags?: any;
+  documents?: {
+    types: {
+      label: string;
+      contentType: string;
+    }[];
+  };
+  properties?: IPropertyMeta;
+  references?: TRecordReferencesConfig;
+  approval?: TApprovalConfig;
+  permissions?: IPermissionConfig;
+  beforeResolvers?: BeforeResolversConfig;
+  importExport?: ImportExportConfigs;
+  relations?: {
+    subscribedTypes: string[];
+  };
+};
+
+type ApiHandler = {
+  method: TAPIMethod;
+  path: string;
+  resolver: (req: ApiRequest, res: ApiResponse) => Promise<void> | void;
+};
+type ResolverObject = {
+  [key: string]: (...args: any[]) => any;
+};
+
+type GraphqlResolver = {
+  [key: string]: ResolverObject | GraphQLScalarType;
+};
+
+type ConfigTypes = {
+  name: string;
+  port: number;
+  graphql: () => Promise<{
+    resolvers: GraphqlResolver;
+    typeDefs: DocumentNode;
+  }>;
+  expressRouter?: Router;
+  apolloServerContext: (
+    subdomain: string,
+    context: any,
+    req: ApiRequest,
+    res: ApiResponse,
+  ) => Promise<IMainContext>;
+  onServerInit?: (app: express.Express) => Promise<void>;
+  middlewares?: any;
+  apiHandlers?: ApiHandler[];
+  hasSubscriptions?: boolean;
+  corsOptions?: any;
+  subscriptionPluginPath?: any;
+  trpcAppRouter?: {
+    router: any;
+    createContext: <TContext>(
+      subdomain: string,
+      context: any,
+    ) => Promise<TContext>;
+  };
+  /**
+   * tRPC procedure paths to exclude from the agent capability manifest.
+   * Agent-tools endpoints are mounted automatically on every plugin that
+   * supplies a `trpcAppRouter`. Only procedures declaring
+   * `.meta({ agent: { permission } })` appear in the manifest; this list
+   * removes specific annotated procedures when needed.
+   */
+  agentToolsExclude?: string[];
+  meta?: IMeta;
+};
+
+export async function startPlugin(
+  configs: ConfigTypes,
+): Promise<express.Express> {
+  const {
+    //common
+    name,
+    port,
+    // api configs
+    corsOptions = {},
+    expressRouter,
+    middlewares,
+    apiHandlers,
+    // graphql
+    hasSubscriptions,
+    subscriptionPluginPath,
+    graphql,
+    apolloServerContext,
+    trpcAppRouter,
+    onServerInit,
+    // agent capability endpoint exclusions
+    agentToolsExclude,
+    // meta
+    meta,
+  } = configs || {};
+  const PORT = process.env.PORT ? Number(process.env.PORT) : port;
+
+  Sentry.getGlobalScope().setTags({
+    plugin: name,
+    service: name,
+  });
+
+  const app = express();
+  applyTrustProxy(app);
+  app.disable('x-powered-by');
+  app.use(cors(corsOptions));
+  app.use(
+    express.json({
+      limit: '15mb',
+      verify: (req: any, _res, buf: Buffer) => {
+        req.rawBody = buf;
+      },
+    }),
+  );
+  app.use(cookieParser());
+
+  // for health check
+  app.get('/health', async (_req, res) => {
+    res.end('ok');
+  });
+
+  app.get('/debug-sentry', () => {
+    throw new Error('Sentry test error: ' + new Date().toISOString());
+  });
+
+  if (expressRouter) {
+    app.use(expressRouter);
+  }
+
+  if (middlewares) {
+    for (const middleware of middlewares) {
+      app.use(middleware);
+    }
+  }
+
+  if (apiHandlers) {
+    for (const handler of apiHandlers) {
+      const { method, path, resolver } = handler;
+
+      type LowercaseMethod = (typeof API_METHODS)[TAPIMethod];
+
+      // Ensure `method` is one of the keys
+      const METHOD = API_METHODS[method] as LowercaseMethod;
+      type TApiMethodApp = Record<
+        LowercaseMethod,
+        Application[LowercaseMethod]
+      >;
+
+      (app as TApiMethodApp)[METHOD](
+        path,
+        async (req: ApiRequest, res: ApiResponse) => {
+          return await logHandler(async () => await resolver(req, res), {
+            subdomain: getSubdomain(req),
+            source: 'webhook',
+            action: method,
+            payload: {
+              path,
+              headers: req.headers,
+              body: req.body,
+              query: req?.query,
+            },
+            userId: extractUserFromHeader(req.headers)?._id,
+          });
+        },
+      );
+    }
+  }
+
+  if (hasSubscriptions) {
+    if (!subscriptionPluginPath) {
+      throw new Error(
+        'subscriptionPluginPath is required when hasSubscriptions is true',
+      );
+    }
+
+    /**
+     * Protects the public subscription bundle endpoint from request floods.
+     *
+     * The limit is intentionally generous so regular page loads, retries, and
+     * normal multi-user traffic are not blocked. It only throttles abnormal
+     * high-frequency bursts from the same IP.
+     */
+    const subscriptionFileLimiter = rateLimit({
+      windowMs: 15 * 60 * 1000,
+      max: 1000,
+      standardHeaders: true,
+      legacyHeaders: false,
+      message: 'Too many requests from this IP, please try again later.',
+    });
+
+    app.get('/subscriptionPlugin.js', subscriptionFileLimiter, (_req, res) => {
+      res.sendFile(path.resolve(subscriptionPluginPath));
+    });
+  }
+
+  if (trpcAppRouter) {
+    const { router, createContext } = trpcAppRouter;
+    app.use(
+      '/trpc',
+      trpcExpress.createExpressMiddleware({
+        router,
+        createContext: createTRPCContext(createContext),
+      }),
+    );
+  }
+
+  // Agent capability endpoints are mounted automatically on every plugin with
+  // a tRPC router. The manifest is admit-only: only procedures declaring
+  // `.meta({ agent: { permission } })` are exposed, so an empty router
+  // produces an empty manifest and zero callable tools.
+  if (trpcAppRouter) {
+    mountAgentTools(app, {
+      plugin: name,
+      trpcRouter: trpcAppRouter.router,
+      createContext: trpcAppRouter.createContext,
+      exclude: agentToolsExclude || [],
+    });
+  }
+
+  app.use((req: any, _res, next) => {
+    if (req.rawBody === undefined) {
+      req.rawBody = '';
+
+      req.on('data', (chunk: any) => {
+        req.rawBody += chunk.toString();
+      });
+    }
+
+    next();
+  });
+
+  // Error handling middleware
+  // app.use((error: any, _req: any, res: any) => {
+  //   const msg = filterXSS(error.message);
+
+  //   // debugError(`Error: ${msg}`);
+
+  //   res.status(500).send(msg);
+  // });
+
+  const httpServer = http.createServer(
+    { maxHeaderSize: MAX_HEADER_BYTES },
+    app,
+  );
+  httpServer.keepAliveTimeout = 120000;
+  httpServer.headersTimeout = 121000;
+
+  // GRACEFULL SHUTDOWN
+  process.stdin.resume(); // so the program will not close instantly
+
+  async function closeHttpServer() {
+    try {
+      await new Promise<void>((resolve, reject) => {
+        // Stops the server from accepting new connections and finishes existing connections.
+        httpServer.close((error: Error | undefined) => {
+          if (error) {
+            return reject(error);
+          }
+          resolve();
+        });
+      });
+    } catch (e) {
+      console.error(e);
+    }
+  }
+
+  async function leaveServiceDiscovery() {
+    try {
+      await leaveErxesGateway(name, PORT);
+      console.log(`Left service discovery. name=${name} port=${PORT}`);
+    } catch (e) {
+      console.error(e);
+    }
+  }
+
+  // If the Node process ends, close the Mongoose connection
+  (['SIGINT', 'SIGTERM'] as NodeJS.Signals[]).forEach((sig) => {
+    process.on(sig, async () => {
+      await closeHttpServer();
+      await closeMongooose();
+      await leaveServiceDiscovery();
+      process.exit(0);
+    });
+  });
+
+  const generateApolloServer = async () => {
+    // const services = await getServices();
+    // debugInfo(`Enabled services .... ${JSON.stringify(services)}`);
+
+    const { typeDefs, resolvers } = await graphql();
+
+    return new ApolloServer({
+      schema: buildSubgraphSchema([
+        {
+          typeDefs,
+          resolvers: wrapApolloResolvers(resolvers as any),
+        },
+      ]),
+
+      // for graceful shutdown
+      plugins: [
+        ApolloServerPluginDrainHttpServer({ httpServer }),
+        expectedErrorPlugin,
+      ],
+    });
+  };
+
+  const apolloServer = await generateApolloServer();
+  await apolloServer.start();
+
+  app.use(
+    '/graphql',
+    expressMiddleware(apolloServer, {
+      context: generateApolloContext<IMainContext>(apolloServerContext),
+    }),
+  );
+
+  await new Promise<void>((resolve) =>
+    httpServer.listen({ port: PORT }, resolve),
+  );
+
+  console.log(
+    `🚀 ${name} graphql api ready at http://localhost:${PORT}/graphql`,
+  );
+
+  await joinErxesGateway({
+    name,
+    port: PORT,
+    hasSubscriptions,
+    meta,
+  });
+
+  if (meta) {
+    const {
+      automations,
+      segments,
+      afterProcess,
+      notifications,
+      payments,
+      beforeResolvers,
+      references,
+      importExport,
+      approval,
+      properties,
+    } = meta || {};
+
+    if (beforeResolvers) {
+      await startBeforeResolvers(app, name, beforeResolvers);
+    }
+
+    if (afterProcess) {
+      await startAfterProcess(app, name, afterProcess);
+    }
+
+    if (references) {
+      await initRecordReferences(app, name, references);
+    }
+
+    if (approval) {
+      await initApproval(app, name, approval);
+    }
+
+    if (automations) {
+      await startAutomations(app, name, automations);
+    }
+
+    if (segments) {
+      await initSegmentProducers(app, name, segments);
+    }
+
+    if (properties?.valueUsage) {
+      initPropertyProducers(app, properties.valueUsage);
+    }
+
+    if (notifications) {
+      await initializePluginConfig(name, 'notifications', notifications);
+    }
+
+    if (importExport) {
+      startImportExportWorker({
+        pluginName: name,
+        config: importExport,
+        app,
+      });
+    }
+
+    if (payments) {
+      await startPayments(name, payments);
+    }
+  } // end meta if
+
+  if (onServerInit) {
+    onServerInit(app);
+  }
+
+  //   applyInspectorEndpoints(name);
+
+  //   debugInfo(`${name} server is running on port: ${PORT}`);
+
+  Sentry.setupExpressErrorHandler(app);
+
+  return app;
+}
+
+export default startPlugin;

package/src/utils/string.ts

@@ -0,0 +1,159 @@
+import * as crypto from 'crypto';
+
+export const stripAnsi = async (str: string) => {
+  let stripAnsiModule: ((str: string) => string) | null = null;
+
+  if (!stripAnsiModule) {
+    const { default: imported } = await import('strip-ansi');
+    stripAnsiModule = imported;
+  }
+  return stripAnsiModule(str);
+};
+
+/**
+ * Removes the last trailing slash from a string if it exists.
+ *
+ * @param text - the string to remove the trailing slash from
+ * @returns the modified string with the trailing slash removed
+ */
+export const removeLastTrailingSlash = (text: string) => {
+  return text.replace(/\/$/, '');
+};
+
+/**
+ * Replaces multiple whitespace characters in a string with a single space and trims
+ * the string of leading and trailing whitespace.
+ *
+ * @param text - the string to remove extra spaces from
+ * @returns the modified string with extra spaces removed
+ */
+export const removeExtraSpaces = (text: string) => {
+  return text.replaceAll(/\s+/g, ' ').trim();
+};
+
+/**
+ * Checks if the given string is a valid URL.
+ *
+ * @param url - the string to check
+ * @returns true if the string is a valid URL, false otherwise
+ */
+export const isValidURL = (url: string): boolean => {
+  try {
+    return Boolean(new URL(url));
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * Splits text into chunks of strings limited by the given character count.
+ *
+ * Regex explanation:
+ * .{1,100}(\\s|$)
+ * - .         → matches any character (except line terminators)
+ * - {1,100}   → matches 1 to 100 of the preceding token
+ * - (\\s|$)   → ends with a whitespace OR end of string
+ *
+ * @param str - Text to be split
+ * @param size - Character length of each chunk
+ * @returns An array of string chunks
+ */
+export const splitStr = async (
+  str: string,
+  size: number,
+): Promise<string[]> => {
+  const cleanStr = await stripAnsi(str);
+
+  const regex = new RegExp(String.raw`.{1,${size}}(\s|$)`, 'g');
+
+  const matches: string[] = [];
+  let match;
+  while ((match = regex.exec(cleanStr)) !== null) {
+    matches.push(match[0]);
+  }
+  return matches;
+};
+
+/**
+ * Cleans the given HTML content by removing ANSI escape codes
+ * and truncating the result to a maximum of 100 characters.
+ *
+ * @param content - The HTML content to be cleaned.
+ * @returns The cleaned content as a string.
+ */
+
+export const cleanHtml = async (content?: string): Promise<string> =>
+  (await stripAnsi(content || '')).substring(0, 100);
+
+/**
+ * Escapes a string so that it can be used in a regular expression.
+ * This escapes all characters that have special meanings in regular
+ * expressions, so that the string can be used verbatim in a regular
+ * expression.
+ *
+ * @param {string} str the string to escape
+ * @returns {string} the escaped string
+
+/**
+ * Takes an array of strings and returns a single string that is
+ * suitable for use as a search query. The function trims the
+ * resulting string to 512 characters if it is longer than that.
+ *
+ * @param {string[]} values an array of strings to join
+ * @returns {string} a single string suitable for use as a search query
+ */
+
+/**
+ * Generates a random string of a given length using a given pattern.
+ *
+ * The pattern string can contain the following special characters:
+ * - `a`: lowercase letters
+ * - `A`: uppercase letters
+ * - `0`: numbers
+ * - `!`: special characters
+ *
+ * Any other characters in the pattern string are interpreted as literal
+ * characters and are included in the resulting string.
+ *
+ * If no valid pattern is provided, the function throws an error.
+ *
+ * @param {string} pattern the pattern to use when generating the string
+ * @param {number} length the length of the resulting string
+ * @returns {string} the generated string
+ */
+export const random = (pattern: string, length: number) => {
+  const maskMap: Record<string, string> = {
+    a: 'abcdefghijklmnopqrstuvwxyz',
+    A: 'ABCDEFGHIJKLMNOPQRSTUVWXYZ',
+    '0': '0123456789',
+    '!': '~`!@#$%^&*()_+-={}[]:";\'<>?,./|\\',
+  };
+
+  let chars = '';
+
+  for (const char of pattern) {
+    if (maskMap[char]) {
+      chars += maskMap[char];
+    } else {
+      chars += char;
+    }
+  }
+
+  if (!chars) throw new Error('No valid pattern provided.');
+
+  let result = '';
+
+  const maxByte = 255;
+  const maxMultiple = Math.floor(maxByte / chars.length) * chars.length;
+
+  while (result.length < length) {
+    const randomBytes = crypto.randomBytes(1);
+    const randomValue = randomBytes[0];
+
+    if (randomValue < maxMultiple) {
+      result += chars[randomValue % chars.length];
+    }
+  }
+
+  return result;
+};