@erxes/api-shared 0.0.0-stage → 3.2.19-2

package/src/utils/file/upload.ts

@@ -0,0 +1,531 @@
+import { BlobServiceClient } from '@azure/storage-blob';
+import { Storage } from '@google-cloud/storage';
+import AWS from 'aws-sdk';
+import FormData from 'form-data';
+import * as fs from 'fs';
+import fetch from 'node-fetch';
+import * as path from 'path';
+import { tmpdir } from 'os';
+import { sendTRPCMessage } from '../trpc';
+import { getEnv, isImage, isVideo } from '../utils';
+import { sanitizeFilename } from '../sanitize';
+import { randomAlphanumeric } from '../random';
+
+const STORAGE_CONFIG_CODES = [
+  'UPLOAD_SERVICE_TYPE',
+  'AWS_BUCKET',
+  'AWS_ACCESS_KEY_ID',
+  'AWS_SECRET_ACCESS_KEY',
+  'AWS_COMPATIBLE_SERVICE_ENDPOINT',
+  'AWS_FORCE_PATH_STYLE',
+  'AWS_REGION',
+  'AWS_PREFIX',
+  'GOOGLE_CLOUD_STORAGE_BUCKET',
+  'GOOGLE_APPLICATION_CREDENTIALS',
+  'GOOGLE_PROJECT_ID',
+  'CLOUDFLARE_BUCKET_NAME',
+  'CLOUDFLARE_ACCOUNT_ID',
+  'CLOUDFLARE_ACCESS_KEY_ID',
+  'CLOUDFLARE_SECRET_ACCESS_KEY',
+  'CLOUDFLARE_API_TOKEN',
+  'CLOUDFLARE_USE_CDN',
+  'CLOUDFLARE_ACCOUNT_HASH',
+  'AZURE_STORAGE_CONNECTION_STRING',
+  'AZURE_STORAGE_CONTAINER',
+  'FILE_SYSTEM_PUBLIC',
+  'AWS_DISABLE_ACL',
+] as const;
+
+type StorageConfigCode = (typeof STORAGE_CONFIG_CODES)[number];
+type StorageConfigMap = Record<StorageConfigCode | string, string>;
+
+type UploadFileToStorageParams = {
+  subdomain: string;
+  filePath: string;
+  fileName: string;
+  mimetype: string;
+  forcePrivate?: boolean;
+};
+
+const parseBoolean = (value?: string | boolean) => {
+  if (typeof value === 'boolean') {
+    return value;
+  }
+
+  if (!value) {
+    return false;
+  }
+
+  return value.toString().toLowerCase() === 'true';
+};
+
+const readConfigValue = (
+  configs: StorageConfigMap,
+  code: StorageConfigCode,
+  defaultValue = '',
+) => {
+  return configs[code] ?? getEnv({ name: code, defaultValue });
+};
+
+const requireConfigValue = (
+  configs: StorageConfigMap,
+  code: StorageConfigCode,
+): string => {
+  const value = readConfigValue(configs, code);
+
+  if (!value) {
+    throw new Error(`${code} is not configured`);
+  }
+
+  return value;
+};
+
+const fetchStorageConfigs = async (
+  subdomain: string,
+): Promise<StorageConfigMap> => {
+  const configs =
+    (await sendTRPCMessage({
+      subdomain,
+      pluginName: 'core',
+      module: 'configs',
+      action: 'getConfigs',
+      method: 'query',
+      input: { codes: STORAGE_CONFIG_CODES },
+      defaultValue: {},
+    })) || {};
+
+  return configs;
+};
+
+const isValidPath = (filepath: string): boolean => {
+  const resolved = path.resolve(filepath);
+  const tempDir = path.resolve(tmpdir());
+  return resolved.startsWith(tempDir);
+};
+
+const createAwsClient = (configs: StorageConfigMap) => {
+  const accessKeyId = requireConfigValue(configs, 'AWS_ACCESS_KEY_ID');
+  const secretAccessKey = requireConfigValue(configs, 'AWS_SECRET_ACCESS_KEY');
+
+  const clientConfig: AWS.S3.ClientConfiguration = {
+    accessKeyId,
+    secretAccessKey,
+  };
+
+  const endpoint = readConfigValue(
+    configs,
+    'AWS_COMPATIBLE_SERVICE_ENDPOINT',
+    '',
+  );
+  const forcePathStyle = readConfigValue(configs, 'AWS_FORCE_PATH_STYLE', '');
+  const region = readConfigValue(configs, 'AWS_REGION', '');
+
+  if (endpoint) {
+    clientConfig.endpoint = endpoint;
+  }
+
+  if (parseBoolean(forcePathStyle)) {
+    clientConfig.s3ForcePathStyle = true;
+  }
+
+  if (region) {
+    clientConfig.region = region;
+  }
+
+  return new AWS.S3(clientConfig);
+};
+
+const createCloudflareR2Client = (configs: StorageConfigMap) => {
+  const accountId = requireConfigValue(configs, 'CLOUDFLARE_ACCOUNT_ID');
+  const accessKeyId = requireConfigValue(configs, 'CLOUDFLARE_ACCESS_KEY_ID');
+  const secretAccessKey = requireConfigValue(
+    configs,
+    'CLOUDFLARE_SECRET_ACCESS_KEY',
+  );
+
+  return new AWS.S3({
+    endpoint: `https://${accountId}.r2.cloudflarestorage.com`,
+    accessKeyId,
+    secretAccessKey,
+    signatureVersion: 'v4',
+    region: 'auto',
+  });
+};
+
+const uploadToCFImages = async (
+  filePath: string,
+  fileName: string,
+  configs: StorageConfigMap,
+  forcePrivate?: boolean,
+): Promise<string> => {
+  const sanitizedFilename = sanitizeFilename(fileName);
+
+  if (!isValidPath(filePath)) {
+    throw new Error('Unsafe file path');
+  }
+
+  const accountId = requireConfigValue(configs, 'CLOUDFLARE_ACCOUNT_ID');
+  const apiToken = requireConfigValue(configs, 'CLOUDFLARE_API_TOKEN');
+  const bucketName = requireConfigValue(configs, 'CLOUDFLARE_BUCKET_NAME');
+
+  const isPublic = forcePrivate
+    ? false
+    : parseBoolean(readConfigValue(configs, 'FILE_SYSTEM_PUBLIC', 'false'));
+
+  const version = getEnv({ name: 'VERSION' });
+
+  const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/images/v1`;
+  const headers = {
+    Authorization: `Bearer ${apiToken}`,
+  };
+
+  let finalFileName = `${randomAlphanumeric()}${sanitizedFilename}`;
+
+  const extension = finalFileName.split('.').pop();
+
+  if (extension && ['JPEG', 'JPG', 'PNG'].includes(extension)) {
+    const baseName = finalFileName.slice(0, -(extension.length + 1));
+    finalFileName = `${baseName}.${extension.toLowerCase()}`;
+  }
+
+  const formData = new FormData();
+  formData.append('file', fs.createReadStream(filePath));
+  formData.append('id', `${bucketName}/${finalFileName}`);
+
+  const response = await fetch(url, {
+    method: 'POST',
+    headers,
+    body: formData,
+  });
+
+  const data = await response.json();
+
+  if (!data.success) {
+    throw new Error('Error uploading file to Cloudflare Images');
+  }
+
+  if (data.result.variants.length === 0) {
+    throw new Error('Error uploading file to Cloudflare Images - no variants');
+  }
+
+  if (!isPublic || version === 'saas') {
+    return `${bucketName}/${finalFileName}`;
+  }
+
+  return data.result.variants[0];
+};
+
+const uploadToCFStream = async (
+  filePath: string,
+  fileName: string,
+  configs: StorageConfigMap,
+): Promise<string> => {
+  const sanitizedFilename = sanitizeFilename(fileName);
+
+  if (!isValidPath(filePath)) {
+    throw new Error('Unsafe file path');
+  }
+
+  const accountId = requireConfigValue(configs, 'CLOUDFLARE_ACCOUNT_ID');
+  const apiToken = requireConfigValue(configs, 'CLOUDFLARE_API_TOKEN');
+
+  const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/stream`;
+  const headers = {
+    Authorization: `Bearer ${apiToken}`,
+  };
+
+  const finalFileName = `${randomAlphanumeric()}${sanitizedFilename}`;
+
+  const formData = new FormData();
+  formData.append('file', fs.createReadStream(filePath));
+  formData.append('id', finalFileName);
+
+  const response = await fetch(url, {
+    method: 'POST',
+    headers,
+    body: formData,
+  });
+
+  const data = await response.json();
+
+  if (!data.success) {
+    throw new Error('Error uploading file to Cloudflare Stream');
+  }
+
+  return data.result.playback.hls;
+};
+
+const uploadToAzure = async (
+  filePath: string,
+  fileName: string,
+  mimetype: string,
+  configs: StorageConfigMap,
+): Promise<string> => {
+  const sanitizedFilename = sanitizeFilename(fileName);
+
+  if (!isValidPath(filePath)) {
+    throw new Error('Unsafe file path');
+  }
+
+  const isPublic = parseBoolean(
+    readConfigValue(configs, 'FILE_SYSTEM_PUBLIC', 'true'),
+  );
+
+  const connectionString = requireConfigValue(
+    configs,
+    'AZURE_STORAGE_CONNECTION_STRING',
+  );
+  const container = requireConfigValue(configs, 'AZURE_STORAGE_CONTAINER');
+
+  const blobServiceClient =
+    BlobServiceClient.fromConnectionString(connectionString);
+  const containerClient = blobServiceClient.getContainerClient(container);
+
+  const finalFileName = `${randomAlphanumeric()}${sanitizedFilename}`;
+  const blockBlobClient = containerClient.getBlockBlobClient(finalFileName);
+
+  const response = await blockBlobClient.uploadFile(filePath, {
+    blobHTTPHeaders: { blobContentType: mimetype },
+  });
+
+  if (!response) {
+    throw new Error('Error uploading file to Azure Blob Storage');
+  }
+
+  return isPublic ? blockBlobClient.url : finalFileName;
+};
+
+const uploadToAWS = async (
+  filePath: string,
+  fileName: string,
+  mimetype: string,
+  configs: StorageConfigMap,
+  forcePrivate?: boolean,
+): Promise<string> => {
+  const sanitizedFilename = sanitizeFilename(fileName);
+
+  if (!isValidPath(filePath)) {
+    throw new Error('Unsafe file path');
+  }
+
+  const isPublic = forcePrivate
+    ? false
+    : parseBoolean(readConfigValue(configs, 'FILE_SYSTEM_PUBLIC', 'true'));
+
+  const disableAcl = parseBoolean(
+    readConfigValue(configs, 'AWS_DISABLE_ACL', ''),
+  );
+
+  const awsPrefix = readConfigValue(configs, 'AWS_PREFIX', '');
+  const bucket = requireConfigValue(configs, 'AWS_BUCKET');
+
+  const s3 = createAwsClient(configs);
+  const finalFileName = `${awsPrefix}${randomAlphanumeric()}${sanitizedFilename}`;
+  const buffer = await fs.promises.readFile(filePath);
+
+  const response: any = await new Promise((resolve, reject) => {
+    s3.upload(
+      {
+        ContentType: mimetype,
+        Bucket: bucket,
+        Key: finalFileName,
+        Body: buffer,
+        ...(isPublic && !disableAcl ? { ACL: 'public-read' } : {}),
+      },
+      (err, res) => {
+        if (err) {
+          return reject(err);
+        }
+
+        return resolve(res);
+      },
+    );
+  });
+
+  return isPublic ? response.Location : finalFileName;
+};
+
+const uploadToGCS = async (
+  filePath: string,
+  fileName: string,
+  mimetype: string,
+  configs: StorageConfigMap,
+): Promise<string> => {
+  const sanitizedFilename = sanitizeFilename(fileName);
+
+  if (!isValidPath(filePath)) {
+    throw new Error('Unsafe file path');
+  }
+
+  const bucket = requireConfigValue(configs, 'GOOGLE_CLOUD_STORAGE_BUCKET');
+  const projectId = requireConfigValue(configs, 'GOOGLE_PROJECT_ID');
+  const credentialsPath = requireConfigValue(
+    configs,
+    'GOOGLE_APPLICATION_CREDENTIALS',
+  );
+
+  const isPublic = parseBoolean(
+    readConfigValue(configs, 'FILE_SYSTEM_PUBLIC', ''),
+  );
+
+  const storage = new Storage({
+    projectId,
+    keyFilename: credentialsPath,
+  });
+
+  const bucketClient = storage.bucket(bucket);
+  const finalFileName = `${randomAlphanumeric()}${sanitizedFilename}`;
+
+  const response: any = await new Promise((resolve, reject) => {
+    bucketClient.upload(
+      filePath,
+      {
+        metadata: { contentType: mimetype },
+        public: isPublic,
+      },
+      (err, res) => {
+        if (err) {
+          return reject(err);
+        }
+
+        if (res) {
+          return resolve(res);
+        }
+      },
+    );
+  });
+
+  const { metadata, name } = response;
+
+  return isPublic ? metadata.mediaLink : name;
+};
+
+const uploadToCloudflare = async (
+  filePath: string,
+  fileName: string,
+  mimetype: string,
+  configs: StorageConfigMap,
+  forcePrivate?: boolean,
+): Promise<string> => {
+  const isPublic = forcePrivate
+    ? false
+    : parseBoolean(readConfigValue(configs, 'FILE_SYSTEM_PUBLIC', 'false'));
+
+  const sanitizedFilename = sanitizeFilename(fileName);
+
+  if (!isValidPath(filePath)) {
+    throw new Error('Unsafe file path');
+  }
+
+  const bucketName = requireConfigValue(configs, 'CLOUDFLARE_BUCKET_NAME');
+  const useCdn = parseBoolean(
+    readConfigValue(configs, 'CLOUDFLARE_USE_CDN', ''),
+  );
+
+  const { fileTypeFromBuffer } = await import('file-type');
+  const detectedType = await fileTypeFromBuffer(new Uint8Array(fs.readFileSync(filePath)));
+
+  let adjustedFileName = fileName;
+  if (path.extname(fileName).toLowerCase() === '.jfif') {
+    adjustedFileName = fileName.replace('.jfif', '.jpeg');
+  }
+
+  if (
+    useCdn &&
+    detectedType &&
+    isImage(detectedType.mime) &&
+    ![
+      'image/heic',
+      'image/heif',
+      'image/x-icon',
+      'image/vnd.microsoft.icon',
+    ].includes(detectedType.mime)
+  ) {
+    return uploadToCFImages(filePath, adjustedFileName, configs, forcePrivate);
+  }
+
+  if (useCdn && detectedType && isVideo(detectedType.mime)) {
+    return uploadToCFStream(filePath, adjustedFileName, configs);
+  }
+
+  const finalFileName = `${randomAlphanumeric()}${sanitizedFilename}`;
+  const buffer = await fs.promises.readFile(filePath);
+  const r2 = createCloudflareR2Client(configs);
+
+  const response: any = await new Promise((resolve, reject) => {
+    r2.upload(
+      {
+        ContentType: mimetype,
+        Bucket: bucketName,
+        Key: finalFileName,
+        Body: buffer,
+        ...(isPublic ? { ACL: 'public-read' } : {}),
+      },
+      (err, res) => {
+        if (err) {
+          return reject(err);
+        }
+
+        return resolve(res);
+      },
+    );
+  });
+
+  return isPublic ? response.Location : finalFileName;
+};
+
+export const uploadFileToStorage = async ({
+  subdomain,
+  filePath,
+  fileName,
+  mimetype,
+  forcePrivate = false,
+}: UploadFileToStorageParams): Promise<string> => {
+  const configs = await fetchStorageConfigs(subdomain);
+  const uploadType = readConfigValue(
+    configs,
+    'UPLOAD_SERVICE_TYPE',
+    'AWS',
+  ).toUpperCase();
+
+  const version = getEnv({ name: 'VERSION' });
+  const shouldForcePrivate = forcePrivate || version === 'saas';
+
+  if (uploadType === 'GCS') {
+    return await uploadToGCS(filePath, fileName, mimetype, configs);
+  }
+
+  if (uploadType === 'AWS') {
+    return await uploadToAWS(
+      filePath,
+      fileName,
+      mimetype,
+      configs,
+      shouldForcePrivate,
+    );
+  }
+
+  if (uploadType === 'CLOUDFLARE') {
+    return await uploadToCloudflare(
+      filePath,
+      fileName,
+      mimetype,
+      configs,
+      shouldForcePrivate,
+    );
+  }
+
+  if (uploadType === 'AZURE') {
+    return await uploadToAzure(filePath, fileName, mimetype, configs);
+  }
+
+  if (uploadType === 'LOCAL') {
+    throw new Error(
+      'Local storage cannot be accessed from shared utilities. Use server-side file upload instead.',
+    );
+  }
+
+  throw new Error(
+    `Unsupported upload service type "${uploadType}" while uploading "${fileName}"`,
+  );
+};

package/src/utils/graphqlPubSub.ts

@@ -0,0 +1,28 @@
+import * as dotenv from 'dotenv';
+import { RedisPubSub } from 'graphql-redis-subscriptions';
+import Redis from 'ioredis';
+
+// load environment variables
+dotenv.config();
+
+const { REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, SKIP_REDIS } = process.env;
+
+const redisOptions = {
+  host: REDIS_HOST,
+  port: Number.parseInt(REDIS_PORT || '6379', 10),
+  password: REDIS_PASSWORD,
+};
+
+const createPubsubInstance = () => {
+  return new RedisPubSub({
+    connectionListener: (error) => {
+      if (error) {
+        console.error(error);
+      }
+    },
+    publisher: new Redis(redisOptions),
+    subscriber: new Redis(redisOptions),
+  });
+};
+
+export const graphqlPubsub = createPubsubInstance();

package/src/utils/headers/get-hostname.ts

@@ -0,0 +1,18 @@
+import { IncomingMessage } from 'http';
+
+type HostnameGetter = (req: IncomingMessage) => string | undefined;
+
+// trimmed version of getHostname, since we don't enable trust
+export const getHostnameZeroTrust: HostnameGetter = (
+  req: IncomingMessage,
+): string | undefined => {
+  const host = req.headers.host;
+
+  if (!host) return;
+
+  // IPv6 literal support
+  const offset = host[0] === '[' ? host.indexOf(']') + 1 : 0;
+  const index = host.indexOf(':', offset);
+
+  return index !== -1 ? host.substring(0, index) : host;
+};

package/src/utils/headers/index.ts

@@ -0,0 +1,4 @@
+export * from './sanitize';
+export * from './user';
+export * from './get-hostname';
+export * from './subdomain';

package/src/utils/headers/sanitize.ts

@@ -0,0 +1,9 @@
+import { IncomingHttpHeaders } from 'http';
+import { userHeaderName } from './user';
+import { erxesSubdomainHeaderName } from './subdomain';
+
+export function sanitizeHeaders(headers: IncomingHttpHeaders) {
+  delete headers[erxesSubdomainHeaderName];
+  delete headers[userHeaderName];
+  delete headers['userid'];
+}

package/src/utils/headers/subdomain.ts

@@ -0,0 +1,13 @@
+import { IncomingMessage } from 'http';
+
+export const erxesSubdomainHeaderName = 'erxes-subdomain';
+
+export const getSubomdainHeader = (
+  req: IncomingMessage,
+): string | undefined => {
+  const subdomainHeader = req.headers[erxesSubdomainHeaderName];
+  if (Array.isArray(subdomainHeader)) {
+    throw new Error(`Multiple ${erxesSubdomainHeaderName} headers!`);
+  }
+  return subdomainHeader;
+};

package/src/utils/headers/user.test.ts

@@ -0,0 +1,76 @@
+import { IncomingHttpHeaders } from 'http';
+import {
+  compactUserForHeader,
+  extractUserFromHeader,
+  setUserHeader,
+  USER_HEADER_OMITTED_FIELDS,
+} from './user';
+
+describe('user header', () => {
+  const user = {
+    _id: 'u1',
+    email: 'mj@example.com',
+    isOwner: true,
+    groupIds: ['g1'],
+    oauthScopes: ['sales-deal:manage', 'contacts:read'],
+    emailSignatures: [{ brandId: 'b1', signature: 'x'.repeat(4000) }],
+    customFieldsData: [{ field: 'f1', value: 'y'.repeat(3000) }],
+    propertiesData: [{ field: 'f2', value: 'z' }],
+    links: { website: 'https://example.com' },
+    loginToken: 'jwt.'.repeat(200),
+  };
+
+  it('forwards identity, permissions and oauth scopes but not profile bulk', () => {
+    const headers: IncomingHttpHeaders = {};
+    setUserHeader(headers, user);
+    const forwarded = extractUserFromHeader(headers);
+
+    expect(forwarded._id).toBe('u1');
+    expect(forwarded.isOwner).toBe(true);
+    expect(forwarded.groupIds).toEqual(['g1']);
+    expect(forwarded.oauthScopes).toEqual([
+      'sales-deal:manage',
+      'contacts:read',
+    ]);
+    expect(headers.userid).toBe('u1');
+
+    for (const field of USER_HEADER_OMITTED_FIELDS) {
+      expect(forwarded).not.toHaveProperty(field);
+    }
+  });
+
+  it('keeps the header well under the default 16 KB request-header limit', () => {
+    const headers: IncomingHttpHeaders = {};
+    setUserHeader(headers, {
+      ...user,
+      oauthScopes: Array.from(
+        { length: 120 },
+        (_, i) => `plugin-module${i}:manage`,
+      ),
+    });
+
+    expect(String(headers.user).length).toBeLessThan(8 * 1024);
+  });
+
+  it('warns before a header outgrows the service budget', () => {
+    const warn = jest
+      .spyOn(console, 'warn')
+      .mockImplementation(() => undefined);
+    const headers: IncomingHttpHeaders = {};
+
+    setUserHeader(headers, {
+      ...user,
+      departmentIds: Array(4000).fill('d'.repeat(10)),
+    });
+
+    expect(warn).toHaveBeenCalledTimes(1);
+    expect(warn.mock.calls[0][0]).toContain('u1');
+    warn.mockRestore();
+  });
+
+  it("does not mutate the caller's user object", () => {
+    const copy = { ...user };
+    compactUserForHeader(copy);
+    expect(copy.emailSignatures).toBeDefined();
+  });
+});