@joystick.js/node-canary 0.0.0-canary.47 → 0.0.0-canary.471

package/src/app/uploaders/run_upload.js

@@ -0,0 +1,136 @@
+import fs from 'fs';
+import path from 'path';
+import { S3Client } from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+import emit_websocket_event from "../websockets/emit_event.js";
+import float_to_decimal_place from "../../lib/float_to_decimal_place.js";
+import path_exists from "../../lib/path_exists.js";
+
+const { mkdir, writeFile } = fs.promises;
+
+const upload_to_s3 = async (upload = {}, upload_options = {}, on_upload_progress = {}) => {
+  return new Promise(async (resolve, reject) => {
+    try {
+      const s3_upload = new Upload({
+        client: new S3Client({
+          region: upload?.s3?.region,
+          credentials:{
+            accessKeyId: upload?.s3?.accessKeyId || upload?.s3?.access_key_id,
+            secretAccessKey: upload?.s3?.secretAccessKey || upload?.s3?.secret_access_key,
+          },
+        }),
+        params: {
+          Bucket: upload?.s3?.bucket,
+          Key: upload.file_name,
+          Body: upload.content,
+          ContentType: upload.mime_type,
+        },
+        queueSize: 4, // NOTE: How many parts to upload concurrently.
+        partSize: 1024 * 1024 * 5, // NOTE: Size of each part (5MB).
+      });
+
+      let previous = 0;
+
+      s3_upload.on("httpUploadProgress", (progress) => {
+        on_upload_progress('s3', progress?.loaded - previous);
+        previous = progress?.loaded;
+      });
+
+      const upload_result = await s3_upload.done();
+      
+      return resolve({
+        id: upload_options?.req?.headers['x-joystick-upload-id'],
+        provider: 's3',
+        url: upload_result?.Location,
+        size: upload?.fileSize,
+        file_name: upload?.file_name,
+        fileName: upload?.file_name,
+        original_file_name: upload?.original_file_name,
+        originalFileName: upload?.original_file_name,
+        mime_type: upload?.mime_type,
+        mimeType: upload?.mime_type,
+      });
+    } catch (error) {
+      reject(error?.message || 'There was an error uploading your file to Amazon S3. Check the server logs for more information.');
+    }
+  });
+};
+
+const upload_to_local = async (upload = {}, upload_options = {}) => {
+  if (upload?.local?.path) {
+  	return new Promise(async (resolve, reject) => {
+	    try {
+        if (!(await path_exists(upload?.local?.path))) {
+          await mkdir(upload?.local?.path, { recursive: true });
+        }
+
+        const directory_path = path.dirname(`${upload?.local?.path}/${upload?.file_name}`);
+        
+        if (!fs.existsSync(directory_path)) {
+          fs.mkdirSync(directory_path, { recursive: true });
+        }
+
+        const file_path = `${upload?.local?.path}/${upload?.file_name}` || `_uploads/${upload?.file_name}`;
+
+        await writeFile(file_path, upload.content);
+
+        resolve({
+          id: upload_options?.req?.headers['x-joystick-upload-id'],
+          provider: 'local',
+          url: file_path,
+          size: upload?.file_size,
+          file_name: upload?.file_name,
+          fileName: upload?.file_name,
+          original_file_name: upload?.original_file_name,
+          originalFileName: upload?.original_file_name,
+          mime_type: upload?.mime_type,
+          mimeType: upload?.mime_type,
+        });
+      } catch(error) {
+        reject(error);
+      }
+  	});
+  }
+};
+
+const run_upload = async (run_upload_options = {}) => {
+  const uploads = [];
+  let existing_upload_progress = run_upload_options?.existing_upload_progress;
+
+  // NOTE: Use a closure here so we can pass it into the provider uploaders and avoid the
+  // existing_upload_progress value being cached (leading to incorrect progress percentage).
+  const on_upload_progress = (provider = '', chunk = 0) => {
+    const progress = existing_upload_progress + chunk;
+    const percentage = float_to_decimal_place(((progress / run_upload_options?.total_upload_size_all_providers) * 100));
+
+    emit_websocket_event(
+    	`uploaders_${run_upload_options?.req?.headers['x-joystick-upload-id']}`,
+    	'progress',
+      { provider, progress: percentage }
+    );
+
+    existing_upload_progress += chunk;
+  };
+
+  for (let i = 0; i < run_upload_options?.uploads?.length; i += 1) {
+    const upload = run_upload_options?.uploads[i];
+
+    if (upload?.providers?.includes('local')) {
+      // NOTE: Do not pass existing_upload_progress or onProgress to upload_to_local as the existing total reflects
+      // the transfer from the browser to the server (we assume that value is 1:1 with writing to disk
+      // as the disk write is near-instant).
+      const result = await upload_to_local(upload, run_upload_options);
+
+      uploads.push(result);
+    }
+
+    if (upload?.providers?.includes('s3')) {
+      const result = await upload_to_s3(upload, run_upload_options, on_upload_progress);
+      uploads.push(result);
+    }
+  }
+
+  return uploads;
+};
+
+export default run_upload;

package/src/app/uploaders/validate_options.js

@@ -0,0 +1,74 @@
+const allowed_options = [
+  'mimeTypes',
+  'mime_types',
+  'maxSizeInMegabytes',
+  'max_size_in_megabytes',
+  'fileName',
+  'file_name',
+  'providers',
+  'local',
+  's3',
+  'onBeforeUpload',
+  'on_before_upload',
+  'onAfterUpload',
+  'on_after_upload',
+];
+
+const allowed_local_options = [
+  'path',
+];
+
+const allowed_s3_options = [
+  'region',
+  'accessKeyId',
+  'access_key_id',
+  'secretAccessKey',
+  'secret_access_key',
+  'bucket',
+  'acl',
+];
+
+const validate_options = (uploader_options = {}) => {
+  const errors = [];
+  const option_names = Object.keys(uploader_options || {});
+
+  for (let i = 0; i < option_names?.length; i += 1) {
+  	const option_name = option_names[i];
+  	if (!allowed_options.includes(option_name)) {
+  		errors.push(`${option_name} is not an allowed uploader option.`);
+  	}
+  }
+
+  if (uploader_options?.providers?.includes('local') && !uploader_options?.local) {
+    errors.push(`If an uploader provider is 'local', local object must be specified in uploader_options with a storage path.`);
+  }
+
+  if (uploader_options?.providers?.includes('s3') && !uploader_options?.s3) {
+    errors.push(`If an uploader provider is 's3', s3 object must be specified with configuration.`);
+  }
+
+  if (uploader_options?.provider?.includes('local') && uploader_options.local) {
+  	const local_uploader_options = Object.keys(uploader_options?.local);
+  	for (let i = 0; i < local_uploader_options?.length; i += 1) {
+  		const local_uploader_option = local_uploader_options[i];
+  		if (!allowed_local_options.includes(local_uploader_option)) {
+        errors.push(`local.${local_uploader_option} is not an allowed uploader option.`);
+      }
+  	}
+  }
+
+  if (uploader_options?.provider?.includes('s3') && uploader_options.s3) {
+  	const s3_uploader_options = Object.keys(uploader_options.s3 || {});
+
+  	for (let i = 0; i < s3_uploader_options?.length; i += 1) {
+  		const s3_uploader_option = s3_uploader_options[i];
+  		if (!allowed_s3_options.includes(s3_uploader_option)) {
+        errors.push(`local.${s3_uploader_option} is not an allowed uploader option.`);
+      }
+  	}
+  }
+
+  return errors;
+};
+
+export default validate_options;

package/src/app/uploaders/validate_uploads.js

@@ -0,0 +1,127 @@
+import log from "../../lib/log.js";
+import track_function_call from "../../test/track_function_call.js";
+import types from "../../lib/types.js";
+
+const handle_check_upload = ({
+  uploader_name,
+  max_size_in_megabytes,
+  mime_types,
+  original_file_name,
+  file_size,
+  mime_type,
+}) => {
+  const max_size_in_bytes = max_size_in_megabytes * 1024 * 1024;
+  const errors = [];
+
+  if (types.is_number(max_size_in_bytes) && file_size && file_size > max_size_in_bytes) {
+    const error = `The file ${original_file_name} is too big. Max file size is ${max_size_in_megabytes}MB.`;
+
+    log(error, {
+      level: 'danger',
+      docs: 'https://docs.cheatcode.co/joystick/node/uploaders',
+    });
+
+    errors.push({ status: 400, error: new Error(error), message: error, location: `uploaders.${uploader_name}` });
+  }
+
+  if (mime_types?.length > 0 && !mime_types.includes(mime_type)) {
+    const error = `The MIME type for the file ${original_file_name} is not supported by the uploader ${uploader_name} (only ${mime_types?.join(', ')}).`;
+
+    log(error, {
+      level: 'danger',
+      docs: 'https://docs.cheatcode.co/joystick/node/uploaders',
+    });
+
+    errors.push({ status: 400, error: new Error(error), message: error, location: `uploaders.${uploader_name}` });
+  }
+
+  return errors;
+};
+
+const handle_check_uploads = (uploads = []) => {
+  return uploads.flatMap((upload = {}) => {
+    return handle_check_upload(upload);
+  });
+};
+
+const format_uploads = ({
+  files: uploads = [],
+  uploader_name = '',
+  uploader_options = {},
+  uploader_input = {},
+}) => {
+  return Promise.all(uploads.map(async (upload) => {
+  	// NOTE: The upload passed in here is controlled by multer, not Joystick.
+  	// This is why property names are not converted to snake_case.
+    const file_extension = upload?.mimetype?.split('/').pop();
+    const file_name_is_function = types.is_function(uploader_options?.fileName) || types.is_function(uploader_options?.file_name);
+    const max_size_is_function = types.is_function(uploader_options?.maxSizeInMegabytes) || types.is_function(uploader_options?.max_size_in_megabytes);
+
+    if (file_name_is_function) {
+      track_function_call(`node.uploaders.${uploader_name}.file_name`, [{
+        uploader_input,
+        file_name: upload?.originalname,
+        file_size: upload?.size,
+        file_extension,
+        mime_type: upload?.mimetype
+      }]);
+    }
+
+    if (max_size_is_function) {
+      track_function_call(`node.uploaders.${uploader_name}.max_size_in_megabytes`, [
+        { input: uploader_input, upload }
+      ]);
+    }
+
+    const max_size_in_megabytes = max_size_is_function ?
+      await (uploader_options?.maxSizeInMegabytes || uploader_options?.max_size_in_megabytes)({ input: uploader_input, upload }) :
+      (uploader_options?.maxSizeInMegabytes || uploader_options?.max_size_in_megabytes);
+
+    const file_name = file_name_is_function ?
+        (uploader_options.fileName || uploader_options?.file_name)({
+          input: uploader_input,
+          fileName: upload?.originalname,
+          file_name: upload?.originalname,
+          fileSize: upload?.size,
+          file_size: upload?.size,
+          fileExtension: file_extension,
+          file_extension,
+          mimeType: upload?.mimetype,
+          mime_type: upload?.mimetype
+        }) :
+        upload?.originalname;
+
+    return {
+      uploader_name,
+      providers: uploader_options?.providers,
+      local: uploader_options?.local,
+      s3: uploader_options?.s3,
+      maxSizeInMegabytes: max_size_in_megabytes,
+      max_size_in_megabytes,
+      mimeTypes: uploader_options?.mimeTypes,
+      mime_types: uploader_options?.mimeTypes,
+      fileName: file_name,
+      file_name,
+      original_file_name: upload?.originalname,
+      originalFileName: upload?.originalname,
+      file_size: upload?.size,
+      fileSize: upload?.size,
+      mime_type: upload?.mimetype,
+      mimeType: upload?.mimetype,
+      content: upload?.buffer,
+    };
+  }));
+};
+
+const validate_uploads = async (validate_uploads_options = {}) => {
+  const formatted_uploads = await format_uploads(validate_uploads_options);
+  const errors = handle_check_uploads(formatted_uploads);
+
+  if (errors?.length > 0) {
+    return { errors };
+  }
+  
+  return formatted_uploads;
+};
+
+export default validate_uploads;

package/src/app/websockets/emit_event.js

@@ -0,0 +1,14 @@
+import types from "../../lib/types.js";
+
+const emit_event = (emitter_id = '', event_name = '', payload = {}) => {
+  const emitters = joystick?.emitters[emitter_id];
+
+  if (types.is_array(emitters)) {
+    for (let i = 0; i < emitters?.length; i += 1) {
+      const emitter_recipient = emitters[i];
+      emitter_recipient.emit(event_name, payload);
+    }
+  }
+};
+
+export default emit_event;

package/src/app/websockets/index.js

@@ -0,0 +1,16 @@
+import emit_websocket_event from "./emit_event.js";
+import track_function_call from "../../test/track_function_call.js";
+
+const websockets = (server_name = '') => {
+  return {
+    send: (payload = {}, unique_connection_id = '') => {
+      const emitter_name = unique_connection_id ? `${server_name}_${unique_connection_id}` : server_name;
+      emit_websocket_event(emitter_name, 'message', payload);
+      track_function_call(`node.websockets.${server_name}.send`, [
+        payload
+      ]);
+    },
+  };
+};
+
+export default websockets;

package/src/app/websockets/register.js

@@ -0,0 +1,175 @@
+import EventEmitter from "events";
+import queryString from 'query-string';
+import * as WebSocket from "ws";
+import track_function_call from "../../test/track_function_call.js";
+import types from "../../lib/types.js";
+
+const handle_websocket_connection_upgrade = (express_server = {}, websocket_servers = {}) => {
+	express_server.on('upgrade', (request, socket, head) => {
+		if (request?.url?.includes('/api/_websockets')) {
+			const websocket_name = (request?.url?.replace('/api/_websockets/', '')?.split('?') || [])[0];
+			const websocket_definition = websocket_servers[websocket_name];
+
+			if (websocket_definition) {
+				websocket_definition.server.handleUpgrade(request, socket, head, (socket) => {
+					websocket_definition.server.emit('connection', socket, request);
+				});
+			}
+		}
+	});
+};
+
+const handle_websocket_connection_events = (websocket_connection = {}, connection_params = {}, websocket_definition = {}) => {
+  const connection = Object.assign(websocket_connection, { params: connection_params });
+
+  if (types.is_function(websocket_definition?.onOpen) || types.is_function(websocket_definition?.on_open)) {
+    (websocket_definition?.onOpen || websocket_definition?.on_open)(connection);
+  }
+
+  // NOTE: This is a message being sent inbound by the client.
+  websocket_connection.on("message", (message) => {
+    const message_as_json = JSON.parse(message);
+
+    if (types.is_function(websocket_definition?.on_message)) {
+    	websocket_definition?.on_message(message_as_json, websocket_connection);
+  	}
+  });
+
+  websocket_connection.on("close", (code = 0, reason = "") => {
+  	if (types.is_function(websocket_definition?.on_close)) {
+    	websocket_definition?.on_close(code, reason?.toString(), websocket_connection);
+  	}
+  });
+};
+
+const register_websocket_event_emitter = (websocket_name = '', connection_params = {}, websocket_connection = {}) => {
+	const emitter = new EventEmitter();
+	const emitter_id = connection_params?.id ? `${websocket_name}_${connection_params?.id}` : websocket_name;
+
+	if (joystick?.emitters && joystick?.emitters[emitter_id]) {
+	  joystick.emitters[emitter_id].push(emitter);
+	} else {
+	  joystick.emitters = {
+	    ...(joystick?.emitters || {}),
+	    [emitter_id]: [emitter],
+	  };
+	}
+
+  // NOTE: This is a message being sent outbound by the server. This is already contextualized
+  // to a unique ID (if applicable) via the joystick.emitters.<emitterId> pattern above.
+  emitter.on("message", (message = {}) => {
+    websocket_connection.send(JSON.stringify(message));
+  });
+
+  // NOTE: This is an internal emitter for handling progress events via uploaders.
+  emitter.on("progress", (progress = {}) => {
+    websocket_connection.send(
+      JSON.stringify({ type: "PROGRESS", ...progress })
+    );
+  });
+};
+
+const get_websocket_connection_params = (websocket_connection_request = {}) => {
+  const [_path, params] = websocket_connection_request?.url?.split("?");
+  return queryString.parse(params);
+};
+
+const handle_websocket_connection = (websocket_name = '', websocket_definition = {}) => {
+	websocket_definition?.server.on('connection', (websocket_connection = {}, websocket_connection_request = {}) => {
+		const connection_params = get_websocket_connection_params(websocket_connection_request);
+		register_websocket_event_emitter(websocket_name, connection_params, websocket_connection);
+		handle_websocket_connection_events(websocket_connection, connection_params, websocket_definition);
+	});
+};
+
+const handle_on_close_event = (user_websocket_name = '', user_websocket_definition = {}, websocket_event_args = []) => {
+  track_function_call(`node.websockets.${user_websocket_name}.on_close`, websocket_event_args);
+  return (user_websocket_definition?.onClose || user_websocket_definition?.on_close) ?
+  	(user_websocket_definition?.onClose || user_websocket_definition?.on_close)(...websocket_event_args) :
+  	null;
+};
+
+const handle_on_message_event = (user_websocket_name = '', user_websocket_definition = {}, websocket_event_args = []) => {
+  track_function_call(`node.websockets.${user_websocket_name}.on_message`, websocket_event_args);
+  return (user_websocket_definition?.onMessage || user_websocket_definition?.on_message) ?
+  	(user_websocket_definition?.onMessage || user_websocket_definition?.on_message)(...websocket_event_args) :
+  	null;
+};
+
+const handle_on_open_event = (user_websocket_name = '', user_websocket_definition = {}, websocket_event_args = []) => {
+  track_function_call(`node.websockets.${user_websocket_name}.on_open`, websocket_event_args);
+  return (user_websocket_definition?.onOpen || user_websocket_definition?.on_open) ?
+  	(user_websocket_definition?.onOpen || user_websocket_definition?.on_open)(...websocket_event_args) :
+  	null;
+};
+
+const get_websocket_server = (websocket_name = '') => {
+	return new WebSocket.WebSocketServer({
+    noServer: true,
+    path: `/api/_websockets/${websocket_name}`,
+  });
+};
+
+const get_websocket_servers = (user_websocket_definitions = {}) => {
+	return {
+		uploaders: {
+			server: get_websocket_server('uploaders'),
+		},
+		...Object.entries(user_websocket_definitions || {}).reduce(
+			(websockets_to_define = {}, [user_websocket_name, user_websocket_definition]) => {
+				websockets_to_define[user_websocket_name] = {
+					server: get_websocket_server(user_websocket_name),
+					on_open: (...websocket_event_args) => handle_on_open_event(
+						user_websocket_name,
+						user_websocket_definition,
+						websocket_event_args,
+					),
+					onOpen: (...websocket_event_args) => handle_on_open_event(
+						user_websocket_name,
+						user_websocket_definition,
+						websocket_event_args,
+					),
+					on_message: (...websocket_event_args) => handle_on_message_event(
+						user_websocket_name,
+						user_websocket_definition,
+						websocket_event_args,
+					),
+					onMessage: (...websocket_event_args) => handle_on_message_event(
+						user_websocket_name,
+						user_websocket_definition,
+						websocket_event_args,
+					),
+					on_close: (...websocket_event_args) => handle_on_close_event(
+						user_websocket_name,
+						user_websocket_definition,
+						websocket_event_args,
+					),
+					onClose: (...websocket_event_args) => handle_on_close_event(
+						user_websocket_name,
+						user_websocket_definition,
+						websocket_event_args,
+					),
+				};
+
+				return websockets_to_define;
+			},
+			{}
+		),
+	};
+};
+
+const register = (user_websocket_definitions = {}, app_instance = {}) => {
+	const websocket_servers_to_create = get_websocket_servers(user_websocket_definitions);
+	const websocket_servers = Object.entries(websocket_servers_to_create);
+
+	for (let i = 0; i < websocket_servers?.length; i += 1) {
+		const [websocket_name, websocket_definition] = websocket_servers[i];
+		handle_websocket_connection(websocket_name, websocket_definition);
+	}
+
+	// NOTE: Doing this here is intentional. Because the upgrade emits a connection event,
+  // the connection event handler must be defined first (what we're doing above).
+  handle_websocket_connection_upgrade(app_instance.express.server, websocket_servers_to_create);
+};
+
+export default register;

package/src/index.js

@@ -0,0 +1,81 @@
+import fs from 'fs';
+import _accounts from './app/accounts/index.js';
+import _action from "./action/index.js";
+import _escape_html from './lib/escape_html.js';
+import _fixture from './app/fixture/index.js';
+import _sql from './app/databases/sql.js';
+import _validate_input from './app/api/validate_input.js';
+import _websockets from './app/websockets/index.js';
+import app from './app/index.js';
+import generate_id from './lib/generate_id.js';
+import get_origin from './lib/get_origin.js';
+import load_settings from './app/settings/load.js';
+import node_path_polyfills from './lib/node_path_polyfills.js';
+import path_exists from './lib/path_exists.js';
+import send_email from './app/email/send.js';
+
+const { readFile } = fs.promises;
+
+// NOTE: Ensure backwards compatibility for existing apps by offering
+// original camelCase versions of methods.
+export const accounts = {
+	..._accounts,
+  deleteUser: _accounts.delete_user,
+  recoverPassword: _accounts.recover_password,
+  resetPassword: _accounts.reset_password,
+  sendEmailVerification: _accounts.send_email_verification,
+  setPassword: _accounts.set_password,
+  verifyEmail: _accounts.verify_email,
+};
+
+export const action = _action;
+
+export const __dirname = node_path_polyfills.__dirname;
+
+export const email = {
+  send: send_email,
+};
+
+export const escape_html = _escape_html;
+
+export const __filename = node_path_polyfills.__filename;
+
+export const fixture = _fixture;
+
+export const origin = get_origin();
+
+export const push = {
+  continent: (await path_exists('/root/push/continent.txt')) ? (await readFile('/root/push/continent.txt', 'utf-8'))?.replace('\n', '') : null,
+  instance_token: (await path_exists('/root/push/instance_token.txt')) ? (await readFile('/root/push/instance_token.txt', 'utf-8'))?.replace('\n', '') : null,
+  current_version: (await path_exists('/root/push/versions/current')) ? (await readFile('/root/push/versions/current', 'utf-8'))?.replace('\n', '') : null,
+};
+
+export const settings = load_settings();
+
+export const sql = _sql;
+
+export const validate_input = _validate_input;
+
+export const websockets = _websockets;
+
+const joystick = {
+	app,
+	accounts,
+  action,
+  email,
+  emitters: {},
+  escape_html,
+  fixture,
+  id: generate_id,
+  origin,
+  push,
+  settings,
+  sql,
+  validate_input,
+  websockets,
+  ...node_path_polyfills,
+};
+
+global.joystick = joystick;
+
+export default joystick;

package/{dist/lib/camelPascalToSnake.js → src/lib/camel_pascal_to_snake.js}

@@ -1,8 +1,7 @@
-var camelPascalToSnake_default = (string = "") => {
+const camel_pascal_to_snake = (string = '') => {
   return string.replace(/[A-Z]/g, (letter, index) => {
     return index == 0 ? letter.toLowerCase() : `_${letter.toLowerCase()}`;
   });
 };
-export {
-  camelPascalToSnake_default as default
-};
+
+export default camel_pascal_to_snake;

package/src/lib/constants.js

@@ -0,0 +1,21 @@
+export const HTML_ENTITY_MAP = {
+  '&': '&',
+  '<': '&lt;',
+  '>': '&gt;',
+  '"': '&quot;',
+  "'": '&#39;',
+  '/': '&#x2F;',
+  '`': '&#x60;',
+  '=': '&#x3D;'
+};
+
+export const KILOBYTE = 1000;
+export const MEGABYTE = KILOBYTE * 1000;
+
+const constants = {
+	HTML_ENTITY_MAP,
+	KILOBYTE,
+	MEGABYTE,
+};
+
+export default constants;

package/src/lib/dynamic_import.js

@@ -0,0 +1,8 @@
+import get_platform_safe_path from "./get_platform_safe_path.js";
+
+const dynamic_import = async (path = '') => {
+  const file = await import(process.platform === 'win32' ? `file://${path}` : path);
+  return file?.default;
+};
+
+export default dynamic_import;

package/src/lib/escape_html.js

@@ -0,0 +1,9 @@
+import constants from "./constants.js";
+
+const escape_html = (string = '') => {
+  return String(string).replace(/[&<>"'`=]/g, function (match) {
+    return constants.HTML_ENTITY_MAP[match];
+  });
+};
+
+export default escape_html;