@joystick.js/cli-canary 0.0.0-canary.162 → 0.0.0-canary.1621

package/src/lib/development_server/ava_config.js

@@ -0,0 +1,14 @@
+const ava_config = {
+  files: [
+    `tests/**/*.test.js`,
+  ],
+  ignoredByWatcher: ['.joystick', './uploads'],
+  // NOTE: Intentionally limit to 1 test at a time to avoid concurrency creating
+  // race conditions in tests that have users or other global data that needs
+  // to be cleaned up. The extra time needed is worth it to guarantee that tests
+  // don't step on each other's toes and our code is properly verified.
+  concurrency: 1,
+  cache: false,
+};
+
+export default ava_config;

package/src/lib/development_server/check_if_port_occupied.js

@@ -0,0 +1,31 @@
+import net from 'net';
+
+const reset_client = (client = {}) => {
+  client.end();
+  client.destroy();
+  client.unref();  
+};
+
+const check_if_port_occupied = (port = 2600) => {
+  return new Promise((resolve) => {
+    const client = new net.Socket();
+
+    // NOTE: This is inverted. If we can connect, the port is occupied. If we can't, the port is free.
+    client.once('connect', () => {
+      reset_client(client);
+      resolve(true);
+    });
+
+    client.once('error', () => {
+      reset_client(client);
+      resolve(false);
+    });
+
+    client.connect({
+      port,
+      host: '127.0.0.1',
+    }, function() {});
+  });
+};
+
+export default check_if_port_occupied;

package/src/lib/development_server/cleanup.js

@@ -0,0 +1,15 @@
+// NOTE: This module is intended to be run as a child_process as part of
+// a SIGINT or SIGTERM event.
+
+import kill_process_ids from "./kill_process_ids.js";
+
+process.title = "joystick_cleanup";
+
+process.on('message', async (message) => {
+  const parsed_message = JSON.parse(message);
+  const process_ids = parsed_message?.process_ids;
+
+  kill_process_ids(process_ids);
+
+  process.exit();
+});

package/src/lib/development_server/databases/download_database_binary.js

@@ -0,0 +1,35 @@
+import os from 'os';
+import mongodb_macos from './mongodb/installers/macos.js';
+import mongodb_windows from './mongodb/installers/windows.js';
+import mongodb_linux from './mongodb/installers/linux.js';
+import postgresql_macos from './postgresql/installers/macos.js';
+import postgresql_windows from './postgresql/installers/windows.js';
+import postgresql_linux from './postgresql/installers/linux.js';
+
+const installers = {
+  darwin: {
+    mongodb: mongodb_macos,
+    postgresql: postgresql_macos,
+  },
+  win32: {
+    mongodb: mongodb_windows,
+    postgresql: postgresql_windows,
+  },
+  linux: {
+    mongodb: mongodb_linux,
+    postgresql: postgresql_linux,
+  },
+};
+
+const download_database_binary = async (database_name = '') => {
+  const platform = os.platform();
+  const installer = installers[platform] && installers[platform][database_name];
+
+  if (typeof installer === 'function') {
+    await installer();
+  } else {
+    console.warn(`${database_name} not supported on ${platform}`);
+  }
+};
+
+export default download_database_binary;

package/src/lib/{dev/databases/mongodb/availableQueryParameters.js → development_server/databases/mongodb/available_query_parameters.js}

@@ -1,4 +1,4 @@
-export default [
+const available_query_parameters = [
   "replicaSet",
   "tls",
   "ssl",
@@ -37,3 +37,5 @@ export default [
   "retryWrites",
   "uuidRepresentation"
 ];
+
+export default available_query_parameters;

package/src/lib/development_server/databases/mongodb/build_connection_string.js

@@ -0,0 +1,36 @@
+import build_query_parameters from "./build_query_parameters.js";
+import serialize_query_parameters from "../../../serialize_query_parameters.js";
+import types from "../../../types.js";
+
+const build_connection_string = (connection = {}) => {
+  let connection_string = "mongodb://";
+
+  // NOTE: Swap protocols if we're using a DNS Seedlist URL (mongodb+srv://).
+  if (connection?.srv) {
+    connection_string = "mongodb+srv://";
+  }
+
+  if (connection && (connection.username || connection.password)) {
+    connection_string = `${connection_string}${connection.username || connection.password ? `${connection.username || ""}${!!connection.username && !!connection.password ? ':' : ''}${connection.password || ""}@` : ''}`;
+  }
+
+  if (connection && connection.hosts && types.is_array(connection.hosts)) {
+    connection_string = `${connection_string}${connection.hosts
+      .map((host) => connection?.srv ? host.hostname : `${host.hostname}:${host.port}`)
+      .join(",")}`;
+  }
+
+  if (connection && connection.database) {
+    connection_string = `${connection_string}/${connection.database}`;
+  }
+
+  const query_parameters = build_query_parameters(connection);
+
+  if (Object.keys(query_parameters)?.length > 0 ) {
+    connection_string = `${connection_string}?${serialize_query_parameters(query_parameters)}`;
+  }
+
+  return connection_string;
+};
+
+export default build_connection_string;

package/src/lib/development_server/databases/mongodb/build_query_parameters.js

@@ -0,0 +1,17 @@
+import available_query_parameters from "./available_query_parameters.js";
+
+const build_query_parameters = (connection = {}) => {
+  const query_parameters = {};
+
+  for (let i = 0; i < available_query_parameters.length; i += 1) {
+    const available_query_parameter = available_query_parameters[i];
+
+    if (connection && connection[available_query_parameter]) {
+      query_parameters[available_query_parameter] = connection[available_query_parameter];
+    }
+  }
+
+  return query_parameters;
+};
+
+export default build_query_parameters;

package/src/lib/development_server/databases/mongodb/check_connection.js

@@ -0,0 +1,41 @@
+import { MongoClient } from "mongodb";
+import fs from 'fs';
+import build_connection_string from "./build_connection_string.js";
+import color_log from '../../../color_log.js';
+
+const { readFile } = fs.promises;
+
+const check_connection = async (connection = {}, mongodb_options = {}) => {
+  const connection_string = build_connection_string(connection);
+  
+  try {
+    const connectionOptions = {
+      connectTimeoutMS: 3000,
+      socketTimeoutMS: 3000,
+      useNewUrlParser: true,
+      useUnifiedTopology: true,
+      ssl: false,
+      ...(mongodb_options || {}),
+    };
+
+    if (mongodb_options?.ca) {
+      connectionOptions.ca = await readFile(mongodb_options?.ca);
+    }
+
+    const connection = await MongoClient.connect(connection_string, connectionOptions);
+    connection.close();
+
+    return true;
+  } catch (exception) {
+    console.warn(exception);
+
+    color_log(
+    	`\n✖ Failed to connect to MongoDB. Please double-check connection settings and try again.`,
+    	'yellow'
+    );
+
+    process.exit(1);
+  }
+};
+
+export default check_connection;

package/src/lib/development_server/databases/mongodb/connect.js

@@ -0,0 +1,27 @@
+import check_connection from "./check_connection.js";
+import mongodb from "./index.js";
+
+const connect = async (settings = {}, port = 2610) => {
+  const has_connection = settings.connection && Object.keys(settings.connection).length > 0;
+
+  if (has_connection) {
+    await check_connection(settings.connection, settings.options);
+  }
+
+  return {
+    pid: !has_connection ? await mongodb(port) : null,
+    connection: has_connection ? settings.connection : {
+      hosts: [
+        {
+          hostname: "127.0.0.1",
+          port,
+        },
+      ],
+      database: "app",
+      username: "",
+      password: "",
+    },
+  };
+};
+
+export default connect;

package/src/lib/development_server/databases/mongodb/index.js

@@ -0,0 +1,102 @@
+import child_process from "child_process";
+import fs from "fs";
+import os from "os";
+import get_platform_safe_path from '../../../get_platform_safe_path.js';
+import get_process_id_from_port from "../../../get_process_id_from_port.js";
+import kill_port_process from "../../../kill_port_process.js";
+import path_exists from "../../../path_exists.js";
+
+const { rename, mkdir } = fs.promises;
+
+const get_mongo_shell_command = () => {
+  if (process.platform === 'win32') {
+    return 'mongosh.exe';
+  }
+
+  return 'mongosh';
+};
+
+const get_mongo_server_command = () => {
+  if (process.platform === 'win32') {
+    return 'mongod.exe';
+  }
+
+  return 'mongod';
+};
+
+const start_mongodb_process = (mongodb_port = 2610) => {
+  return new Promise((resolve) => {
+    // TODO: Does this hold up on Linux?
+    const mongo_server_command = get_mongo_server_command();
+    const joystick_mongod_path = `${os.homedir()}/.joystick/databases/mongodb/bin/bin/${mongo_server_command}`;
+    const database_process_flags = [
+      '--port',
+      mongodb_port,
+      '--dbpath',
+      get_platform_safe_path(`./.joystick/data/mongodb_${mongodb_port}`),
+      '--quiet',
+      '--replSet',
+      `joystick_${mongodb_port}`,
+    ];
+
+    const database_process = child_process.spawn(
+      joystick_mongod_path,
+      database_process_flags.filter((command) => !!command),
+    );
+
+    database_process.stdout.on('data', async (data) => {
+      const stdout = data?.toString();
+
+      if (stdout.includes('Waiting for connections')) {
+        const mongo_shell_command = get_mongo_shell_command();
+        const joystick_mongo_shell_path = `${os.homedir()}/.joystick/databases/mongodb/bin/bin/${mongo_shell_command}`;
+        child_process.exec(`${joystick_mongo_shell_path} --eval "rs.initiate()" --verbose --port ${mongodb_port}`, async (error, _stdout, _stderr) => {
+          if (error && !error?.message?.includes('already initialized')) {
+            console.log(error);
+          }
+
+          const process_id = await get_process_id_from_port(mongodb_port);
+          return resolve(parseInt(process_id, 10));
+        });
+      }
+    });
+
+    database_process.stderr.on('data', async (data) => {
+      const stderr = data.toString();
+      console.log(stderr);
+    });
+  });
+};
+
+const setup_data_directory = async (mongodb_port = 2610) => {
+  // NOTE: MongoDB was originally started as a standalone server. To enable additional functionality,
+  // we moved to a replica set config which necessitated multiple data directories in order to support
+  // running the same app on multiple ports (a limitation of MongoDB).
+  const legacy_data_directory_exists = await path_exists(".joystick/data/mongodb");
+  const data_directory_exists = await path_exists(`.joystick/data/mongodb_${mongodb_port}`);
+
+  if (legacy_data_directory_exists && !data_directory_exists) {
+    await rename('.joystick/data/mongodb', `.joystick/data/mongodb_${mongodb_port}`);
+  }
+
+  if (!data_directory_exists) {
+    await mkdir(`.joystick/data/mongodb_${mongodb_port}`, { recursive: true });
+  }
+
+  return data_directory_exists;
+};
+
+const start_mongodb = async (mongodb_port = 2610) => {
+  await setup_data_directory(mongodb_port);
+
+  try {
+    await kill_port_process(mongodb_port);
+    const mongo_process_id = await start_mongodb_process(mongodb_port);
+    return mongo_process_id;
+  } catch (exception) {
+    console.warn(exception);
+    process.exit(1);
+  }
+};
+
+export default start_mongodb;

package/src/lib/development_server/databases/mongodb/installers/linux.js

@@ -0,0 +1,107 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { promisify } from 'util';
+import { pipeline } from 'stream';
+import { execFile } from 'child_process';
+
+const streamPipeline = promisify(pipeline);
+const execFileAsync = promisify(execFile);
+
+const download_mongodb_linux = async (version_path = null) => {
+  const base_directory = path.join(os.homedir(), '.joystick', 'databases', 'mongodb');
+  const bin_directory = path.join(base_directory, 'bin');
+  const bin_bin_directory = path.join(bin_directory, 'bin');
+
+  if (await check_if_file_exists(base_directory)) {
+    return;
+  }
+
+  await create_directories(base_directory, bin_directory, bin_bin_directory);
+
+  const mongodb_url = 'https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2204-7.0.2.tgz';
+  const mongosh_url = 'https://downloads.mongodb.com/compass/mongosh-2.2.12-linux-x64.tgz';
+
+  const mongodb_file_name = path.basename(new URL(mongodb_url).pathname);
+  const mongodb_file_path = path.join(base_directory, mongodb_file_name);
+
+  const mongosh_file_name = path.basename(new URL(mongosh_url).pathname);
+  const mongosh_file_path = path.join(base_directory, mongosh_file_name);
+
+  process.loader.print('MongoDB not found. Downloading... (this may take a few minutes)');
+  await download_file(mongodb_url, mongodb_file_path);
+  await download_file(mongosh_url, mongosh_file_path);
+
+  process.loader.print('Installing MongoDB...');
+  
+  // Extract MongoDB
+  await execFileAsync('tar', ['-xzf', mongodb_file_path, '-C', base_directory]);
+  const mongodb_extracted_dir = (await fs.promises.readdir(base_directory)).find(dir => dir.startsWith('mongodb-'));
+  if (!mongodb_extracted_dir) {
+    throw new Error('Could not find extracted MongoDB directory');
+  }
+  const mongodb_bin_path = path.join(base_directory, mongodb_extracted_dir, 'bin');
+  
+  // Move MongoDB binaries to bin/bin
+  const mongodb_binaries = await fs.promises.readdir(mongodb_bin_path);
+  for (const binary of mongodb_binaries) {
+    await fs.promises.rename(path.join(mongodb_bin_path, binary), path.join(bin_bin_directory, binary));
+  }
+  
+  // Extract mongosh
+  const mongosh_temp_directory = path.join(base_directory, 'mongosh_temp');
+  await fs.promises.mkdir(mongosh_temp_directory, { recursive: true });
+  await execFileAsync('tar', ['-xzf', mongosh_file_path, '-C', mongosh_temp_directory]);
+  
+  // Find and copy mongosh executable
+  const mongosh_contents = await fs.promises.readdir(mongosh_temp_directory);
+  const mongosh_dir = mongosh_contents.find(item => item.startsWith('mongosh-'));
+  if (!mongosh_dir) {
+    throw new Error('Could not find mongosh directory in extracted contents');
+  }
+  
+  const mongosh_bin_path = path.join(mongosh_temp_directory, mongosh_dir, 'bin', 'mongosh');
+  const final_mongosh_path = path.join(bin_bin_directory, 'mongosh');
+  await fs.promises.copyFile(mongosh_bin_path, final_mongosh_path);
+
+  // Clean up
+  await fs.promises.unlink(mongodb_file_path);
+  await fs.promises.unlink(mongosh_file_path);
+  await fs.promises.rm(mongosh_temp_directory, { recursive: true, force: true });
+  await fs.promises.rm(path.join(base_directory, mongodb_extracted_dir), { recursive: true, force: true });
+
+  await make_file_executable(bin_bin_directory);
+  process.loader.print('MongoDB installed!');
+};
+
+const download_file = async (url, file_path) => {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to download: ${response.statusText}`);
+  }
+  await streamPipeline(response.body, fs.createWriteStream(file_path));
+};
+
+const check_if_file_exists = async (file_path) => {
+  try {
+    await fs.promises.access(file_path);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+const create_directories = async (base_directory, bin_directory, bin_bin_directory) => {
+  await fs.promises.mkdir(base_directory, { recursive: true });
+  await fs.promises.mkdir(bin_directory, { recursive: true });
+  await fs.promises.mkdir(bin_bin_directory, { recursive: true });
+};
+
+const make_file_executable = async (directory) => {
+  const files = await fs.promises.readdir(directory);
+  for (const file of files) {
+    await fs.promises.chmod(path.join(directory, file), '755');
+  }
+};
+
+export default download_mongodb_linux;

package/src/lib/development_server/databases/mongodb/installers/macos.js

@@ -0,0 +1,109 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { promisify } from 'util';
+import { pipeline } from 'stream';
+import { execFile } from 'child_process';
+
+const streamPipeline = promisify(pipeline);
+const execFileAsync = promisify(execFile);
+
+const download_mongodb_macos = async (version_path = null) => {
+  const base_directory = path.join(os.homedir(), '.joystick', 'databases', 'mongodb');
+  const bin_directory = path.join(base_directory, 'bin');
+  const bin_bin_directory = path.join(bin_directory, 'bin');
+
+  if (await check_if_file_exists(base_directory)) {
+    return;
+  }
+
+  await create_directories(base_directory, bin_directory, bin_bin_directory);
+
+  const cpu_info = await get_cpu_info();
+  const is_arm = cpu_info.includes('ARM');
+  
+  const mongodb_url = is_arm
+    ? 'https://fastdl.mongodb.org/osx/mongodb-macos-arm64-7.0.2.tgz'
+    : 'https://fastdl.mongodb.org/osx/mongodb-macos-x86_64-7.0.2.tgz';
+
+  const mongosh_url = is_arm
+    ? 'https://downloads.mongodb.com/compass/mongosh-2.2.12-darwin-arm64.zip'
+    : 'https://downloads.mongodb.com/compass/mongosh-2.2.12-darwin-x64.zip';
+
+  const mongodb_file_name = path.basename(new URL(mongodb_url).pathname);
+  const mongodb_file_path = path.join(base_directory, mongodb_file_name);
+
+  const mongosh_file_name = path.basename(new URL(mongosh_url).pathname);
+  const mongosh_file_path = path.join(base_directory, mongosh_file_name);
+
+  process.loader.print('MongoDB not found. Downloading... (this may take a few minutes)');
+  await download_file(mongodb_url, mongodb_file_path);
+  await download_file(mongosh_url, mongosh_file_path);
+
+  process.loader.print('Installing MongoDB...');
+  
+  // Extract MongoDB
+  await execFileAsync('tar', ['-xzf', mongodb_file_path, '-C', bin_directory, '--strip-components=1']);
+  
+  // Extract mongosh
+  const mongosh_temp_directory = path.join(base_directory, 'mongosh_temp');
+  await fs.promises.mkdir(mongosh_temp_directory, { recursive: true });
+  await execFileAsync('unzip', ['-q', mongosh_file_path, '-d', mongosh_temp_directory]);
+  
+  // Find and copy mongosh executable
+  const mongosh_contents = await fs.promises.readdir(mongosh_temp_directory);
+  const mongosh_dir = mongosh_contents.find(item => item.startsWith('mongosh-'));
+  if (!mongosh_dir) {
+    throw new Error('Could not find mongosh directory in extracted contents');
+  }
+  
+  const mongosh_bin_path = path.join(mongosh_temp_directory, mongosh_dir, 'bin', 'mongosh');
+  const final_mongosh_path = path.join(bin_bin_directory, 'mongosh');
+  await fs.promises.copyFile(mongosh_bin_path, final_mongosh_path);
+
+  // Clean up
+  await fs.promises.unlink(mongodb_file_path);
+  await fs.promises.unlink(mongosh_file_path);
+  await fs.promises.rm(mongosh_temp_directory, { recursive: true, force: true });
+
+  await make_file_executable(bin_directory);
+  await make_file_executable(bin_bin_directory);
+  process.loader.print('MongoDB installed!');
+};
+
+const download_file = async (url, file_path) => {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to download: ${response.statusText}`);
+  }
+  await streamPipeline(response.body, fs.createWriteStream(file_path));
+};
+
+const check_if_file_exists = async (file_path) => {
+  try {
+    await fs.promises.access(file_path);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+const get_cpu_info = async () => {
+  const { stdout } = await execFileAsync('sysctl', ['-n', 'machdep.cpu.brand_string']);
+  return stdout.trim();
+};
+
+const create_directories = async (base_directory, bin_directory, bin_bin_directory) => {
+  await fs.promises.mkdir(base_directory, { recursive: true });
+  await fs.promises.mkdir(bin_directory, { recursive: true });
+  await fs.promises.mkdir(bin_bin_directory, { recursive: true });
+};
+
+const make_file_executable = async (directory) => {
+  const files = await fs.promises.readdir(directory);
+  for (const file of files) {
+    await fs.promises.chmod(path.join(directory, file), '755');
+  }
+};
+
+export default download_mongodb_macos;

package/src/lib/development_server/databases/mongodb/installers/windows.js

@@ -0,0 +1,90 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { promisify } from 'util';
+import { pipeline } from 'stream';
+import { execFile } from 'child_process';
+
+const stream_pipeline = promisify(pipeline);
+const exec_file_async = promisify(execFile);
+
+const download_mongodb_windows = async () => {
+  const base_directory = path.join(os.homedir(), '.joystick', 'databases', 'mongodb');
+  const bin_directory = path.join(base_directory, 'bin');
+
+  if (await check_if_file_exists(base_directory)) {
+    return;
+  }
+
+  await create_directories(base_directory, bin_directory);
+
+  const download_url = 'https://fastdl.mongodb.org/windows/mongodb-windows-x86_64-7.0.2.zip';
+  const file_name = path.basename(new URL(download_url).pathname);
+  const file_path = path.join(base_directory, file_name);
+
+  if (await check_if_file_exists(file_path)) {
+    return;
+  }
+
+  process.loader.print('MongoDB not found. Downloading... (this may take a few minutes)');
+  await download_file(download_url, file_path);
+  process.loader.print('Installing MongoDB...');
+  
+  const temp_directory = path.join(base_directory, 'temp');
+  await exec_file_async('powershell', ['Expand-Archive', '-Path', file_path, '-DestinationPath', temp_directory]);
+  const extracted_folder = (await fs.promises.readdir(temp_directory))[0];
+  const extracted_path = path.join(temp_directory, extracted_folder);
+  
+  // Remove existing bin directory if it exists
+  if (await check_if_file_exists(bin_directory)) {
+    await fs.promises.rm(bin_directory, { recursive: true, force: true });
+  }
+  
+  // Copy the entire contents of the extracted folder to the bin directory
+  await fs.promises.cp(extracted_path, bin_directory, { recursive: true });
+
+  // Download and install mongosh
+  const mongosh_url = 'https://downloads.mongodb.com/compass/mongosh-2.2.12-win32-x64.zip';
+  const mongosh_file_path = path.join(base_directory, 'mongosh.zip');
+  await download_file(mongosh_url, mongosh_file_path);
+  
+  const mongosh_temp_directory = path.join(base_directory, 'mongosh_temp');
+  await exec_file_async('powershell', ['Expand-Archive', '-Path', mongosh_file_path, '-DestinationPath', mongosh_temp_directory]);
+  
+  const mongosh_extracted_folder = (await fs.promises.readdir(mongosh_temp_directory))[0];
+  const mongosh_bin_path = path.join(mongosh_temp_directory, mongosh_extracted_folder, 'bin', 'mongosh.exe');
+  
+  await fs.promises.copyFile(mongosh_bin_path, path.join(bin_directory, 'bin', 'mongosh.exe'));
+  
+  // Clean up temporary files
+  await fs.promises.unlink(file_path);
+  await fs.promises.unlink(mongosh_file_path);
+  await fs.promises.rm(temp_directory, { recursive: true, force: true });
+  await fs.promises.rm(mongosh_temp_directory, { recursive: true, force: true });
+
+  process.loader.print('MongoDB installed!');
+};
+
+const download_file = async (url, file_path) => {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to download: ${response.statusText}`);
+  }
+  await stream_pipeline(response.body, fs.createWriteStream(file_path));
+};
+
+const check_if_file_exists = async (file_path) => {
+  try {
+    await fs.promises.access(file_path);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+const create_directories = async (base_directory, bin_directory) => {
+  await fs.promises.mkdir(base_directory, { recursive: true });
+  await fs.promises.mkdir(bin_directory, { recursive: true });
+};
+
+export default download_mongodb_windows;