@joystick.js/node-canary 0.0.0-canary.47 → 0.0.0-canary.471

package/src/app/databases/queries/postgresql/queues.js

@@ -0,0 +1,294 @@
+import cron from 'node-cron';
+import handle_cleanup_queues from '../../postgresql/handle_cleanup_queues.js';
+import timestamps from "../../../../lib/timestamps.js";
+import wait from '../../../../lib/wait.js';
+
+const queues = {
+  add_job: async function (job_to_add = {}) {
+    const db = this?.db;
+
+    // NOTE: Ensure that table was created via initialize_database on first startup. If not, wait 1s
+    // while PostgreSQL creates the table.
+    const [existing_table] = await db?.query(`SELECT * FROM information_schema.tables WHERE table_name = $1`, [
+      `queue_${this.queue.name}`
+    ]);
+
+    if (!existing_table) {
+      await wait(1);
+    }
+
+    return db?.query(`
+      INSERT INTO queue_${this.queue.name} (
+        _id,
+        status,
+        environment,
+        job,
+        payload,
+        next_run_at,
+        attempts
+      ) VALUES (
+        $1, $2, $3, $4, $5, $6, $7
+      )
+    `, [
+      job_to_add?._id,
+      job_to_add?.status,
+      job_to_add?.environment,
+      job_to_add?.job,
+      JSON.stringify(job_to_add?.payload),
+      job_to_add?.nextRunAt || job_to_add?.next_run_at,
+      0
+    ]);
+  },
+  count_jobs: async function (status = '') {
+    const db = this?.db;
+
+    const [jobs] = await db?.query(`
+      SELECT
+        count(*)
+      FROM
+        queue_${this.queue.name}
+      WHERE
+        status = $1
+      AND
+        locked_by = $2
+    `, [
+      status,
+      this.machine_id
+    ]);
+
+    return Promise.resolve(jobs.count);
+  },
+  delete_job: function (job_id = '') {
+    const db = this?.db;
+
+    return db?.query(`
+      DELETE FROM
+        queue_${this.queue.name}
+      WHERE
+        _id = $1
+    `, [
+      job_id
+    ]);
+  },
+  delete_incomplete_jobs_for_machine: function () {
+    const db = this?.db;
+    return db?.query(`
+      DELETE FROM
+        queue_${this.queue.name}
+      WHERE
+        status = ANY($1)
+      AND
+        locked_by = $2
+    `, [
+      ['incomplete', 'running'],
+      this.machine_id,
+    ]);
+  },
+  get_jobs: function (query = {}) {
+    const db = this?.db;
+
+    return db?.query(`
+      SELECT * FROM
+        queue_${this.queue.name}
+      ${query?.status ? `
+        WHERE
+          status = $1
+        AND
+          environment = $2
+      ` : ''}
+    `, [
+      query?.status,
+      process.env.NODE_ENV,
+    ]);
+  },
+  get_next_job_to_run: async function () {
+    const db = this?.db;
+
+    const [next_job] = await db?.query(`
+      SELECT * FROM
+        queue_${this.queue.name}
+      WHERE
+        status = $1
+      AND
+        environment = $2
+      AND
+        next_run_at::timestamp <= $3
+      AND
+        locked_by IS NULL
+      ORDER BY
+        next_run_at ASC
+    `, [
+      'pending',
+      process.env.NODE_ENV,
+      timestamps.get_current_time()
+    ]);
+
+    if (next_job?._id) {
+      await db?.query(`
+        UPDATE
+          queue_${this.queue.name}
+        SET
+          status = $1,
+          started_at = $2,
+          locked_by = $3
+        WHERE
+          _id = $4
+      `, [
+        'running',
+        timestamps.get_current_time(),
+        this.machine_id,
+        next_job?._id,
+      ]);
+    }
+
+    return next_job ? {
+      ...next_job,
+      payload: next_job?.payload ? JSON.parse(next_job?.payload || '') : {},
+    } : {};
+  },
+  initialize_database: async function () {
+    // NOTE: Add this check to avoid clustered apps from creating a race condition
+    // when initializing indexes below (they step on each other's toes and cause
+    // errors to be thrown). Only a primary or 1st worker should run this.
+
+    if (cluster.isPrimary || (cluster.isWorker && cluster.worker.id === 1)) {
+      const db = this?.db;
+
+      await db?.query(`
+        CREATE TABLE IF NOT EXISTS queue_${this.queue.name} (
+          _id text PRIMARY KEY,
+          status text,
+          job text,
+          payload text,
+          next_run_at text,
+          locked_by text,
+          started_at text,
+          completed_at text,
+          failed_at text,
+          error text,
+          environment text,
+          attempts smallint
+        )
+      `);
+
+      // NOTE: Add additional attempts field as a standalone column to support existing queue_ tables.
+      await db?.query(`ALTER TABLE queue_${this.queue.name} ADD COLUMN IF NOT EXISTS environment text`);
+      await db?.query(`ALTER TABLE queue_${this.queue.name} ADD COLUMN IF NOT EXISTS attempts smallint`);
+
+      await db?.query(`CREATE INDEX IF NOT EXISTS status_index ON queue_${this.queue.name} (status)`);
+      await db?.query(`CREATE INDEX IF NOT EXISTS status_next_run_at_index ON queue_${this.queue.name} (status, next_run_at)`);
+      await db?.query(`CREATE INDEX IF NOT EXISTS next_job_index ON queue_${this.queue.name} (status, environment, next_run_at, locked_by)`);
+
+      await db?.query(`CREATE INDEX IF NOT EXISTS completed_at_index ON queue_${this.queue.name} (completed_at)`);
+      await db?.query(`CREATE INDEX IF NOT EXISTS failed_at_index ON queue_${this.queue.name} (failed_at)`);
+
+      // NOTE: PostgreSQL does NOT have a TTL index or event-based row expiration feature,
+      // so we "polyfill" here with 30 second cron jobs to do the cleanup for us.
+      if (this.queue.options?.cleanup?.completedAfterSeconds || this.queue.options?.cleanup?.completed_after_seconds) {
+        cron.schedule('*/30 * * * * *', () => {
+          handle_cleanup_queues({
+            database: db,
+            table: `queue_${this.queue.name}`,
+            seconds: this.queue.options?.cleanup?.completedAfterSeconds || this.queue.options?.cleanup?.completed_after_seconds,
+          });
+        });
+      }
+
+      if (this.queue.options?.cleanup?.failedAfterSeconds || this.queue.options?.cleanup?.failed_after_seconds) {
+        cron.schedule('*/30 * * * * *', () => {
+          handle_cleanup_queues({
+            database: db,
+            table: `queue_${this.queue.name}`,
+            seconds: this.queue.options?.cleanup?.failedAfterSeconds || this.queue.options?.cleanup?.failed_after_seconds,
+          });
+        });
+      }
+    }
+  },
+  log_attempt: function (job_id = '') {
+    const db = this?.db;
+    return db?.query(`
+      UPDATE
+        queue_${this.queue.name}
+      SET
+        attempts = attempts + 1
+      WHERE
+        _id = $1
+    `, [
+      job_id,
+    ]);
+  },
+  requeue_job: function (job_id = '', next_run_at = null) {
+    const db = this?.db;
+    return db?.query(`
+      UPDATE
+        queue_${this.queue.name}
+      SET
+        status = $1,
+        next_run_at = $2,
+        locked_by = $3
+      WHERE
+        _id = $4
+    `, [
+      'pending',
+      next_run_at,
+      null,
+      job_id,
+    ]);
+  },
+  set_job_completed: function (job_id = '') {
+    const db = this?.db;
+    return db?.query(`
+      UPDATE
+        queue_${this.queue.name}
+      SET
+        status = $1,
+        completed_at = $2
+      WHERE
+        _id = $3
+    `, [
+      'completed',
+      timestamps.get_current_time(),
+      job_id
+    ]);
+  },
+  set_job_failed: function (job_id = '', error = '') {
+    const db = this?.db;
+    return db?.query(`
+      UPDATE
+        queue_${this.queue.name}
+      SET
+        status = $1,
+        failed_at = $2,
+        error = $3
+      WHERE
+        _id = $4
+    `, [
+      'failed',
+      timestamps.get_current_time(),
+      error,
+      job_id,
+    ]);
+  },
+  set_jobs_for_machine_pending: function () {
+    const db = this?.db;
+    return db?.query(`
+      UPDATE
+        queue_${this.queue.name}
+      SET
+        status = $1,
+        locked_by = $2
+      WHERE
+        status = ANY($3)
+      AND
+        locked_by = $4
+    `, [
+      'pending',
+      null,
+      ['pending', 'running'],
+      this.machine_id,
+    ]);
+  },
+};
+
+export default queues;

package/src/app/databases/queries/postgresql/sessions.js

@@ -0,0 +1,37 @@
+import generate_id from "../../../../lib/generate_id.js";
+
+const sessions = {
+	create_session: async (input = {}) => {
+		const session_id = generate_id();
+
+		await process.databases._sessions?.query(`
+			INSERT INTO
+				sessions (
+					session_id,
+					csrf,
+					created_at
+				)
+			VALUES ($1, $2, $3)
+		`, [
+			session_id,
+			generate_id(32),
+			new Date().toISOString(),
+		]);
+
+		return session_id;
+	},
+	get_session: async (input = {}) => {
+		const [session] = await process.databases._sessions?.query(`
+			SELECT * FROM
+				sessions
+			WHERE
+				session_id = $1
+		`, [
+			input?.session_id,
+		]);
+
+		return session;
+	},
+};
+
+export default sessions;

package/src/app/databases/queries/sessions.js

@@ -0,0 +1,17 @@
+import get_target_database_connection from "../get_target_database_connection.js";
+import query_map from "./map.js";
+
+const sessions_query = async (query_name = "", inputs = {}) => {
+  const sessions_database = get_target_database_connection('sessions');
+  const query_map_for_database = sessions_database && query_map && query_map[sessions_database?.provider] && query_map[sessions_database?.provider]?.sessions;
+  const query = query_map_for_database && query_map_for_database[query_name];
+
+  if (sessions_database?.connection && query) {
+    const response = await query_map_for_database[query_name](inputs, sessions_database?.connection);
+    return Promise.resolve(response);
+  }
+
+  return null;
+};
+
+export default sessions_query;

package/src/app/databases/register_database.js

@@ -0,0 +1,30 @@
+import connect_mongodb from "./mongodb/connect.js";
+import connect_postgresql from "./postgresql/connect.js";
+
+const register_database = async (database_from_settings = {}, database_port = 2610, has_multiple_of_provider = false) => {
+	if (database_from_settings?.provider === 'mongodb') {
+		const mongodb_connection = await connect_mongodb(database_from_settings, database_port);
+    process.databases = {
+      ...(process.databases || {}),
+      mongodb: !has_multiple_of_provider ? mongodb_connection : {
+        ...(process?.databases?.mongodb || {}),
+        [database_from_settings?.name || `mongodb_${database_port}`]: mongodb_connection,
+      },
+    };
+	}
+
+	if (database_from_settings?.provider === 'postgresql') {
+		const postgresql_connection = await connect_postgresql(database_from_settings, database_port);
+		process.databases = {
+      ...(process.databases || {}),
+      postgresql: !has_multiple_of_provider ? postgresql_connection : {
+        ...(process?.databases?.postgresql || {}),
+        [database_from_settings?.name || `postgresql_${database_port}`]: postgresql_connection,
+      },
+    };
+	}
+
+  return Promise.resolve(process.databases);
+};
+
+export default register_database;

package/src/app/databases/sql.js

@@ -0,0 +1,63 @@
+const sql = {
+  create_table: (options = {}) => {
+    const columns = Object.entries(options?.columns)?.map(([column_name, column_type]) => {
+      return `${column_name} ${column_type}`;
+    })?.join(',');
+
+    return {
+      statement: `CREATE TABLE IF NOT EXISTS ${options?.table} (${columns})`,
+      columns,
+    };
+  },
+  add_column: (options = {}) => {
+    return {
+      statement: `ALTER TABLE ${options?.table} ADD COLUMN IF NOT EXISTS ${options?.column_name} ${options?.column_type}`,
+    };
+  },
+  select: (options = {}) => {
+    const column_names = Array.isArray(options?.columns) ? options?.columns?.join(',') : '*';
+    const whereEntries = Object.entries(options?.where);
+    const where = whereEntries?.map(([key], index) => {
+        return `${key} = $${index + 1}`;
+    })?.join(',');
+
+    return {
+      statement: `SELECT ${column_names} FROM ${options?.table} ${options?.where ? `WHERE ${where}` : ''}`,
+      column_names,
+      where,
+      values: Object.values(options?.where),
+    };
+  },
+  insert: (options = {}) => {
+    const column_names = Object.keys(options?.data)?.join(',');
+    const value_placeholders = Object.keys(options?.data)?.map((_, index) => `$${index + 1}`)?.join(',');
+    
+    return {
+      statement: `INSERT INTO ${options?.table} (${column_names}) VALUES (${value_placeholders})`,
+      column_names,
+      value_placeholders,
+      values: Object.values(options?.data),
+    };
+  },
+  update: (options = {}) => {
+    const whereEntries = Object.entries(options?.where);
+    const sets = Object.keys(options?.data).map((key, index) => {
+      return `${key} = $${whereEntries.length + index + 1}`;
+    })?.join(',');
+    const where = whereEntries?.map(([key], index) => {
+        return `${key} = $${index + 1}`;
+    })?.join(',');
+  
+    return {
+      statement: `UPDATE ${options?.table} SET ${sets} WHERE ${where}`,
+      sets,
+      where,
+      values: [
+        ...(Object.values(options?.where)),
+        ...(Object.values(options?.data))
+      ],
+    };
+  },
+};
+
+export default sql;

package/src/app/email/send.js

@@ -0,0 +1,108 @@
+import chalk from "chalk";
+import { htmlToText } from "html-to-text";
+import juice from "juice";
+import nodemailer from "nodemailer";
+import dynamic_import from "../../lib/dynamic_import.js";
+import get_joystick_build_path from "../../lib/get_joystick_build_path.js";
+import get_translations from "../../lib/get_translations.js";
+import load_settings from "../settings/load.js";
+import path_exists from "../../lib/path_exists.js";
+import ssr from "../ssr/index.js";
+import track_function_call from "../../test/track_function_call.js";
+import validate_smtp_settings from "./validate_smtp_settings.js";
+
+const settings = load_settings();
+
+const send_email = async (send_email_options = {}, smtp_overrides = {}) => {
+  if (process.env.NODE_ENV === 'test') {
+    // NOTE: In a test environment, we do not actually want to send an email. Instead,
+    // track the call and then return.
+    track_function_call('node.email.send', [send_email_options]);
+    return;
+  }
+
+  const valid_smtp_settings = validate_smtp_settings(settings?.config?.email?.smtp);
+
+  if (!valid_smtp_settings) {
+    console.warn(chalk.redBright("Invalid SMTP settings. Cannot send email."));
+    return Promise.resolve(null);
+  }
+
+  // NOTE: Check the port number as nodemailer notes most SMTP providers required a plaintext
+  // connection *first* before upgrading the connection via STARTTLS. This prevents an SSL error
+  // when sending emails via SMTP port 587 or 25. See: https://nodemailer.com/smtp/#tls-options.
+  const smtp_port = parseInt(smtp_overrides?.port || settings?.config?.email?.smtp?.port, 10);
+  const should_use_secure = smtp_port === 465 || ![25, 587].includes(smtp_port);
+
+  const smtp = valid_smtp_settings
+    ? nodemailer.createTransport({
+        host: smtp_overrides?.host || settings?.config?.email?.smtp?.host,
+        port: smtp_port,
+        secure: should_use_secure,
+        auth: {
+          user: smtp_overrides?.username || settings?.config?.email?.smtp?.username,
+          pass: smtp_overrides?.password || settings?.config?.email?.smtp?.password,
+        },
+      })
+    : null;
+
+  const joystick_build_path = get_joystick_build_path();
+  const template_path = `${joystick_build_path}email/${send_email_options?.template}.js`;
+  const template_exists = send_email_options?.template && await path_exists(template_path);
+
+  const nodemailer_options = {
+    from: settings?.config?.email?.from,
+    ...send_email_options,
+  };
+
+  if (smtp_overrides?.headers) {
+    nodemailer_options.headers = smtp_overrides?.headers;
+  }
+
+  if (template_exists) {
+    const email_template_component = await dynamic_import(`${template_path}?v=${new Date().getTime()}`);
+
+    const translations = await get_translations({
+    	is_email: true,
+    	email_template_name: send_email_options?.template,
+    	joystick_build_path,
+    	req: {
+    		context: {
+    			user: send_email_options?.user,
+    		},
+    	},
+    });
+
+    const html = await ssr({
+    	is_email: true,
+      component_options: {
+        props: send_email_options?.props,
+        translations,
+      },
+      component_to_render: email_template_component,
+      email_options: {
+        base_html_name: send_email_options?.base,
+      	subject: send_email_options?.subject,
+      	preheader: send_email_options?.preheader,
+      },
+    });
+
+    const text = htmlToText(html);
+    const html_with_styles_inlined = juice(html, {
+      preserveMediaQueries: true,
+      preserveImportant: true,
+      removeStyleTags: false,
+    });
+
+    nodemailer_options.html = html_with_styles_inlined;
+    nodemailer_options.text = text;
+    
+    return smtp.sendMail(nodemailer_options);
+  }
+
+  console.warn(`Email template at /email/${send_email_options?.template}.js could not be found.`);
+
+  return Promise.resolve();
+};
+
+export default send_email;

package/src/app/email/validate_smtp_settings.js

@@ -0,0 +1,53 @@
+import chalk from "chalk";
+
+const validate_smtp_settings = (settings = null) => {
+  if (!settings) {
+    console.warn(
+      chalk.redBright(
+        `Invalid SMTP settings: config.smtp not defined in settings.${process.env.NODE_ENV}.js`
+      )
+    );
+    return false;
+  }
+
+  if (settings && !settings.host) {
+    console.warn(
+      chalk.redBright(
+        `Invalid SMTP settings: config.smtp.host not defined in settings.${process.env.NODE_ENV}.js`
+      )
+    );
+    return false;
+  }
+
+  if (settings && !settings.port) {
+    console.warn(
+      chalk.redBright(
+        `Invalid SMTP settings: config.smtp.port not defined in settings.${process.env.NODE_ENV}.js`
+      )
+    );
+    return false;
+  }
+
+  if (settings && !settings.username) {
+    console.warn(
+      chalk.redBright(
+        `Invalid SMTP settings: config.smtp.username not defined in settings.${process.env.NODE_ENV}.js`
+      )
+    );
+    return false;
+  }
+
+  if (settings && !settings.password) {
+    console.warn(
+      chalk.redBright(
+        `Invalid SMTP settings: config.smtp.password not defined in settings.${process.env.NODE_ENV}.js`
+      )
+    );
+    return false;
+  }
+
+  return true;
+};
+
+export default validate_smtp_settings;
+

package/src/app/fixture/index.js

@@ -0,0 +1,50 @@
+import types from "../../lib/types.js";
+
+class Fixture {
+  constructor(options = {}) {
+    this.options = options;
+    this.quantity = options?.quantity;
+    this.run = this.run.bind(this);
+
+    return this.run;
+  }
+
+  async run(input = {}) {
+    this.input = input;
+
+    const skip = typeof this?.options?.skip === 'function' ? await this.options.skip(this, input) : false;
+    
+    let data_to_create = [];
+
+    if (!skip) {
+      data_to_create = await this.generate_data_to_create(input);
+
+      if (types.is_function(this?.options?.onCreate) || types.is_function(this?.options?.on_create)) {
+        await (this.options.onCreate || this.options.on_create)(this, data_to_create, (on_after_create_each_input = {}) => {
+          return (this?.options?.onAfterCreateEach || this.options.on_after_create_each)(this, on_after_create_each_input, input);
+        });
+      }
+    }
+
+    if (types.is_function(this?.options?.onAfterCreateAll) || types.is_function(this?.options?.on_after_create_all)) {
+      (this.options.onAfterCreateAll || this.options.on_after_create_all)(this, data_to_create, input);
+    }
+  }
+
+  async generate_data_to_create(input = {}) {
+    const data = [];
+
+    for (let i = 0; i < this?.quantity; i += 1) {
+      if (typeof this?.options?.template === 'function') {
+        const data_to_create = await this.options?.template(this, i, input);
+        data.push(data_to_create);
+      }
+    }
+
+    return data;
+  }
+}
+
+export default (options = {}) => {
+  return new Fixture(options);
+}

package/src/app/generate_machine_id.js

@@ -0,0 +1,26 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import generate_id from '../lib/generate_id.js';
+import path_exists from '../lib/path_exists.js';
+
+const { mkdir, writeFile, readFile } = fs.promises;
+
+const generate_machine_id = async () => {
+  const home = os.homedir();
+  const joystick_machine_id_path = `${home}/.cheatcode/MACHINE_ID`;
+  const has_joystick_machine_id = await path_exists(joystick_machine_id_path);
+
+  if (!has_joystick_machine_id) {
+    const joystick_machine_id = generate_id(32);
+    await mkdir(path.dirname(joystick_machine_id_path), { recursive: true });
+    await writeFile(joystick_machine_id_path, joystick_machine_id);
+    return joystick_machine_id;
+  }
+
+  const joystick_machine_id = await readFile(joystick_machine_id_path, 'utf-8');
+
+  return joystick_machine_id?.trim();
+;}
+
+export default generate_machine_id;