@joystick.js/node-canary 0.0.0-canary.34 → 0.0.0-canary.340

package/dist/app/databases/mongodb/queries/queues.js

@@ -1,40 +1,52 @@
-import dayjs from "dayjs";
+import getTargetDatabaseProvider from "../../getTargetDatabaseProvider.js";
 var queues_default = {
   addJob: function(jobToAdd = {}) {
-    const db = process.databases._queues?.collection(`queue_${this.queue.name}`);
-    return db.insertOne(jobToAdd);
+    const db = this.db?.collection(`queue_${this.queue.name}`);
+    return db.insertOne({
+      ...jobToAdd,
+      attempts: 0
+    });
   },
   countJobs: function(status = "") {
-    const db = process.databases._queues?.collection(`queue_${this.queue.name}`);
-    return db.countDocuments({ status });
+    const db = this.db?.collection(`queue_${this.queue.name}`);
+    return db.countDocuments({ status, lockedBy: this.machineId });
   },
   deleteJob: function(jobId = "") {
-    const db = process.databases._queues?.collection(`queue_${this.queue.name}`);
+    const db = this.db?.collection(`queue_${this.queue.name}`);
     return db.deleteOne({ _id: jobId });
   },
+  deleteIncompleteJobsForMachine: function() {
+    const db = this.db?.collection(`queue_${this.queue.name}`);
+    return db.deleteMany({ status: { $in: ["incomplete", "running"] }, lockedBy: this.machineId });
+  },
   getJobs: function(query = {}) {
-    const db = process.databases._queues?.collection(`queue_${this.queue.name}`);
-    return db.find(query).toArray();
+    const db = this.db?.collection(`queue_${this.queue.name}`);
+    return db.find({
+      ...query,
+      environment: process.env.NODE_ENV
+    }).toArray();
   },
   getNextJobToRun: async function() {
-    const db = process.databases._queues?.collection(`queue_${this.queue.name}`);
+    const db = this.db?.collection(`queue_${this.queue.name}`);
     const nextJob = await db.findOneAndUpdate({
       $or: [
         {
           status: "pending",
-          nextRunAt: { $lte: dayjs().format() },
+          environment: process.env.NODE_ENV,
+          nextRunAt: { $lte: new Date().toISOString() },
           lockedBy: { $exists: false }
         },
         {
           status: "pending",
-          nextRunAt: { $lte: dayjs().format() },
+          environment: process.env.NODE_ENV,
+          nextRunAt: { $lte: new Date().toISOString() },
           lockedBy: null
         }
       ]
     }, {
       $set: {
         status: "running",
-        startedAt: dayjs().format(),
+        startedAt: new Date().toISOString(),
         lockedBy: this.machineId
       }
     }, {
@@ -44,19 +56,36 @@ var queues_default = {
     });
     return nextJob?.value;
   },
-  initializeDatabase: function() {
-    const db = process.databases._queues?.collection(`queue_${this.queue.name}`);
+  initializeDatabase: async function() {
+    await this.db.createCollection(`queue_${this.queue.name}`);
+    const db = this.db?.collection(`queue_${this.queue.name}`);
+    const indexes = await db?.indexes();
     db.createIndex({ status: 1 });
     db.createIndex({ status: 1, nextRunAt: 1 });
+    db.createIndex({ status: 1, environment: 1, nextRunAt: 1, lockedBy: 1 });
     if (this.queue.options?.cleanup?.completedAfterSeconds) {
+      if (indexes?.find((index) => index?.name === "completedAt_1")) {
+        await db.dropIndex({ completedAt: 1 });
+      }
       db.createIndex({ completedAt: 1 }, { expireAfterSeconds: this?.queue?.options?.cleanup?.completedAfterSeconds });
     }
     if (this.queue.options?.cleanup?.failedAfterSeconds) {
+      if (indexes?.find((index) => index?.name === "failedAt_1")) {
+        await db.dropIndex({ failedAt: 1 });
+      }
       db.createIndex({ failedAt: 1 }, { expireAfterSeconds: this?.queue?.options?.cleanup?.failedAfterSeconds });
     }
   },
+  logAttempt: function(jobId = "") {
+    const db = this.db?.collection(`queue_${this.queue.name}`);
+    return db.updateOne({ _id: jobId }, {
+      $inc: {
+        attempts: 1
+      }
+    });
+  },
   requeueJob: function(jobId = "", nextRunAt = null) {
-    const db = process.databases._queues?.collection(`queue_${this.queue.name}`);
+    const db = this.db?.collection(`queue_${this.queue.name}`);
     return db.updateOne({ _id: jobId }, {
       $set: {
         status: "pending",
@@ -67,36 +96,31 @@ var queues_default = {
       }
     });
   },
-  setJobsForMachineIncomplete: function() {
-    const db = process.databases._queues?.collection(`queue_${this.queue.name}`);
-    return db.updateMany({ status: "running", lockedBy: this.machineId }, {
-      $set: {
-        status: "incomplete"
-      }
-    });
-  },
   setJobCompleted: function(jobId = "") {
-    const db = process.databases._queues?.collection(`queue_${this.queue.name}`);
+    const db = this.db?.collection(`queue_${this.queue.name}`);
     return db.updateOne({ _id: jobId }, {
       $set: {
         status: "completed",
-        completedAt: dayjs().format()
+        completedAt: new Date(new Date().toISOString())
       }
     });
   },
   setJobFailed: function(jobId = "", error = "") {
-    const db = process.databases._queues?.collection(`queue_${this.queue.name}`);
+    const db = this.db?.collection(`queue_${this.queue.name}`);
     return db.updateOne({ _id: jobId }, {
       $set: {
         status: "failed",
-        failedAt: dayjs().format(),
+        failedAt: new Date(new Date().toISOString()),
         error
       }
     });
   },
   setJobsForMachinePending: function() {
-    const db = process.databases._queues?.collection(`queue_${this.queue.name}`);
-    return db.updateMany({ status: { $in: ["pending", "running"] }, lockedBy: this.machineId }, {
+    const db = this.db?.collection(`queue_${this.queue.name}`);
+    return db.updateMany({
+      status: { $in: ["pending", "running"] },
+      lockedBy: this.machineId
+    }, {
       $set: {
         status: "pending"
       },

package/dist/app/databases/mongodb/queries/sessions.js

@@ -0,0 +1,26 @@
+import dayjs from "dayjs";
+import generateId from "../../../../lib/generateId";
+var sessions_default = {
+  create_session: async () => {
+    const session_id = generateId();
+    await process.databases._sessions?.collection("sessions").insertOne({
+      _id: session_id,
+      csrf: generateId(32),
+      createdAt: new Date().toISOString()
+    });
+    return session_id;
+  },
+  get_session: async (input = {}) => {
+    return process.databases._sessions?.collection("sessions").findOne({
+      _id: input?.session_id
+    });
+  },
+  delete_expired_sessions: async () => {
+    return process.databases._sessions?.collection("sessions").deleteMany({
+      createdAt: { $lte: dayjs().subtract(1, "hour").format() }
+    });
+  }
+};
+export {
+  sessions_default as default
+};

package/dist/app/databases/postgresql/createSessionsIndexes.js

@@ -0,0 +1,10 @@
+const createIndex = async (indexName = "", tableName = "", tableColumns = []) => {
+  return process.databases._sessions?.query(`CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName}(${tableColumns.join(", ")})`);
+};
+var createSessionsIndexes_default = async () => {
+  await createIndex("session_by_id", "sessions", ["session_id"]);
+  await createIndex("session_created_at", "sessions", ["created_at"]);
+};
+export {
+  createSessionsIndexes_default as default
+};

package/dist/app/databases/postgresql/createSessionsTables.js

@@ -0,0 +1,14 @@
+const createTable = async (table = "", tableColumns = []) => {
+  return process.databases._sessions?.query(`CREATE TABLE IF NOT EXISTS ${table} (${tableColumns.join(", ")})`);
+};
+var createSessionsTables_default = async () => {
+  await createTable("sessions", [
+    "id bigserial primary key",
+    "session_id text",
+    "csrf text",
+    "created_at text"
+  ]);
+};
+export {
+  createSessionsTables_default as default
+};

package/dist/app/databases/postgresql/handleCleanupQueues.js

@@ -0,0 +1,36 @@
+const get_future_timestamp = (timestamp, seconds = 0) => {
+  const date = new Date(timestamp);
+  date.setSeconds(date.getSeconds() + seconds);
+  return date.toISOString();
+};
+var handleCleanupQueues_default = async ({
+  database = {},
+  table = "",
+  seconds = 0
+}) => {
+  const jobs_with_status = await database.query(`
+		SELECT * FROM ${table} WHERE status = ANY($1) AND environment = $2
+	`, [
+    ["completed", "failed"],
+    process.env.NODE_ENV
+  ]);
+  const jobs_to_cleanup = jobs_with_status?.filter((job = {}) => {
+    if (job?.status === "completed") {
+      const completed_at_with_seconds = get_future_timestamp(job?.completed_at, seconds);
+      return completed_at_with_seconds <= new Date().toISOString();
+    }
+    if (job?.status === "failed") {
+      const failed_at_with_seconds = get_future_timestamp(job?.failed_at, seconds);
+      return failed_at_with_seconds <= new Date().toISOString();
+    }
+    return false;
+  });
+  await database.query(`
+		DELETE FROM ${table} WHERE _id = ANY($1)
+	`, [
+    jobs_to_cleanup?.map(({ _id }) => _id)
+  ]);
+};
+export {
+  handleCleanupQueues_default as default
+};

package/dist/app/databases/postgresql/index.js

@@ -1,6 +1,8 @@
 import postgresql from "pg";
 import chalk from "chalk";
 import os from "os";
+import fs from "fs";
+import generate_sql_from_object from "../generate_sql_from_object";
 const { Pool } = postgresql;
 var postgresql_default = async (settings = {}, databasePort = 2610) => {
   const connection = settings?.connection || {
@@ -13,14 +15,20 @@ var postgresql_default = async (settings = {}, databasePort = 2610) => {
   };
   try {
     const host = connection.hosts && connection.hosts[0];
-    const pool = new Pool({
+    const connection_config = {
       user: connection?.username || "",
       database: connection?.database,
       password: connection?.password || "",
       host: host?.hostname,
       port: host?.port,
       ...settings?.options || {}
-    });
+    };
+    if (settings?.options?.ssl?.ca) {
+      connection_config.ssl = {
+        ca: fs.readFileSync(settings?.options?.ssl?.ca)
+      };
+    }
+    const pool = new Pool(connection_config);
     return {
       pool,
       query: (...args) => {
@@ -39,6 +47,84 @@ Failed Values:
           console.log(args[1]);
           throw error;
         });
+      },
+      add_column: (options = {}) => {
+        const column = generate_sql_from_object.add_column(options);
+        return pool.query(column.statement).then((response) => {
+          return response?.rows || [];
+        }).catch((error) => {
+          console.log(chalk.redBright(`
+Failed SQL Statement:
+`));
+          console.log(column.statement);
+          throw error;
+        });
+      },
+      create_table: (options = {}) => {
+        const table = generate_sql_from_object.create_table(options);
+        return pool.query(table.statement).then((response) => {
+          return response?.rows || [];
+        }).catch((error) => {
+          console.log(chalk.redBright(`
+Failed SQL Statement:
+`));
+          console.log(table.statement);
+          throw error;
+        });
+      },
+      insert: (options = {}) => {
+        const insert = generate_sql_from_object.insert(options);
+        return pool.query(insert.statement, insert.values).then((response) => {
+          return response?.rows || [];
+        }).catch((error) => {
+          console.log(chalk.redBright(`
+Failed SQL Statement:
+`));
+          console.log(insert.statement);
+          console.log(`
+`);
+          console.log(chalk.redBright(`
+Failed Values:
+`));
+          console.log(insert.values);
+          throw error;
+        });
+      },
+      select: (options = {}) => {
+        const select = generate_sql_from_object.select(options);
+        return pool.query(select.statement, select.values).then((response) => {
+          return response?.rows || [];
+        }).catch((error) => {
+          console.log(chalk.redBright(`
+Failed SQL Statement:
+`));
+          console.log(select.statement);
+          console.log(`
+`);
+          console.log(chalk.redBright(`
+Failed Values:
+`));
+          console.log(select.where);
+          throw error;
+        });
+      },
+      update: (options = {}) => {
+        const update = generate_sql_from_object.update(options);
+        return pool.query(update.statement, update.values).then((response) => {
+          return response?.rows || [];
+        }).catch((error) => {
+          console.log(chalk.redBright(`
+Failed SQL Statement:
+`));
+          console.log(update.statement);
+          console.log(`
+`);
+          console.log(chalk.redBright(`
+Failed Values:
+`));
+          console.log(update.values);
+          throw error;
+        });
       }
     };
   } catch (exception) {

package/dist/app/databases/postgresql/queries/accounts.js

@@ -21,7 +21,7 @@ var accounts_default = {
   },
   createUser: async (input = {}) => {
     const userId = generateId();
-    const keys = ["user_id", ...Object.keys(input) || []]?.map((inputKey) => {
+    const keys = ["user_id", ...Object.keys(input || {}) || []]?.map((inputKey) => {
       return camelPascalToSnake(inputKey);
     })?.join(",");
     const values = [userId, ...Object.values(input) || []];
@@ -43,8 +43,11 @@ var accounts_default = {
     }
     return null;
   },
+  deleteUser: async (input = {}) => {
+    await process.databases._users?.query(`DELETE FROM users WHERE user_id = $1`, [input?.userId]);
+  },
   deleteOldSessions: async (input = {}) => {
-    await process.databases._users?.query(`DELETE FROM users_sessions WHERE user_id = $1 AND token_expires_at::date < NOW()`, [input?.userId]);
+    await process.databases._users?.query(`DELETE FROM users_sessions WHERE user_id = $1 AND token_expires_at::timestamp < NOW()`, [input?.userId]);
   },
   addSession: async (input = {}) => {
     await process.databases._users?.query(`INSERT INTO users_sessions(user_id, token, token_expires_at) VALUES ($1, $2, $3);`, [input?.userId, input?.session?.token, input?.session?.tokenExpiresAt]);

package/dist/app/databases/postgresql/queries/queues.js

@@ -1,27 +1,32 @@
-import dayjs from "dayjs";
+import cron from "node-cron";
+import handleCleanupQueues from "../handleCleanupQueues";
 var queues_default = {
   addJob: function(jobToAdd = {}) {
-    const db = process.databases._queues;
+    const db = this?.db;
     return db?.query(`
       INSERT INTO queue_${this.queue.name} (
         _id,
         status,
+        environment,
         job,
         payload,
-        next_run_at
+        next_run_at,
+        attempts
       ) VALUES (
-        $1, $2, $3, $4, $5
+        $1, $2, $3, $4, $5, $6, $7
       )
     `, [
       jobToAdd?._id,
       jobToAdd?.status,
+      jobToAdd?.environment,
       jobToAdd?.job,
       JSON.stringify(jobToAdd?.payload),
-      jobToAdd?.nextRunAt
+      jobToAdd?.nextRunAt,
+      0
     ]);
   },
   countJobs: async function(status = "") {
-    const db = process.databases._queues;
+    const db = this?.db;
     const [jobs] = await db?.query(`
       SELECT
         count(*)
@@ -29,13 +34,16 @@ var queues_default = {
         queue_${this.queue.name}
       WHERE
         status = $1
+      AND
+        locked_by = $2
     `, [
-      status
+      status,
+      this.machineId
     ]);
     return Promise.resolve(jobs.count);
   },
   deleteJob: function(jobId = "") {
-    const db = process.databases._queues;
+    const db = this?.db;
     return db?.query(`
       DELETE FROM
         queue_${this.queue.name}
@@ -45,34 +53,55 @@ var queues_default = {
       jobId
     ]);
   },
+  deleteIncompleteJobsForMachine: function() {
+    const db = this?.db;
+    return db?.query(`
+      DELETE FROM
+        queue_${this.queue.name}
+      WHERE
+        status = ANY($1)
+      AND
+        locked_by = $2
+    `, [
+      ["incomplete", "running"],
+      this.machineId
+    ]);
+  },
   getJobs: function(query = {}) {
-    const db = process.databases._queues;
+    const db = this?.db;
     return db?.query(`
       SELECT * FROM
         queue_${this.queue.name}
       ${query?.status ? `
         WHERE
           status = $1
+        AND
+          environment = $2
       ` : ""}
     `, [
-      query?.status
+      query?.status,
+      process.env.NODE_ENV
     ]);
   },
   getNextJobToRun: async function() {
-    const db = process.databases._queues;
+    const db = this?.db;
     const [nextJob] = await db?.query(`
       SELECT * FROM
         queue_${this.queue.name}
       WHERE
         status = $1
       AND
-        next_run_at::date <= NOW()
+        environment = $2
+      AND
+        next_run_at::timestamp <= $3
       AND
         locked_by IS NULL
       ORDER BY
         next_run_at ASC
     `, [
-      "pending"
+      "pending",
+      process.env.NODE_ENV,
+      new Date().toISOString()
     ]);
     if (nextJob?._id) {
       await db?.query(`
@@ -86,7 +115,7 @@ var queues_default = {
           _id = $4
       `, [
         "running",
-        dayjs().format(),
+        new Date().toISOString(),
         this.machineId,
         nextJob?._id
       ]);
@@ -97,11 +126,12 @@ var queues_default = {
     } : {};
   },
   initializeDatabase: async function() {
-    const db = process.databases._queues;
+    const db = this?.db;
     await db?.query(`
       CREATE TABLE IF NOT EXISTS queue_${this.queue.name} (
         _id text PRIMARY KEY,
         status text,
+        environment text,
         job text,
         payload text,
         next_run_at text,
@@ -109,51 +139,69 @@ var queues_default = {
         started_at text,
         completed_at text,
         failed_at text,
-        error text
+        error text,
+        attempts smallint
       )
     `);
+    db?.query(`ALTER TABLE queue_${this.queue.name} ADD COLUMN IF NOT EXISTS attempts smallint`);
+    db?.query(`ALTER TABLE queue_${this.queue.name} ADD COLUMN IF NOT EXISTS environment text`);
     db?.query(`CREATE INDEX IF NOT EXISTS status_index ON queue_${this.queue.name} (status)`);
     db?.query(`CREATE INDEX IF NOT EXISTS status_nextRunAt_index ON queue_${this.queue.name} (status, next_run_at)`);
+    db?.query(`CREATE INDEX IF NOT EXISTS nextJob_index ON queue_${this.queue.name} (status, environment, next_run_at, locked_by)`);
     db?.query(`CREATE INDEX IF NOT EXISTS completedAt_index ON queue_${this.queue.name} (completed_at)`);
     db?.query(`CREATE INDEX IF NOT EXISTS failedAt_index ON queue_${this.queue.name} (failed_at)`);
+    if (this.queue.options?.cleanup?.completedAfterSeconds) {
+      cron.schedule("*/30 * * * * *", () => {
+        handleCleanupQueues({
+          database: db,
+          table: `queue_${this.queue.name}`,
+          seconds: this.queue.options?.cleanup?.completedAfterSeconds
+        });
+      });
+    }
+    if (this.queue.options?.cleanup?.failedAfterSeconds) {
+      cron.schedule("*/30 * * * * *", () => {
+        handleCleanupQueues({
+          database: db,
+          table: `queue_${this.queue.name}`,
+          seconds: this.queue.options?.cleanup?.failedAfterSeconds
+        });
+      });
+    }
   },
-  requeueJob: function(jobId = "", nextRunAt = null) {
-    const db = process.databases._queues;
+  logAttempt: function(jobId = "") {
+    const db = this?.db;
     return db?.query(`
       UPDATE
         queue_${this.queue.name}
       SET
-        status = $1,
-        next_run_at = $2,
-        locked_by = $3
+        attempts = attempts + 1
       WHERE
-        _id = $4
+        _id = $1
     `, [
-      "pending",
-      nextRunAt,
-      null,
       jobId
     ]);
   },
-  setJobsForMachineIncomplete: function() {
-    const db = process.databases._queues;
+  requeueJob: function(jobId = "", nextRunAt = null) {
+    const db = this?.db;
     return db?.query(`
       UPDATE
         queue_${this.queue.name}
       SET
-        status = $1
-      WHERE
-        status = $2
-      AND
+        status = $1,
+        next_run_at = $2,
         locked_by = $3
+      WHERE
+        _id = $4
     `, [
-      "incomplete",
-      "running",
-      this.machineId
+      "pending",
+      nextRunAt,
+      null,
+      jobId
     ]);
   },
   setJobCompleted: function(jobId = "") {
-    const db = process.databases._queues;
+    const db = this?.db;
     return db?.query(`
       UPDATE
         queue_${this.queue.name}
@@ -164,12 +212,12 @@ var queues_default = {
         _id = $3
     `, [
       "completed",
-      dayjs().format(),
+      new Date().toISOString(),
       jobId
     ]);
   },
   setJobFailed: function(jobId = "", error = "") {
-    const db = process.databases._queues;
+    const db = this?.db;
     return db?.query(`
       UPDATE
         queue_${this.queue.name}
@@ -181,13 +229,13 @@ var queues_default = {
         _id = $4
     `, [
       "failed",
-      dayjs().format(),
+      new Date().toISOString(),
       error,
       jobId
     ]);
   },
   setJobsForMachinePending: function() {
-    const db = process.databases._queues;
+    const db = this?.db;
     return db?.query(`
       UPDATE
         queue_${this.queue.name}

package/dist/app/databases/postgresql/queries/sessions.js

@@ -0,0 +1,43 @@
+import generateId from "../../../../lib/generateId";
+var sessions_default = {
+  create_session: async (input = {}) => {
+    const session_id = generateId();
+    await process.databases._sessions?.query(`
+			INSERT INTO
+				sessions (
+					session_id,
+					csrf,
+					created_at
+				)
+			VALUES ($1, $2, $3)
+		`, [
+      session_id,
+      generateId(32),
+      new Date().toISOString()
+    ]);
+    return session_id;
+  },
+  get_session: async (input = {}) => {
+    const [session] = await process.databases._sessions?.query(`
+			SELECT * FROM
+				sessions
+			WHERE
+				session_id = $1
+		`, [
+      input?.session_id
+    ]);
+    return session;
+  },
+  delete_expired_sessions: async () => {
+    const date = new Date();
+    date.setHours(date.getHours() - 1);
+    return process.databases.postgresql.query(`
+			DELETE FROM sessions WHERE created_at::timestamp < $1
+		`, [
+      date.toISOString()
+    ]);
+  }
+};
+export {
+  sessions_default as default
+};