neosqlite 1.0.0

package/src/client/index.ts

@@ -0,0 +1,70 @@
+import Database from "better-sqlite3";
+
+import type { ExecuteQueryParams, NeosqliteClient, NeosqliteConfig } from "../types";
+
+import { parseQuery, runQuery } from "./util";
+
+export function createClient(config: NeosqliteConfig): NeosqliteClient {
+  const { file } = config;
+  const db = new Database(file);
+
+  const transaction = (fn: Function) => {
+    db.transaction(() => {
+      fn();
+    });
+  };
+
+  /**
+   * Read data from the database, returns all matching rows
+   * @returns An array of objects
+   */
+  const read = (params: ExecuteQueryParams) => {
+    const query = parseQuery(params);
+    const statement = db.prepare(query.sql);
+
+    return runQuery(config, params, () => statement.all(query.args));
+  };
+
+  /**
+   * Read data from the database, returns the first matching row
+   * @returns An object
+   */
+  const readOne = (params: ExecuteQueryParams) => {
+    const query = parseQuery(params);
+    const statement = db.prepare(query.sql);
+
+    return runQuery(config, params, () => statement.get(query.args));
+  };
+
+  /**
+   * Write data to the database, returns the number of
+   * rows affected
+   */
+  const write = (params: ExecuteQueryParams) => {
+    const query = parseQuery(params);
+    const statement = db.prepare(query.sql);
+
+    return runQuery(config, params, () => statement.run(query.args));
+  };
+
+  /**
+   * Write data to the database, returns all matching rows
+   * via RETURNING
+   */
+  const writeReturning = (params: ExecuteQueryParams) => {
+    const query = parseQuery(params);
+    const statement = db.prepare(query.sql);
+
+    return runQuery(config, params, () => statement.all(query.args));
+  };
+
+  return {
+    read,
+    readOne,
+    write,
+    transaction,
+    writeReturning,
+    pragma: db.pragma,
+    backup: db.backup,
+  };
+}

package/src/client/util.ts

@@ -0,0 +1,231 @@
+import { ExecuteQueryParams, NeosqliteConfig } from "../types";
+
+const NEWLINE_CHAR = "\t";
+
+export const parseQuery = (params: ExecuteQueryParams) => {
+  if (typeof params === "string") {
+    return { sql: params, args: {} };
+  }
+
+  return params;
+};
+
+export const runQuery = (config: NeosqliteConfig, params: ExecuteQueryParams, fn: Function) => {
+  const startTime = process.hrtime();
+  const result = fn();
+  const endTime = process.hrtime(startTime);
+  const endTimeMs = endTime[0] * 1_000 + endTime[1] / 1_000_000;
+
+  if (config.onQueryComplete || config.onQueryLog) {
+    const queryString = getFullQueryString(params);
+
+    config.onQueryComplete?.({ query: queryString, time: endTimeMs });
+
+    if (typeof params === "object" && params.logQuery) {
+      config.onQueryLog?.(queryString);
+    }
+  }
+
+  return result;
+};
+
+/**
+ * Join a list of strings into a single string for
+ * SQL queries
+ *
+ * @example
+ * queryString(
+ *   "SELECT * FROM users",
+ *   "WHERE id = :id"
+ * )
+ * // Returns
+ * "SELECT * FROM users WHERE id = :id"
+ */
+export const queryString = (...args: (string | boolean | undefined)[]) => {
+  const filteredArgs = args.filter(Boolean);
+  return filteredArgs.join(NEWLINE_CHAR);
+};
+
+/**
+ * Take a list and return an object with its sql placeholder names, and
+ * its prepared arguments
+ *
+ * @example
+ * parameterize("id", [1, 2, 3])
+ * // Returns
+ * [
+ *   // Placeholders
+ *   ':id0, :id1, :id2',
+ *   // Args
+ *   { id0: 1, id1: 2, id2: 3 }
+ * ]
+ */
+export const parameterizePrimitiveArray = (key: string, value: Array<string | number>) => {
+  const args = Object.fromEntries(value.map((v, i) => [`${key}${i}`, v]));
+  const placeholders = value.map((_, i) => `:${key}${i}`).join(", ");
+
+  return [placeholders, args] as const;
+};
+
+/**
+ * Take an array of objects and extract specific fields as needed to be used
+ * as placeholders, and prepared arguments
+ *
+ * @example
+ * parameterizeComplexArray([{ id: 1, name: "foo" }, { id: 2, name: "bar" }], ["id", "name"])
+ * // Returns
+ * [
+ *   // Placeholders
+ *   '(:id0, :name0), (:id1, :name1)',
+ *   // Args
+ *   { id0: 1, name0: "foo", id1: 2, name1: "bar" }
+ * ]
+ */
+export const parameterizeComplexArray = <T>(value: T[], fields: (keyof T)[]) => {
+  const placeholders = value.map((_, i) => `(${fields.map((field) => `:${String(field)}${i}`).join(",")})`).join(", ");
+  const args = value.reduce((acc, item, i) => {
+    return { ...acc, ...Object.fromEntries(fields.map((field) => [`${String(field)}${i}`, item[field] ?? null])) };
+  }, {});
+
+  return [placeholders, args] as const;
+};
+
+/**
+ * Sanitizes a string to be used in a SQL query
+ * @remarks
+ * You should still prefer using parameterized queries over string concatenation, but this
+ * is sometimes necessary
+ */
+export const sanitize = (value: string | number | boolean | null | undefined) => {
+  if (value === null || value === undefined) return "NULL";
+  const sanitizedString = String(value)
+    .replace(/\\/g, "\\\\") // backslashes
+    .replace(/\u0008/g, "\\b") // backspace
+    .replace(/\t/g, "\\t") // tab
+    .replace(/\n/g, "\\n") // newline
+    .replace(/\f/g, "\\f") // form feed
+    .replace(/\r/g, "\\r") // carriage return
+    .replace(/'/g, "''"); // single quotes
+  return `'${sanitizedString}'`;
+};
+
+/**
+ * Parse a column as JSON, where it is either parsed as an object, or as
+ * null
+ *
+ * @returns A parsed object, or null
+ */
+export const Jsonify = <T extends Record<string, any> | Array<any> = Record<string, any>>(value: string) => {
+  try {
+    return JSON.parse(value) as T;
+  } catch (err) {
+    return null;
+  }
+};
+
+/**
+ * Logs the full query string that will be executed.
+ *
+ * @remarks
+ * ⚠️ DO NOT EXECUTE THIS STRING. By default, this is for logging only, to see
+ * what the query would look like with its parameters in place. This does NOT
+ * sanitize the parameters. In fact, it unsanitizes them.
+ */
+export const getFullQueryString = (params: ExecuteQueryParams) => {
+  if (typeof params === "string") {
+    return params;
+  }
+
+  let queryString = params.sql;
+
+  const args = Object.entries(params.args ?? {}).sort(([aKey], [bKey]) => {
+    return bKey.length - aKey.length;
+  });
+
+  for (const [key, value] of args) {
+    const regex = new RegExp(`:${key}\\b`, "g");
+    queryString = queryString.replace(regex, `'${value}'`);
+  }
+
+  let fullString = "";
+  const lines = queryString.split(NEWLINE_CHAR);
+
+  for (const line of lines) {
+    if (!line.trim()) continue;
+    fullString += `${line}\n`;
+  }
+
+  return fullString;
+};
+
+/**
+ * Sanitize a string to remove characters that would break glob patterns
+ * from the string
+ *
+ * @example
+ * sanitizeLike('foo%bar')
+ * // Returns
+ * 'foo\\%bar'
+ */
+export const sanitizeLike = (str: string) => {
+  return str
+    .replace(/\\/g, "\\\\") // escape backslashes
+    .replace(/%/g, "\\%") // escape %
+    .replace(/_/g, "\\_"); // escape _
+};
+
+/**
+ * Fully sanitize a string down to only alphanumeric characters. Useful for JSON key paths
+ * or other string values that cannot be parameterized, but need to be sanitized
+ *
+ * @example
+ * sanitizeSqlPath('foo';--bar')
+ * // Returns
+ * 'foobar'
+ */
+export const sanitizeSqlPath = (str: string) => {
+  return str.replace(/[^a-zA-Z0-9_]/g, "");
+};
+
+/**
+ * Converts a date to the standard sqlite date format `YYYY-MM-DD HH:MM:SS`
+ * Also converts timezone to UTC
+ */
+export const toSqliteDateString = (date: Date | string) => {
+  if (typeof date === "string") {
+    date = new Date(date);
+  }
+
+  const yyyy = date.getUTCFullYear();
+  const mm = String(date.getUTCMonth() + 1).padStart(2, "0");
+  const dd = String(date.getUTCDate()).padStart(2, "0");
+  const hh = String(date.getUTCHours()).padStart(2, "0");
+  const min = String(date.getUTCMinutes()).padStart(2, "0");
+  const ss = String(date.getUTCSeconds()).padStart(2, "0");
+
+  return `${yyyy}-${mm}-${dd} ${hh}:${min}:${ss}`;
+};
+
+/**
+ * Converts a database row to a String or null value
+ */
+export const StringOrNull = (value: unknown) => {
+  if (value === null) return null;
+  return String(value);
+};
+
+/**
+ * Converts a column value to a Number or null value
+ */
+export const NumberOrNull = (value: unknown) => {
+  if (value === null) return null;
+  return Number(value);
+};
+
+/**
+ * Converts a column value to a Boolean or null value
+ */
+export const BooleanOrNull = (value: unknown) => {
+  if (value === null) return null;
+  return Boolean(value);
+};

package/src/index.ts

@@ -0,0 +1 @@
+export * from "./types";

package/src/jobs/index.ts

@@ -0,0 +1,286 @@
+import parser from "cron-parser";
+import { parseDate } from "chrono-node";
+
+import { createClient } from "../client";
+import { Jsonify, queryString, toSqliteDateString } from "../client/util";
+import { CreateJobsOptions, JobOptions, JobStatus, Row, NeosqliteClient } from "../types";
+
+interface JobRegistryValue {
+  name: string;
+  options: JobOptions;
+  fn: (data: any) => void;
+  onFailure?: (data: any) => void;
+}
+
+export class NeosqliteJobs {
+  private db: NeosqliteClient;
+  private options: CreateJobsOptions;
+  private jobsRegistry: Record<string, JobRegistryValue> = {};
+
+  // Jobs worker information
+  private activeJobs = 0;
+  private isRunning = false;
+
+  constructor(options: CreateJobsOptions) {
+    this.options = options;
+    this.db = createClient(options);
+  }
+
+  private get jobsTable() {
+    return this.options.jobsTable ?? "jobs";
+  }
+
+  private get maxJobs() {
+    return this.options.maxJobs ?? 10;
+  }
+
+  private get processEvery() {
+    return this.options.processEvery ?? 5000;
+  }
+
+  private get maxRetries() {
+    return this.options.maxRetries ?? 3;
+  }
+
+  /**
+   * Setup tables and start the job worker for neosqlite jobs
+   */
+  public async start() {
+    await this.setupTables();
+    await this.setupJobWorker();
+  }
+
+  /**
+   * Stop the job worker from listening for jobs
+   */
+  public async stop() {
+    this.isRunning = false;
+  }
+
+  /**
+   * Register a job to be run
+   */
+  public register<T extends Record<string, any> = Record<string, any>>(
+    name: string,
+    options: JobOptions,
+    fn: (data: T) => void,
+    onFailure?: (data: T) => void,
+  ) {
+    this.jobsRegistry[name] = { name, options, fn, onFailure };
+  }
+
+  /**
+   * Queue a job to run instantly based on its
+   * priority
+   */
+  public async queue<T extends Record<string, any> = Record<string, any>>(name: string, data?: T) {
+    const job = this.jobsRegistry[name];
+    if (!job) return console.error(`Job ${name} does not exist`);
+
+    this.db.write({
+      sql: queryString(
+        "INSERT INTO " + this.jobsTable,
+        "(name, data, runAt, priority)",
+        "VALUES (:name, :data, CURRENT_TIMESTAMP, :priority)",
+      ),
+      args: {
+        name: job.name,
+        priority: job.options.priority ?? 0,
+        data: data ? JSON.stringify(data) : null,
+      },
+    });
+  }
+
+  /**
+   * Schedule a job to run in the future. This can use a date string, or a human-readable string
+   * such as 'in 10 minutes' or 'next week'
+   */
+  public async schedule<T extends Record<string, any> = Record<string, any>>(date: string, name: string, data?: T) {
+    const job = this.jobsRegistry[name];
+    if (!job) return console.error(`Job ${name} does not exist`);
+
+    const parsedDate = parseDate(date);
+    if (!parsedDate) return;
+
+    this.db.write({
+      sql: queryString(
+        "INSERT INTO " + this.jobsTable,
+        "(name, data, runAt, priority)",
+        "VALUES (:name, :data, :runAt, :priority)",
+      ),
+      args: {
+        name: job.name,
+        priority: job.options.priority ?? 0,
+        data: data ? JSON.stringify(data) : null,
+        runAt: toSqliteDateString(parsedDate),
+      },
+    });
+  }
+
+  /**
+   * Schedule a job to run every time the cron string is met
+   */
+  public async every<T extends Record<string, any> = Record<string, any>>(cronString: string, name: string, data?: T) {
+    const job = this.jobsRegistry[name];
+    if (!job) return console.error(`Job ${name} does not exist`);
+
+    const interval = parser.parse(cronString);
+    const runAt = interval.next().toDate();
+
+    this.db.write({
+      sql: queryString(
+        "INSERT OR IGNORE INTO " + this.jobsTable,
+        "(name, data, runAt, priority, cron)",
+        "VALUES (:name, :data, :runAt, :priority, :cron)",
+      ),
+      args: {
+        name: job.name,
+        priority: job.options.priority ?? 0,
+        data: data ? JSON.stringify(data) : null,
+        runAt: toSqliteDateString(runAt),
+        cron: cronString,
+      },
+    });
+  }
+
+  /**
+   * Creates the jobs table and sets up indexes on it
+   */
+  private async setupTables() {
+    const jobStatuses = Object.values(JobStatus)
+      .map((s) => `'${s}'`)
+      .join(", ");
+
+    this.db.write(
+      queryString(
+        "CREATE TABLE IF NOT EXISTS " + this.jobsTable + " (",
+        "  id INTEGER PRIMARY KEY AUTOINCREMENT,",
+        "  name TEXT NOT NULL,",
+        "  data TEXT,",
+        "  runAt DATETIME NOT NULL,",
+        "  priority INTEGER DEFAULT 0,",
+        "  cron TEXT,",
+        "  attempts INTEGER DEFAULT 0,",
+        "  status TEXT CHECK (status IN ( " + jobStatuses + ")) DEFAULT '" + JobStatus.Pending + "',",
+        "  updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP,",
+        "  createdAt DATETIME DEFAULT CURRENT_TIMESTAMP",
+        ")",
+      ),
+    );
+
+    // Prevent multiple cron jobs from getting scheduled for the same dates
+    this.db.write(
+      `CREATE UNIQUE INDEX IF NOT EXISTS idx_${this.jobsTable}_name_cron_runAt ON ${this.jobsTable} (name, cron, runAt)`,
+    );
+    this.db.write(
+      `CREATE INDEX IF NOT EXISTS idx_${this.jobsTable}_status_runAt_priority ON ${this.jobsTable} (status, runAt, priority DESC)`,
+    );
+  }
+
+  /**
+   * Sets up the jobs worker to run automatically
+   */
+  private async setupJobWorker() {
+    if (this.isRunning) return;
+    this.isRunning = true;
+
+    const loop = async () => {
+      if (!this.isRunning) return;
+
+      try {
+        if (this.activeJobs < this.maxJobs) {
+          const limit = this.maxJobs - this.activeJobs;
+
+          const rows = this.db.writeReturning({
+            sql: queryString(
+              "UPDATE jobs",
+              "SET status = :runningStatus,",
+              "  attempts = attempts + 1,",
+              "  updatedAt = CURRENT_TIMESTAMP",
+              "WHERE id IN (",
+              "  SELECT id",
+              "  FROM jobs",
+              "  WHERE status = :pendingStatus",
+              "    AND runAt <= CURRENT_TIMESTAMP",
+              "  ORDER BY priority DESC, runAt ASC",
+              "  LIMIT :limit",
+              ")",
+              "RETURNING *",
+            ),
+            args: {
+              limit,
+              pendingStatus: JobStatus.Pending,
+              runningStatus: JobStatus.Running,
+            },
+          });
+
+          for (const row of rows) {
+            this.activeJobs++;
+            this.processJob(row).finally(() => {
+              this.activeJobs--;
+            });
+          }
+        }
+      } catch (err) {
+        console.error(err);
+      }
+
+      setTimeout(loop, this.processEvery);
+    };
+
+    loop();
+  }
+
+  private async processJob(row: Row) {
+    const id = Number(row["id"]);
+    const name = String(row["name"]);
+    const attempts = Number(row["attempts"]);
+
+    const cron = row["cron"] ? String(row["cron"]) : null;
+    const data = row["data"] ? Jsonify(String(row["data"])) : null;
+
+    const job = this.jobsRegistry[name];
+    if (!job) return;
+
+    try {
+      await Promise.resolve(job.fn(data));
+
+      this.db.write({
+        sql: queryString("UPDATE " + this.jobsTable, "SET status = :status, updatedAt = CURRENT_TIMESTAMP", "WHERE id = :id"),
+        args: { id, status: JobStatus.Completed },
+      });
+
+      // If this is a cronjob, schedule the next run
+      if (cron) {
+        const interval = parser.parse(cron);
+        const nextRun = interval.next().toDate();
+
+        this.db.write({
+          sql: queryString(
+            "INSERT OR IGNORE INTO " + this.jobsTable,
+            "(name, data, runAt, priority, cron)",
+            "VALUES (:name, :data, :runAt, :priority, :cron)",
+          ),
+          args: {
+            cron,
+            name: job.name,
+            priority: job.options.priority ?? 0,
+            data: data ? JSON.stringify(data) : null,
+            runAt: toSqliteDateString(nextRun),
+          },
+        });
+      }
+    } catch (err) {
+      const updatedStatus = attempts >= this.maxRetries ? JobStatus.Failed : JobStatus.Pending;
+
+      this.db.write({
+        sql: queryString("UPDATE " + this.jobsTable, "SET status = :status, updatedAt = CURRENT_TIMESTAMP", "WHERE id = :id"),
+        args: { id, status: updatedStatus },
+      });
+
+      if (updatedStatus === JobStatus.Failed && job.onFailure) {
+        await Promise.resolve(job.onFailure(err));
+      }
+    }
+  }
+}

package/src/migrations/commands/down.ts

@@ -0,0 +1,50 @@
+import fs from "fs/promises";
+
+import type { MigrationFile } from "..";
+import type { CliCommandOptions } from "../../types";
+
+import { createClient } from "../../client";
+import { createMigrationTableIfNotExists, logError, logSuccess, validateMigrationDirectory } from "../util";
+
+export default async function Down({ migrationTable, migrationPath, ...config }: CliCommandOptions) {
+  const db = createClient(config);
+
+  await validateMigrationDirectory(migrationPath);
+  createMigrationTableIfNotExists(db, migrationTable);
+
+  // Query the last migration that was ran
+  const result = db.readOne("SELECT id, filepath FROM " + migrationTable + " ORDER BY timestamp DESC LIMIT 1");
+
+  if (!result) {
+    logError("No migrations have been ran");
+    process.exit(0);
+  }
+
+  const id = String(result["id"]);
+  const filepath = String(result["filepath"]);
+
+  const files = await fs.readdir(migrationPath);
+  const matchingFile = files.filter((file) => {
+    const fileName = file.split(".")[0];
+    return fileName === filepath;
+  });
+
+  if (!matchingFile.length) {
+    logError(`Unable to find migration ${filepath}`);
+    process.exit(0);
+  }
+
+  const migration: MigrationFile = (await import(`${migrationPath}/${matchingFile[0]}`)).default;
+
+  // Run the migration
+  db.transaction(() => migration.down(db));
+
+  // Store the migration data
+  db.write({
+    sql: "DELETE FROM " + migrationTable + " WHERE id = :id",
+    args: { id },
+  });
+
+  logSuccess(`Reverted migration ${filepath}`);
+  process.exit(0);
+}

package/src/migrations/commands/new.ts

@@ -0,0 +1,33 @@
+import { mkdir, writeFile } from "fs/promises";
+
+import { CliCommandOptions } from "../../types";
+import { logError, logSuccess } from "../util";
+
+interface NewCommandOptions extends CliCommandOptions {
+  name: string;
+}
+
+export default async function New({ name, migrationPath }: NewCommandOptions) {
+  const epochTime = Date.now();
+  const fileName = `${epochTime}_${name}.ts`;
+
+  const fileContents = `
+import type { NeosqliteClient } from 'neosqlite';
+
+export default class Migration_${epochTime}_${name} {
+  public static async up(db: NeosqliteClient) {
+    // Add your migration logic here
+  }
+
+  public static async down(db: NeosqliteClient) {
+    // Add your rollback logic here
+  }
+}
+  `;
+
+  await mkdir(migrationPath, { recursive: true }).catch((e) => logError(e.message));
+  await writeFile(`${migrationPath}/${fileName}`, fileContents).catch((e) => logError(e.message));
+
+  logSuccess(`Created migration ${fileName}`);
+  process.exit(0);
+}