@ghom/orm 1.7.1 → 1.8.0

package/src/app/backup.ts

@@ -0,0 +1,211 @@
+import fs from "fs"
+import path from "path"
+import util from "util"
+import csv from "json-2-csv"
+import csvParser from "csv-parser"
+import { ORM } from "./orm.js"
+import { Table } from "./table.js"
+import {
+  DEFAULT_BACKUP_CHUNK_SIZE,
+  DEFAULT_BACKUP_LOCATION,
+  DEFAULT_LOGGER_HIGHLIGHT,
+  DEFAULT_LOGGER_RAW_VALUE,
+} from "./util.js"
+
+export async function backupTable(table: Table, dirname?: string) {
+  if (!table.orm) throw new Error("missing ORM")
+
+  let offset = 0
+  let chunkIndex = 0
+
+  const chunkDir = path.join(
+    table.orm.config.backups?.location ?? DEFAULT_BACKUP_LOCATION,
+    dirname ?? "",
+  )
+
+  if (!fs.existsSync(chunkDir)) {
+    fs.mkdirSync(chunkDir, { recursive: true })
+
+    console.log(
+      `Backup directory ${util.styleText(
+        table.orm.config.loggerStyles?.highlight ?? DEFAULT_LOGGER_HIGHLIGHT,
+        path.relative(process.cwd(), chunkDir),
+      )} created.`,
+    )
+  }
+
+  try {
+    // Compter le nombre total d'enregistrements dans la table
+    const rowCount = await table.count()
+    const limit = 1000 // Limite par requête
+    const chunkCount = Math.ceil(rowCount / limit)
+
+    let writeStream: fs.WriteStream | null = null
+    const closePromises = [] // Tableau pour stocker les promesses de fermeture
+
+    while (offset < rowCount) {
+      // Récupérer un "chunk" de données
+      const rows = await table.query.select("*").limit(limit).offset(offset)
+
+      // Convertir les données en CSV
+      const csvData = csv.json2csv(rows)
+
+      // Si aucun fichier n'est créé ou qu'on a dépassé la taille max du chunk, on crée un nouveau fichier CSV
+      if (
+        !writeStream ||
+        writeStream.bytesWritten + Buffer.byteLength(csvData, "utf8") >
+          (table.orm.config.backups?.chunkSize ?? DEFAULT_BACKUP_CHUNK_SIZE)
+      ) {
+        if (writeStream) {
+          closePromises.push(
+            new Promise((resolve) => writeStream!.end(resolve)),
+          ) // Ajouter la promesse de fermeture
+        }
+
+        const chunkFile = path.join(
+          chunkDir,
+          `${table.options.name}_chunk_${chunkIndex}.csv`,
+        )
+        writeStream = fs.createWriteStream(chunkFile, { flags: "a" })
+        chunkIndex++
+      }
+
+      // Écrire les données dans le stream
+      writeStream.write(csvData)
+      offset += limit
+
+      process.stdout.write(
+        `\rBacking up table ${util.styleText(
+          table.orm.config.loggerStyles?.highlight ?? DEFAULT_LOGGER_HIGHLIGHT,
+          table.options.name,
+        )}: ${util.styleText(
+          table.orm.config.loggerStyles?.rawValue ?? DEFAULT_LOGGER_RAW_VALUE,
+          String(Math.round((chunkIndex / chunkCount) * 100)),
+        )}%`,
+      )
+    }
+
+    if (writeStream) {
+      closePromises.push(new Promise((resolve) => writeStream!.end(resolve))) // Ajouter la promesse de fermeture pour le dernier stream
+    }
+
+    // Attendre que tous les flux d'écriture soient fermés
+    await Promise.all(closePromises)
+
+    console.log(
+      `\nBackup of table ${util.styleText(
+        table.orm.config.loggerStyles?.highlight ?? DEFAULT_LOGGER_HIGHLIGHT,
+        table.options.name,
+      )} completed.`,
+    )
+  } catch (error) {
+    console.error(
+      `\nError while backing up table ${util.styleText(
+        table.orm.config.loggerStyles?.highlight ?? DEFAULT_LOGGER_HIGHLIGHT,
+        table.options.name,
+      )}:`,
+      error,
+    )
+  }
+}
+
+export async function restoreBackup(table: Table, dirname?: string) {
+  if (!table.orm) throw new Error("missing ORM")
+
+  const chunkDir = path.join(
+    table.orm.config.backups?.location ?? DEFAULT_BACKUP_LOCATION,
+    dirname ?? "",
+  )
+
+  const chunkFiles = fs
+    .readdirSync(chunkDir)
+    .filter((file) => file.split("_chunk_")[0] === table.options.name)
+
+  await table.query.truncate()
+
+  try {
+    const limit = 1000 // Limite par requête
+
+    for (let chunkFile of chunkFiles) {
+      const filePath = path.join(chunkDir, chunkFile)
+
+      let rows: any[] = []
+
+      await new Promise<void>((resolve, reject) => {
+        fs.createReadStream(filePath)
+          .pipe(csvParser())
+          .on("data", async (row) => {
+            rows.push(row)
+
+            if (rows.length > limit) {
+              const rowsCopy = rows.slice()
+              rows = []
+              await table.query.insert(rowsCopy)
+            }
+          })
+          .on("end", async () => {
+            // Insérer les données dans la table une fois le fichier entièrement lu
+            if (rows.length > 0) await table.query.insert(rows)
+
+            console.log(
+              `Restored chunk ${util.styleText(
+                table.orm!.config.loggerStyles?.highlight ??
+                  DEFAULT_LOGGER_HIGHLIGHT,
+                chunkFile,
+              )} into table ${util.styleText(
+                table.orm!.config.loggerStyles?.highlight ??
+                  DEFAULT_LOGGER_HIGHLIGHT,
+                table.options.name,
+              )}.`,
+            )
+
+            resolve()
+          })
+          .on("error", reject)
+      })
+    }
+  } catch (error) {
+    console.error(
+      `Error while restoring backup of table ${util.styleText(
+        table.orm.config.loggerStyles?.highlight ?? DEFAULT_LOGGER_HIGHLIGHT,
+        table.options.name,
+      )}:`,
+      error,
+    )
+  }
+
+  console.log(
+    `Backup of table ${util.styleText(
+      table.orm.config.loggerStyles?.highlight ?? DEFAULT_LOGGER_HIGHLIGHT,
+      table.options.name,
+    )} restored.`,
+  )
+}
+
+export async function disableForeignKeys(orm: ORM) {
+  const result = await Promise.allSettled([
+    orm.raw("SET session_replication_role = replica;"), // for pg
+    orm.raw("PRAGMA foreign_keys = OFF;"), // for sqlite3
+    orm.raw("SET FOREIGN_KEY_CHECKS = 0;"), // for mysql2
+  ])
+
+  const errors = result.filter((r) => r.status === "rejected")
+
+  if (errors.length === 3) {
+    throw new Error("Failed to disable foreign key constraints.")
+  }
+}
+
+export async function enableForeignKeys(orm: ORM) {
+  const result = await Promise.allSettled([
+    orm.raw("SET session_replication_role = DEFAULT;"), // for pg
+    orm.raw("PRAGMA foreign_keys = ON;"), // for sqlite3
+    orm.raw("SET FOREIGN_KEY_CHECKS = 1;"), // for mysql2
+  ])
+
+  const errors = result.filter((r) => r.status === "rejected")
+
+  if (errors.length === 3) {
+    throw new Error("Failed to enable foreign key constraints.")
+  }
+}

package/src/app/caching.ts

@@ -0,0 +1,51 @@
+export interface ResponseCacheData<Value> {
+  value: Value
+  expires: number
+  outdated?: boolean
+}
+
+/**
+ * Advanced cache for async queries
+ */
+export class ResponseCache<Params extends any[], Value> {
+  private _cache = new Map<string, ResponseCacheData<Value>>()
+
+  constructor(
+    private _request: (...params: Params) => Value,
+    private _timeout: number,
+  ) {}
+
+  get(id: string, ...params: Params): Value {
+    const cached = this._cache.get(id)
+
+    if (!cached || cached.expires < Date.now()) {
+      this._cache.set(id, {
+        value: this._request(...params),
+        expires: Date.now() + this._timeout,
+      })
+    }
+
+    return this._cache.get(id)!.value
+  }
+
+  fetch(id: string, ...params: Params): Value {
+    this._cache.set(id, {
+      value: this._request(...params),
+      expires: Date.now() + this._timeout,
+    })
+
+    return this._cache.get(id)!.value
+  }
+
+  invalidate(): void
+  invalidate(id: string): void
+  invalidate(id?: string): void {
+    if (!id) {
+      this._cache.clear()
+
+      return
+    }
+
+    this._cache.delete(id)
+  }
+}

package/src/app/orm.ts

@@ -1,17 +1,15 @@
-import fs from "fs"
 import url from "url"
-import path from "path"
 import { Handler } from "@ghom/handler"
 import { Knex, default as knex } from "knex"
+import { isCJS, TextStyle } from "./util.js"
 import { MigrationData, Table } from "./table.js"
-import chalk, { Color } from "chalk"
-
-const defaultBackupDir = path.join(process.cwd(), "backup")
-
-const pack = JSON.parse(
-  fs.readFileSync(path.join(process.cwd(), "package.json"), "utf8"),
-)
-const isCJS = pack.type === "commonjs" || pack.type == void 0
+import {
+  backupTable,
+  restoreBackup,
+  disableForeignKeys,
+  enableForeignKeys,
+} from "./backup.js"
+import { ResponseCache } from "./caching"
 
 export interface ILogger {
   log: (message: string) => void
@@ -23,7 +21,7 @@ export interface ORMConfig {
   /**
    * path to the directory that contains js files of tables
    */
-  location: string
+  tableLocation: string
 
   /**
    * database configuration
@@ -37,19 +35,37 @@ export interface ORMConfig {
 
   /**
    * Pattern used on logs when the table files are loaded or created. <br>
-   * Based on Chalk color-method names.
+   * Based on node:util.styleText style names.
    */
-  loggerColors?: {
-    highlight: typeof Color
-    rawValue: typeof Color
-    description: typeof Color
+  loggerStyles?: {
+    highlight: TextStyle
+    rawValue: TextStyle
+    description: TextStyle
   }
+
+  /**
+   * Configuration for the database backups.
+   */
+  backups?: {
+    location?: string
+    chunkSize?: number
+  }
+
+  /**
+   * The cache time in milliseconds. <br>
+   * Default is `Infinity`.
+   */
+  caching?: number
 }
 
 export class ORM {
+  private _ready = false
+
   public database: Knex
   public handler: Handler<Table<any>>
 
+  public _rawCache: ResponseCache<[sql: string], Knex.Raw>
+
   constructor(public config: ORMConfig) {
     this.database = knex(
       config.database ?? {
@@ -61,13 +77,18 @@ export class ORM {
       },
     )
 
-    this.handler = new Handler(config.location, {
+    this.handler = new Handler(config.tableLocation, {
       loader: (filepath) =>
         import(isCJS ? filepath : url.pathToFileURL(filepath).href).then(
           (file) => file.default,
         ),
       pattern: /\.js$/,
     })
+
+    this._rawCache = new ResponseCache(
+      (raw: string) => this.raw(raw),
+      config.caching ?? Infinity,
+    )
   }
 
   get cachedTables() {
@@ -79,7 +100,7 @@ export class ORM {
   }
 
   hasCachedTable(name: string) {
-    return this.cachedTables.some((table) => table.options.name)
+    return this.cachedTables.some((table) => table.options.name === name)
   }
 
   async hasTable(name: string): Promise<boolean> {
@@ -92,21 +113,19 @@ export class ORM {
   async init() {
     await this.handler.init()
 
-    try {
-      await this.database.raw("PRAGMA foreign_keys = ON;")
-    } catch (error) {}
-
-    const migration = new Table<MigrationData>({
-      name: "migration",
-      priority: Infinity,
-      setup: (table) => {
-        table.string("table").unique().notNullable()
-        table.integer("version").notNullable()
-      },
-    })
+    await enableForeignKeys(this)
 
-    migration.orm = this
-    await migration.make()
+    this.handler.elements.set(
+      "migration",
+      new Table<MigrationData>({
+        name: "migration",
+        priority: Infinity,
+        setup: (table) => {
+          table.string("table").unique().notNullable()
+          table.integer("version").notNullable()
+        },
+      }),
+    )
 
     for (const table of this.cachedTables.sort(
       (a, b) => (b.options.priority ?? 0) - (a.options.priority ?? 0),
@@ -114,82 +133,59 @@ export class ORM {
       table.orm = this
       await table.make()
     }
+
+    this._ready = true
   }
 
   raw(sql: Knex.Value): Knex.Raw {
+    if (this._ready) this.cache.invalidate()
     return this.database.raw(sql)
   }
 
-  // /**
-  //  * Extract the database to a CSV file for each table.
-  //  */
-  // async createBackup(dir = defaultBackupDir) {
-  //   const tables = [...this.handler.elements.values()]
-  //
-  //   for (const table of tables) {
-  //     await this.database
-  //       .select()
-  //       .from(table.options.name)
-  //       .then(async (rows) => {
-  //         const csv = rows.map((row) => Object.values(row).join(",")).join("\n")
-  //
-  //         return fs.promises.writeFile(
-  //           path.join(dir, `${table.options.name}.csv`),
-  //           csv,
-  //           "utf8",
-  //         )
-  //       })
-  //   }
-  // }
-  //
-  // /**
-  //  * Import a CSV file for each table to the database.
-  //  */
-  // async restoreBackup(dir = defaultBackupDir) {
-  //   const tables = [...this.handler.elements.values()].sort(
-  //     (a, b) => (b.options.priority ?? 0) - (a.options.priority ?? 0),
-  //   )
-  //
-  //   for (const table of tables) {
-  //     const columnInfo = await table.getColumns()
-  //
-  //     let csv: string
-  //
-  //     try {
-  //       csv = await fs.promises.readFile(
-  //         path.join(dir, `${table.options.name}.csv`),
-  //         "utf8",
-  //       )
-  //     } catch (error) {
-  //       this.config.logger?.warn(
-  //         `missing backup file for table ${chalk[
-  //           this.config.loggerColors?.highlight ?? "blueBright"
-  //         ](table.options.name)}`,
-  //       )
-  //
-  //       continue
-  //     }
-  //
-  //     if (csv.trim().length === 0) continue
-  //
-  //     const rows = csv
-  //       .split("\n")
-  //       .map((row) => row.split(","))
-  //       .map((row) => {
-  //         const data: any = {}
-  //
-  //         let index = 0
-  //
-  //         for (const [name, info] of Object.entries(columnInfo)) {
-  //           data[name] =
-  //             info.type === "integer" ? Number(row[index]) : row[index]
-  //           index++
-  //         }
-  //
-  //         return data
-  //       })
-  //
-  //     await this.database(table.options.name).insert(rows)
-  //   }
-  // }
+  cache = {
+    raw: (sql: string, anyDataUpdated?: boolean): Knex.Raw => {
+      if (anyDataUpdated) this.cache.invalidate()
+      return this._rawCache!.get(sql, sql)
+    },
+    invalidate: () => {
+      this._rawCache.invalidate()
+      this.cachedTables.forEach((table) => table.cache.invalidate())
+    },
+  }
+
+  /**
+   * Create a backup of the database. <br>
+   * The backup will be saved in the location specified in the config.
+   */
+  async createBackup(dirname?: string) {
+    try {
+      for (let table of this.cachedTables) {
+        await backupTable(table, dirname)
+      }
+
+      console.log("Database backup created.")
+    } catch (error) {
+      console.error("Error while creating backup of the database.", error)
+    }
+  }
+
+  /**
+   * Restore the database from the backup. <br>
+   * @warning This will delete all the data in the tables.
+   */
+  async restoreBackup(dirname?: string) {
+    try {
+      await disableForeignKeys(this)
+
+      for (let table of this.cachedTables) {
+        await restoreBackup(table, dirname)
+      }
+
+      await enableForeignKeys(this)
+
+      console.log("Database restored from backup.")
+    } catch (error) {
+      console.error("Error while restoring backup of the database.", error)
+    }
+  }
 }