ops_backups 0.1.7 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2e41ca2aa4e5285b3c9dc689f2a57b0b0e71fdbf740630d0f95ac9791012a6d
-  data.tar.gz: 4da4c1b4a7f0afcd696c55308e0300502dce4db1d3a6a092dfef5995f52e0b5b
+  metadata.gz: 9c5630e25e938e9e3ff8df947ec38c0135b12a2d94b3a46e9c6d55d3b70f5a0c
+  data.tar.gz: 8bb0cab72c776e01cda428050591c97658de1401cf3420b7bde758ace3b44c06
 SHA512:
-  metadata.gz: 359de7ff5df47e55a476a691bf2c8f911753e9027c5988ab18ed7c07f4af2dc96838f7099ef630a22f008124912b971556f86420c2ef0744eeb6ef6addc7b00e
-  data.tar.gz: 5322ddcfd04196bdf6b7c787d8310da2e03368deb7223045193b042d6f34d9e886639b095dd4c9807c991e195af2351d3cc1534e4b639cb0df93ced264984c3a
+  metadata.gz: 9c06a9d87783b4950fac61f6e60f75d67288cff322d47bf953a1ffd33c7cbf71972b1b24cff28bd5310e0f9d92e4ebd1fa5280ab97a4ff8f37f43ffbf99f2d00
+  data.tar.gz: 357728c90e376f3101b07f948d2a28b379f453ddc8cd8aa7d0e29009e83506a4a8bc7c2ae0d3aeb47038f18fa31d305398b4cf4291ac1d6698c49c0178286b03

data/README.md

@@ -14,6 +14,30 @@ bundle add ops_backups
 rails generate ops_backups:install
 ```
 
+# Scripts
+
+`pgdb` is a script that can be used to backup and restore a postgres database and wraps pg_dump and pg_restore in the context of [Kamal](https://kamal-deploy.org/) for ease of use. Environment here coresponds to the destination in Kamal.
+
+```
+Usage: pgdb <command> <environment> [<additional_args>]
+
+Available commands:
+  console <environment>                   - Opens a database console for the specified environment.
+  create <environment>                    - Creates a new database for the specified environment.
+  revert <environment> <backup_db_name>   - Reverts the database to a specified backup.
+  backup <environment>                    - Creates a clone of the remote database as a backup on the server.
+  backup:local <environment>              - Backs up the remote database to a local dump file.
+  drop <environment>                      - Drops the database of the specified environment after confirmation.
+  pull <environment|url> [local_db_name]  - Pulls the remote database to a local database with an optional custom name (optional).
+  push <environment> <local_db_name>      - Pushes a local database to the remote environment, creating a backup of the remote.
+  rename <environment> [new_db_name]      - Renames the main database of the specified environment, optionally using a new name (optional).
+  copy <source_env> <target_env>          - Copies a database from one environment to another, with backup of the target database.
+  list <environment>                      - Lists databases for the base URL associated with the environment.
+
+set EXCLUDE_TABLE_DATA to exclude data from specific tables on export
+e.g. EXCLUDE_TABLE_DATA=versions   bin/pgdb pull staging
+```
+
 # ActiveAdmin Integration
 
 If you are using ActiveAdmin, you can manage your backups through the admin interface. The generator rails generate ops_backups:activeadmin will create the necessary example configuration.
@@ -30,6 +54,5 @@ rails generate ops_backups:activeadmin
 - Create Unversioned Backup: Create a new unversioned backup, excluding specific tables.
 
 
-
 ## License
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/bin/pgdb

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+exec(File.join(File.dirname(__FILE__), "pgdb.sh"), *ARGV)

data/bin/pgdb.sh

@@ -0,0 +1,383 @@
+#!/bin/bash
+
+# Function to calculate the backup database name
+backup_db_name() {
+  local target_db_name="$1"
+  local current_date=$(date +%Y%m%d)
+  local timestamp=$(date +%H%M%S)
+  echo "${target_db_name}_${current_date}_${timestamp}"
+}
+
+# Function to rename a PostgreSQL database
+# Usage: db_rename <db_url> <old_db_name> <new_db_name>
+db_rename() {
+  local db_url="$1"
+  local old_db_name="$2"
+  local new_db_name="$3"
+
+  if [[ -z "$old_db_name" || -z "$new_db_name" || -z "$db_url" ]]; then
+    echo "Usage: rename <db_url> <old_db_name> <new_db_name>"
+    exit 1
+  fi
+
+  echo "Renaming database from $old_db_name to $new_db_name on $db_url"
+  # Run the ALTER DATABASE command to rename the database
+  psql "$db_url" -c "ALTER DATABASE \"$old_db_name\" RENAME TO \"$new_db_name\";"
+}
+
+# Function to create a PostgreSQL database
+# Usage: db_create <db_url> <new_db_name>
+db_create() {
+  local db_url="$1"
+  local new_db_name="$2"
+
+  if [[ -z "$new_db_name" || -z "$db_url" ]]; then
+    echo "Usage: create <db_url> <new_db_name>"
+    exit 1
+  fi
+
+  echo "Creating database $new_db_name on $db_url"
+  # Run the CREATE DATABASE command to rename the database
+  psql "$db_url" -c "CREATE DATABASE \"$new_db_name\";"
+}
+
+# Function for pulling a database from a remote server to the local machine
+# Usage: db_pull <db_url> <db_name>
+db_pull() {
+  local db_url="$1"
+  local db_name="$2"
+  local dump_file="${db_name}.dump"
+
+  # Step 1: Dump the remote database
+  echo "Dumping database from $db_url to $dump_file..."
+
+  local exclude=""
+  if [[ -n "$EXCLUDE_TABLE_DATA" ]]; then
+    exclude="--exclude-table-data=$EXCLUDE_TABLE_DATA"
+  fi
+  pg_dump --no-owner -v -Fc $exclude -f "$dump_file" "$db_url"
+
+  # Step 2: Check if local database exists, prompt for confirmation, and drop if confirmed
+  if psql -lqt | cut -d \| -f 1 | grep -qw "$db_name"; then
+    read -p "Database $db_name already exists. Do you want to drop it and recreate? (y/n): " confirm
+    if [[ "$confirm" != "y" ]]; then
+      echo "Aborting..."
+      exit 1
+    fi
+    echo "Dropping existing database..."
+    dropdb "$db_name"
+  fi
+
+  # Step 3: Create a new local database
+  echo "Creating local database..."
+  createdb "$db_name"
+
+  # Step 4: Restore the dump to the new local database
+  echo "Restoring database..."
+  pg_restore --no-owner --no-comments -v -d "postgres://localhost/$db_name" "$dump_file"
+
+  echo "Database pull completed: $db_name"
+}
+
+
+# Function for "push" subcommand
+# Usage: db_push <local_db_name> <remote_db_url>
+db_push() {
+  local local_db_name="$1"
+  local remote_db_url="$2"
+  local dump_file="$(backup_db_name "$local_db_name").dump"
+
+  # Dump the local database
+  echo "Dumping local database..."
+  local exclude=""
+  if [[ -n "$EXCLUDE_TABLE_DATA" ]]; then
+    exclude="--exclude-table-data=$EXCLUDE_TABLE_DATA"
+  fi
+  echo pg_dump --no-owner -v -Fc $exclude -f $dump_file postgres://localhost/$local_db_name
+  pg_dump --no-owner -v -Fc $exclude -f $dump_file postgres://localhost/$local_db_name
+
+  # Restore the dump to the new remote database
+  echo pg_restore --no-owner --no-comments -v -d $remote_db_url $dump_file
+  pg_restore --no-owner --no-comments -v -d $remote_db_url $dump_file
+
+  echo "Database push completed: $local_db_name"
+}
+
+# Function for copying a database from one server to another
+# Usage: db_copy <source_db_url> <target_db_url>
+db_copy() {
+  local source_db_url="$1"
+  local target_db_url="$2"
+  local current_date=$(date +%Y%m%d)
+  local timestamp=$(date +%H%M%S)
+  local source_db_name="${source_db_url##*/}"
+  local target_db_name="${target_db_url##*/}"
+  local dump_file="${source_db_name}_${target_db_name}_${current_date}_${timestamp}.dump"
+
+  # Step 1: Dump the source database
+  echo "Dumping source database..."
+  local exclude=""
+  if [[ -n "$EXCLUDE_TABLE_DATA" ]]; then
+    exclude="--exclude-table-data=$EXCLUDE_TABLE_DATA"
+  fi
+  pg_dump --no-owner -v -Fc $exclude -f "$dump_file" "$source_db_url"
+
+  # stop if the dump failed
+  if [[ $? -ne 0 ]]; then
+    echo "Dump failed. Aborting..."
+    exit 1
+  fi
+
+  # Step 2: Create the target database (and backup if needed)
+  local backup_db_name=$(backup_db_name "$target_db_name")
+  local source_base_url="${source_db_url%/*}"
+  local target_base_url="${target_db_url%/*}"
+  db_rename "$target_base_url" "$target_db_name" "$backup_db_name"
+  echo "Creating new target database: $target_db_name..."
+  psql "$target_base_url" -c "CREATE DATABASE \"$target_db_name\";"
+
+  # Step 3: Restore the dump to the target database
+  echo "Restoring dump to target database..."
+  pg_restore --no-owner --no-comments -v -d "$target_db_url" "$dump_file"
+
+  echo "Database copy completed: $source_db_name to $target_db_name"
+}
+
+# function to rename a database
+# Usage: db_rename <base_url> <old_name> <new_name>
+db_rename() {
+  local base_url="$1"
+  local old_name="$2"
+  local new_name="$3"
+  # Run the ALTER DATABASE command to rename the database
+  psql "$base_url" -c "ALTER DATABASE \"$old_name\" RENAME TO \"$new_name\";"
+}
+
+# function to clone a database on the server
+# Usage: db_clone_on_server <base_url> <db_name> <clone_db_name:optional>
+db_clone_on_server() {
+  local base_url="$1"
+  local db_name="$2"
+  # if the third param is available, use it as the backup db name
+  # else append current_date to db_name
+  if [[ -n "$3" ]]; then
+    local clone_db_name="$3"
+  else
+    local clone_db_name=$(backup_db_name "$db_name")
+  fi
+  psql "$base_url" -c "CREATE DATABASE \"$clone_db_name\" WITH TEMPLATE \"$db_name\";"
+  # check if the clone db was created
+  if psql "$base_url" -lqt | cut -d \| -f 1 | grep -qw "$clone_db_name"; then
+    echo "Database $db_name clones as $clone_db_name"
+  fi
+}
+
+# Function to extract DATABASE_URL from the secrets file
+fetch_db_url() {
+  local env="$1"
+
+  # If the env starts with postgres://, return it as is
+  if [[ "$env" == postgres://* ]]; then
+    echo "$env"
+    return
+  fi
+
+  local secrets_file=".kamal/secrets.${env}"
+
+  if [[ ! -f "$secrets_file" ]]; then
+    echo "Secrets file not found: $secrets_file"
+    exit 1
+  fi
+
+  # Extract DATABASE_URL from the secrets file
+  local db_url=$(awk -F '=' '/^DATABASE_URL=/{print $2}' "$secrets_file")
+
+  if [[ -z "$db_url" ]]; then
+    echo "DATABASE_URL not found in $secrets_file"
+    exit 1
+  fi
+
+  echo "$db_url"
+}
+
+command="$1"
+env="$2"
+current_date=$(date +%Y%m%d)
+timestamp=$(date +%H%M%S)
+
+# Fetch the DATABASE_URL from the secrets file
+db_url=$(fetch_db_url "$env")
+# Extract everything before the last '/'
+base_url="${db_url%/*}"
+# Extract the database name after the last '/'
+db_name="${db_url##*/}"
+
+case "$command" in
+console)
+  echo "url: $db_url"
+  psql "$db_url"
+  ;;
+
+copy)
+  if [[ $# -lt 3 ]]; then
+    echo "Usage: pgdb copy <source_environment> <target_environment>"
+    exit 1
+  fi
+
+  source_environment="$2"
+  target_environment="$3"
+
+  kamal app stop -d "$source_environment"
+  kamal app stop -d "$target_environment"
+
+  # Fetch the source and target DATABASE_URLs
+  source_db_url=$db_url
+  target_db_url=$(fetch_db_url "$3")
+
+  source_base_url="${source_db_url%/*}"
+  target_base_url="${target_db_url%/*}"
+
+  # if source and target base are the same, copy on the server
+  # else copy between servers using pg_dump and pg_restore
+  if [[ "$source_base_url" == "$target_base_url" ]]; then
+    echo "Renaming current target database from $3 to $3_${current_date}_${timestamp}"
+    backup_db_name=$(backup_db_name "$3")
+    db_rename "$base_url" "$3" "$backup_db_name"
+    # echo "Cloning database from $2 to $3"
+    db_clone_on_server "$base_url" "$2" "$3"
+  else
+    # Copy the source DB to the target DB
+    db_copy "$source_db_url" "${target_db_url}"
+  fi
+
+  kamal app boot -d "$source_environment"
+  kamal app boot -d "$target_environment"
+  ;;
+
+pull)
+  if (( $# < 2 || $# > 3 )); then
+    echo "Usage: pgdb pull <environment> <local_db_name:optional>"
+    exit 1
+  fi
+  if [[ -n "$3" ]]; then
+    local_db_name="$3"
+  else
+    local_db_name=$(backup_db_name "$env")
+  fi
+  db_pull "$db_url" "$local_db_name"
+  ;;
+
+push)
+  if [[ $# -lt 3 ]]; then
+    echo "Usage: pgdb push <environment> <local_db_name>"
+    exit 1
+  fi
+  local_db_name="$3"
+  backup_db_name=$(backup_db_name "$db_name")
+
+  kamal app stop -d "$env"
+  db_rename "$base_url" "$db_name" "$backup_db_name"
+  db_create "$base_url" "$db_name"
+  db_push "$local_db_name" "$db_url"
+  kamal app boot -d "$env"
+  ;;
+
+rename)
+  if (( $# < 2 || $# > 3 )); then
+    echo "Usage: pgdb rename <environment> <new_db_name:optional>"
+    exit 1
+  fi
+
+  # if $3 present, use it as new_db_name else append current_date to db_name
+  if [[ -n "$3" ]]; then
+    new_db_name="$3"
+  else
+    new_db_name="${db_name}_${current_date}"
+  fi
+
+  db_rename "$base_url" "$db_name" "$new_db_name"
+  ;;
+
+list)
+  echo "Listing databases for $base_url"
+  echo
+  psql "$base_url" -c "\l" | grep "$db_name"
+  ;;
+
+drop)
+  echo "Dropping database $db_name"
+  # ask for confirmation
+  read -p "Are you sure you want to drop the database $db_name? (y/n): " -n 1 -r
+  # abort if not confirmed
+  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo
+    echo "Aborted!"
+    exit 1
+  fi
+  psql "$base_url" -c "DROP DATABASE \"$db_name\";"
+  ;;
+
+backup:local)
+  echo "Backing up database $db_name on the remote to a local dump"
+  backup_db_name=$(backup_db_name "$db_name")
+  db_pull "$db_url" "$backup_db_name"
+  ;;
+
+backup)
+  echo "Backing up database $db_name on the remote"
+  kamal app stop -d "$env"
+  db_clone_on_server "$base_url" "$db_name"
+  kamal app boot -d "$env"
+  ;;
+
+revert)
+  echo "Reverting database $db_name from $3"
+  # ask for confirmation
+  read -p "Are you sure you want to revert the database $db_name to $3? (y/n): " -n 1 -r
+  # abort if not confirmed
+  echo
+  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "Aborted!"
+    exit 1
+  fi
+  # check if the chosen db exists
+  if ! psql "$base_url" -lqt | cut -d \| -f 1 | grep -qw "$3"; then
+    echo "Database $3 does not exist, aborting the revert!"
+    exit 1
+  fi
+  kamal app stop -d "$env"
+  # clone to the default backup name
+  db_clone_on_server "$base_url" "$db_name"
+  # drop the current db
+  psql "$base_url" -c "DROP DATABASE \"$db_name\";"
+  # rename the chosen db to the original db name
+  db_rename "$base_url" "$db_name" "$3"
+  kamal app boot -d "$env"
+  ;;
+
+create)
+  db_create "$base_url" "$env"
+  ;;
+
+*)
+  echo "Unknown command: $command"
+  echo "Usage: pgdb <command> <environment> [<additional_args>]"
+  echo
+  echo "Available commands:"
+  echo "  console <environment>                   - Opens a database console for the specified environment."
+  echo "  create <environment>                    - Creates a new database for the specified environment."
+  echo "  revert <environment> <backup_db_name>   - Reverts the database to a specified backup."
+  echo "  backup <environment>                    - Creates a clone of the remote database as a backup on the server."
+  echo "  backup:local <environment>              - Backs up the remote database to a local dump file."
+  echo "  drop <environment>                      - Drops the database of the specified environment after confirmation."
+  echo "  pull <environment|url> [local_db_name]  - Pulls the remote database to a local database with an optional custom name (optional)."
+  echo "  push <environment> <local_db_name>      - Pushes a local database to the remote environment, creating a backup of the remote."
+  echo "  rename <environment> [new_db_name]      - Renames the main database of the specified environment, optionally using a new name (optional)."
+  echo "  copy <source_env> <target_env>          - Copies a database from one environment to another, with backup of the target database."
+  echo "  list <environment>                      - Lists databases for the base URL associated with the environment."
+  echo
+  echo "set EXCLUDE_TABLE_DATA to exclude data from specific tables on export"
+  echo "e.g. EXCLUDE_TABLE_DATA=versions   bin/pgdb pull staging"
+  exit 1
+  ;;
+esac

data/db/migrate/20241114173612_create_ops_backups.rb

@@ -1,4 +1,4 @@
-class CreateOpsBackups < ActiveRecord::Migration[8.0]
+class CreateOpsBackups < ActiveRecord::Migration[5.0]
   def change
     # check if the table already exists
     return if table_exists?(:ops_backups)

data/lib/ops_backups/version.rb

@@ -1,3 +1,3 @@
 module OpsBackups
-  VERSION = "0.1.7"
+  VERSION = "0.1.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ops_backups
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.9
 platform: ruby
 authors:
 - Koen Handekyn
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-20 00:00:00.000000000 Z
+date: 2024-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -28,7 +28,8 @@ description: This gem provides functionality to backup PostgreSQL databases to A
   (S3) from within a Rails context.
 email:
 - github.com@handekyn.com
-executables: []
+executables:
+- pgdb
 extensions: []
 extra_rdoc_files: []
 files:
@@ -46,6 +47,8 @@ files:
 - app/models/ops_backups/application_record.rb
 - app/models/ops_backups/backup.rb
 - app/views/layouts/ops_backups/application.html.erb
+- bin/pgdb
+- bin/pgdb.sh
 - config/locales/en.yml
 - config/routes.rb
 - db/migrate/20241114173612_create_ops_backups.rb
@@ -57,13 +60,13 @@ files:
 - lib/ops_backups/engine.rb
 - lib/ops_backups/version.rb
 - lib/tasks/ops_backups_tasks.rake
-homepage: https://github.com/koenhandekyn/ops-backups
+homepage: https://github.com/koenhandekyn/ops_backups
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/koenhandekyn/ops-backups
-  source_code_uri: https://github.com/koenhandekyn/ops-backups/tree/main
-  changelog_uri: https://github.com/koenhandekyn/ops-backups/blob/main/CHANGELOG.md
+  homepage_uri: https://github.com/koenhandekyn/ops_backups
+  source_code_uri: https://github.com/koenhandekyn/ops_backups/tree/main
+  changelog_uri: https://github.com/koenhandekyn/ops_backups/blob/main/CHANGELOG.md
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -79,7 +82,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
+rubygems_version: 3.5.23
 signing_key:
 specification_version: 4
 summary: A Ruby gem for managing PostgreSQL backups.