ops_backups 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c11d46d1083b3b1b45c330e4b74db480d82d80ebd49356a74934dd63409f2c01
+  data.tar.gz: d41344f6c9066beac9ba54ab9e96165fec31b33335b85af0ec26beebf2fba875
+SHA512:
+  metadata.gz: 64d6bb17c0bc7e78cf236d4a2077aa3cbb0cf520576544e8b75c16a9745c19e1b2a4886ad548c8446ac6405f1e366c17873ac540a9ec641e98a45002cf5b4571
+  data.tar.gz: 93ba27887a32e0a361187ef94189c6a574849f65fe5542a0f09fe329c1bfb42121675258ed97a8ce9b85136a9634902028869eb9e945b7c4163307556e3b7150

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright Koen Handekyn
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,132 @@
+# Ops
+
+## Description
+Ops is a Ruby gem designed to enhance and to backup your PostgreSQL database.
+
+## Usage
+
+### Configuration
+
+To configure the Ops gem, you need to set up the following environment variables:
+- `DATABASE_URL`: The URL of the PostgreSQL database to backup.
+
+Set up Active Storage for backups (e.g., Amazon S3 for production). We expect a service named `:backup_storage` to be configured.
+
+```yaml
+backup_storage:
+  service: S3
+  access_key_id: <%= ENV['AWS_ACCESS_KEY_ID'] %>
+  secret_access_key: <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
+  region: <%= ENV['AWS_REGION'] %>
+  bucket: backups
+```
+### Backup Method
+
+The `Ops::Backup` class provides a method to perform a PostgreSQL database backup:
+
+- `db_pg_backup(exclude_tables: [], tag: nil)`: This method performs a full backup of the database, optionally excluding specified tables. The backup is tagged and saved as an attachment.
+
+example:
+
+```ruby
+Ops::Backup.new.db_pg_backup(exclude_tables: ["versions"], tag: "db_pg_unveresioned")
+```
+
+### Backup Retention Strategies
+
+Ops gem provides two backup retention strategies:
+
+1. **Tiered Cleanup Policy**:
+    - Keeps all backups from the last day.
+    - Keeps the last backup of each day from the last week (except the last day).
+    - Keeps the last backup of each week from the last month (except the last week).
+    - Keeps the last backup of each month before the last month.
+
+    example:
+    ```ruby
+    Ops::CleanupTiered.new.cleanup("db_pg_unversioned")
+    ```
+
+    This policy can be applied using the `Ops::CleanupTieredJob`.
+
+2. **Limit Cleanup Policy**:
+    - Keeps the last N backups, where N is specified by the `limit` parameter.
+
+    example:
+    ```ruby
+    Ops::CleanupLimit.new.cleanup("db_pg_unversioned", limit: 14)
+    ```
+
+    This policy can be applied using the `Ops::CleanupLimitJob`.
+
+### Using SolidQueue Schedule
+
+1. Schedule the backup job using SolidQueue:
+    ```ruby
+    SolidQueue.schedule(Ops::BackupDbJob.new, cron: '0 0 * * *', args: { tag: "db_pg_full", exclude_tables: [], cleanup: "retain_last_limit" }) # This schedules the backup job to run daily at midnight
+    ```
+
+2. Schedule the cleanup job with a tiered policy:
+    ```ruby
+    SolidQueue.schedule(Ops::CleanupTieredJob.new, cron: '0 1 * * *', args: { tag: "db_pg_full" }) # This schedules the tiered cleanup job to run daily at 1 AM
+    ```
+
+3. Schedule the cleanup job with a limit policy:
+    ```ruby
+    SolidQueue.schedule(Ops::CleanupLimitJob.new, cron: '0 2 * * *', args: { tag: "db_pg_full", limit: 14 }) # This schedules the limit cleanup job to run daily at 2 AM
+    ```
+
+### Using `recurring.yml` from SolidQueue
+
+Alternatively, you can configure recurring tasks using `recurring.yml` in your SolidQueue configuration:
+
+1. Create or update the `recurring.yml` file in your application's config directory:
+    ```yaml
+    # filepath: config/recurring.yml
+    backup_db_job:
+      cron: "0 0 * * *"
+      class: "Ops::BackupDbJob"
+      args:
+        tag: "db_pg_full"
+        exclude_tables: []
+        cleanup: "retain_last_limit"
+
+    cleanup_tiered_job:
+      cron: "0 1 * * *"
+      class: "Ops::CleanupTieredJob"
+      args:
+        tag: "db_pg_full"
+
+    cleanup_limit_job:
+      cron: "0 2 * * *"
+      class: "Ops::CleanupLimitJob"
+      args:
+        tag: "db_pg_full"
+        limit: 14
+    ```
+
+
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem "ops"
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+```bash
+$ gem install ops
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/koenhandekyn/ops.
+
+## License
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/setup"
+
+APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+load "rails/tasks/statistics.rake"
+
+require "bundler/gem_tasks"

data/app/assets/stylesheets/ops/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/ops/backups/application_controller.rb

@@ -0,0 +1,6 @@
+module Ops
+  module Backups
+    class ApplicationController < ActionController::Base
+    end
+  end
+end

data/app/helpers/ops/application_helper.rb

@@ -0,0 +1,4 @@
+module Ops
+  module ApplicationHelper
+  end
+end

data/app/jobs/ops/backups/application_job.rb

@@ -0,0 +1,6 @@
+module Ops
+  module Backups
+    class ApplicationJob < ActiveJob::Base
+    end
+  end
+end

data/app/jobs/ops/backups/backup_db_job.rb

@@ -0,0 +1,15 @@
+# typed: true
+# frozen_string_literal: true
+
+class Ops::Backups::BackupDbJob < ApplicationJob
+  queue_as :operations
+
+  # perform a full backup of the database
+  # @param tag [String] the tag to assign to the backup
+  # @param exclude_tables [Array<String>] the list of tables to exclude from the backup
+  # @param cleanup_policy [String] the cleanup policy to apply to the backup, one of "retain_tiered_cleanup_policy" or "retain_last_limit_cleanup_policy"
+  def perform(tag: "db_pg_full", exclude_tables: [], cleanup: nil)
+    Ops::Backup.new.db_pg_backup(tag:, exclude_tables:)
+    Ops::Backup.send("#{cleanup}_cleanup_policy", tag: tag) if cleanup.present?
+  end
+end

data/app/jobs/ops/backups/cleanup_limit_job.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Cleanup job for limit backup policy
+class Ops::Backups::CleanupLimitJob < ApplicationJob
+  queue_as :operations
+
+  # @param [String] tag
+  # @param [Integer] limit
+  # @return [void]
+  #
+  # @example Tasks::CleanupLimit.perform_now(tag: "db_pg_full", limit: 14)
+  def perform(tag: "db_pg_full", limit: 14)
+    Ops::Backup.retain_last_limit_cleanup_policy(tag: tag, limit: limit)
+  end
+end

data/app/jobs/ops/backups/cleanup_tiered_job.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Cleanup job for tiered backup policy
+class Ops::Backups::CleanupTieredJob < ApplicationJob
+  queue_as :operations
+
+  # @param [String] tag
+  # @return [void]
+  #
+  # @example Tasks::CleanupTiered.perform_now(tag: "db_pg_full")
+  def perform(tag: "db_pg_full")
+    Ops::Backup.retain_tiered_cleanup_policy(tag: tag)
+  end
+end

data/app/models/ops/backups/application_record.rb

@@ -0,0 +1,8 @@
+module Ops
+  module Backups
+    class ApplicationRecord < ActiveRecord::Base
+      self.abstract_class = true
+      self.table_name_prefix = "ops_"
+    end
+  end
+end

data/app/models/ops/backups/backup.rb

@@ -0,0 +1,71 @@
+module Ops
+  puts "Ops module loaded"
+
+  class Ops::Backups::Backup < ApplicationRecord
+    # has_one_attached :backup_file, service: :backup_storage
+    has_one_attached :backup_file
+
+    default_scope { order(updated_at: :desc) }
+
+    def db_pg_backup(exclude_tables: [], tag: nil)
+      tag ||= exclude_tables.empty? ? "db_pg_full" : "db_pg_partial"
+      Rails.logger.info("Backing up database, skipping tables: \#{exclude_tables.join(", ")}")
+      db_url = ENV["DATABASE_URL"]
+      self.tag = tag
+      self.name = filename = "pg_\#{db_url.split('/').last}_backup_\#{Time.now.to_i}.dump"
+      save!
+
+      Tempfile.new("pgbackup") do |tempfile|
+        begin
+          excluded_tables_param = exclude_tables.map { |table| "--exclude-table-data=\#{table}" }.join(" ")
+          command = ["pg_dump", "--no-owner", excluded_tables_param, "-v", "-Fc", "-f", tempfile.path, db_url]
+
+          stdout, stderr, status = Open3.capture3(*command)
+
+          if status.success?
+            Rails.logger.info("PgBackup successful: #{stdout}")
+            backup_file.attach io: File.open(tempfile.path), filename:, content_type: "application/octet-stream"
+          else
+            error_message = "PgBackup failed: #{stderr.strip}"
+            Rails.logger.error(error_message)
+            raise "PgBackup command failed: #{error_message}"
+          end
+        rescue StandardError => e
+          Rails.logger.error("Failed to backup database: \#{e.message}")
+          raise
+        end
+      end
+    end
+  end
+
+  # Keep all the backups of the last day
+  # Keep the last backup of each day in the last week (except the last day)
+  # Keep the last backup of each week in the last month (except the last week)
+  # Keep the last backup of each month before the last month
+  def self.retain_tiered_cleanup_policy(tag: "")
+    week = where(created_at: 1.week.ago..1.day.ago)
+      .group_by { |b| b.created_at.to_date }
+    month = where(created_at: 1.month.ago..1.week.ago)
+      .group_by { |b| b.created_at.beginning_of_week }
+    older = where(created_at: 1.year.ago..1.month.ago)
+      .group_by { |b| b.created_at.beginning_of_month }
+
+    backups = week.merge(month).merge(older)
+
+    ids = backups.flat_map do |_, group|
+      group.sort_by(&:created_at).reverse.drop(1).pluck(:id)
+    end
+
+    records = where(id: ids)
+    records = records.where(tag: tag) if tag.present?
+    records.destroy_all
+  end
+
+  # Keep the last 14 backups
+  def self.retain_last_limit_cleanup_policy(limit: 14, tag: "")
+    records = all
+    records = records.where(tag: tag) if tag.present?
+    records = records.order(updated_at: :desc).offset(limit)
+    records.destroy_all
+  end
+end

data/app/views/layouts/ops/application.html.erb

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Ops</title>
+  <%= csrf_meta_tags %>
+  <%= csp_meta_tag %>
+
+  <%= yield :head %>
+
+  <%= stylesheet_link_tag "ops/application", media: "all" %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/routes.rb

@@ -0,0 +1,2 @@
+Ops::Backups::Engine.routes.draw do
+end

data/db/migrate/20241114173612_create_ops_backups.rb

@@ -0,0 +1,10 @@
+class CreateOpsBackups < ActiveRecord::Migration[8.0]
+  def change
+    create_table :ops_backups do |t|
+      t.string :name, null: false
+      t.string :tag, null: false
+
+      t.timestamps
+    end
+  end
+end

data/lib/ops/backups/backups.rb

@@ -0,0 +1,8 @@
+require "ops/backups/version"
+require "ops/backups/engine"
+
+module Ops
+  module Backups
+    # Your code goes here...
+  end
+end

data/lib/ops/backups/engine.rb

@@ -0,0 +1,7 @@
+module Ops
+  module Backups
+    class Engine < ::Rails::Engine
+      isolate_namespace Ops::Backups
+    end
+  end
+end

data/lib/ops/backups/version.rb

@@ -0,0 +1,5 @@
+module Ops
+  module Backups
+    VERSION = "0.1.0"
+  end
+end

data/lib/tasks/ops_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :ops do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: ops_backups
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Koen Handekyn
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-11-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6'
+description: This gem provides functionality to backup PostgreSQL databases to ActiveStorage
+  (S3) from within a Rails context.
+email:
+- github.com@handekyn.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/assets/stylesheets/ops/application.css
+- app/controllers/ops/backups/application_controller.rb
+- app/helpers/ops/application_helper.rb
+- app/jobs/ops/backups/application_job.rb
+- app/jobs/ops/backups/backup_db_job.rb
+- app/jobs/ops/backups/cleanup_limit_job.rb
+- app/jobs/ops/backups/cleanup_tiered_job.rb
+- app/models/ops/backups/application_record.rb
+- app/models/ops/backups/backup.rb
+- app/views/layouts/ops/application.html.erb
+- config/routes.rb
+- db/migrate/20241114173612_create_ops_backups.rb
+- lib/ops/backups/backups.rb
+- lib/ops/backups/engine.rb
+- lib/ops/backups/version.rb
+- lib/tasks/ops_tasks.rake
+homepage: https://github.com/koenhandekyn/ops-backups
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/koenhandekyn/ops-backups
+  source_code_uri: https://github.com/koenhandekyn/ops-backups/tree/main
+  changelog_uri: https://github.com/koenhandekyn/ops-backups/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: A Ruby gem for managing PostgreSQL backups.
+test_files: []