noop-backup 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 478d0c1b90aca1c432382ecc87e6ee858f6bc5c47fbb563c22952add1c804c50
-  data.tar.gz: 3965b585c105b4a84410d5e76ecaeaaab1f0f9ffcce1ff820cac044c351e0d78
+  metadata.gz: bba6625ab079fd41615d6ff304e7517743a67d2cb394a6644e058f3d93a8dd0e
+  data.tar.gz: 194c11d8d1e0cdb069f05a20b06342a1d421ae4178d3d918090a9f6fc182722a
 SHA512:
-  metadata.gz: 2ed6ed19b704c0ecb39be159bbedf3579a3efe546940cb03d6a1b5c7bc2a536858cb12c6e1dd5606f349e9af6e62decf1771a492be0d8f0644ca2f622029fc97
-  data.tar.gz: 5e32a0bb63c92a0900fa34a2bfeeaa2f529f549cf8483ba7f189a55e36b31bbf23128a226d6c5a1b93ea12d544cc55843e6c56a8be04f75c310280b0f9528fe0
+  metadata.gz: 201fdd9c78813f8f20d1e042d842dc76f2bfff07f94e71ce0ef926e6e606f549b7889075cae371602f33e6d095a1c854ce762daf21de7cbd7bee1a7bd32a730a
+  data.tar.gz: 313b0980120d3330cd6a37959813cd3fbd44f037369eab05fdff7c3a18dd8d06cd1207aaeeaac1c83eea9adb76ee1ed46303f5a1425392e7a7de3f34cfc65a82

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [0.2.0] - 2026-07-04
+
+- Support multiple destination stores in a single backup run
+- Delete partial uploads and fail the job safely when a dump fails
+- Validate configuration before running, with clearer errors
+- A lot of undocumented API and configuration changes
+
 ## [0.1.3] - 2026-07-02
 
 - Pass `access_key_id` and `secret_access_key` in config block

data/README.md

@@ -2,6 +2,8 @@
 
 Ruby gem for backing up PostgreSQL databases with minimum effort.
 
+This gem is very much __unstable__. Expect APIs to change a lot before v1.0.0 is released.
+
 ## Installation
 
 If using bundler, add the gem to your `Gemfile`:
@@ -24,11 +26,24 @@ The gem requires `pg_dump` to be installed on the machine that is running it.
 
 #### Rails 8 with Solid Queue
 
-(soon) NoopBackup will run automatically using the Solid Queue scheduler.
+Add the NoopBackup job to the `config/recurring.yml` file:
+
+```yaml
+production:
+  # other jobs
+  noop_backup:
+    class: NoopBackup::BackupJob
+    schedule: at 3am every day
+```
+
+The job will run on the `:default` queue on your desired schedule.
+If you wish to customize the job, create a new one and make sure to invoke
+`NoopBackup::Commands::Backup.execute` in the `perform` method.
+
 
 #### Sidekiq, Good Job, whenever
 
-(soon)
+Create a new recurring job and have it invoke `NoopBackup::Commands::Backup.execute`
 
 ### Manually
 
@@ -36,33 +51,23 @@ The gem requires `pg_dump` to be installed on the machine that is running it.
 bundle exec nbu backup
 ```
 
-This command will dump the database and stream it to S3 without writing anything to disk. Use any
-scheduler or even cron to run it periodically.
+This command will dump the database and stream it to your configured destinations without writing
+anything to disk. Use any scheduler or even cron to run it periodically.
 
 ## Configuration
 
-The gem needs AWS credentials and a bucket name:
-
-```
-AWS_REGION=region
-AWS_ACCESS_KEY_ID=your-key
-AWS_SECRET_ACCESS_KEY=your-secret-key
-NBU_BUCKET=s3-bucket
-NBU_MIN_SIZE=2048
-```
-
-If your app already has the AWS SDK set up, only the bucket needs to be configured.
-
-Alternatively, the gem can be configured with an initializer:
+All configuration options can be edited in the initializer:
 
 ```rb
 # config/initializers/noop-backup.rb
 
 NoopBackup.configure do |config|
-  config.bucket = 'bucket-name'
-  config.region = 'eu-central-1'
-  config.prefix = Rails.env
-  config.min_size = 2048
+  config.register(:s3) do |store|
+    store.bucket = Settings.aws.bucket
+    store.region = Settings.aws.region
+    store.access_key_id = Settings.aws.access_key_id
+    store.secret_access_key = Settings.aws.secret_access_key
+  end
 
   config.notifier :slack do |slack|
     slack.webhook_url = 'https://hooks.slack.com/services/whatever'
@@ -83,11 +88,13 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/gmitre
 ## Wishlist
 
 - [ ] S3-compatible backends
-- [ ] file backends
+- [ ] file backend
 - [ ] email notifier
 - [ ] SMS notifier
 - [ ] restore command
 - [ ] test command
+- [ ] encryption
+- [ ] delete stale
 
 ## License
 

data/lib/noop_backup/cli.rb

@@ -7,9 +7,10 @@ module NoopBackup
     desc "backup", "Create and store a new backup"
     def backup
       NoopBackup.prepare!
-      NoopBackup::Commands::Backup.execute
+      result = NoopBackup::Commands::Backup.execute
+      exit 1 unless result.success?
     rescue => e
-      warn e.message
+      warn e
       exit 1
     end
   end

data/lib/noop_backup/commands/backup.rb

@@ -1,101 +1,135 @@
-require "aws-sdk-s3"
 require "open3"
 
 module NoopBackup::Commands
+  CommandResult = Struct.new(:error, :store_results, keyword_init: true) do
+    def success?
+      status == :success
+    end
+
+    def status
+      return :error if error
+      return :error if store_results.empty? || store_results.none?(&:success)
+      return :success if store_results.all?(&:success)
+
+      :partial_success
+    end
+
+    def report
+      if error
+        NoopBackup.notify "❌ Fatal error: #{error.message}"
+      else
+        store_results.each do |result|
+          NoopBackup.notify result.message
+        end
+      end
+    end
+  end
+
   class Backup
-    def self.execute
-      new.execute
+    def self.execute(report: NoopBackup.config.report?)
+      result = new.execute
+
+      result.report if report
+
+      result
     end
 
     def initialize
-      now = Time.now.utc
-      @key = [
-        config.prefix,
-        config.pg_env["PGDATABASE"],
-        now.strftime("%Y"),
-        now.strftime("%m"),
-        "#{now.strftime("%d-%H%M%S")}.dump"
-      ].compact.join("/")
-      @bytes = 0
-      @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @key = generate_key
+      @store_results = []
+      @sinks = []
     end
 
     def execute
-      commands = [
-        [config.pg_env, "pg_dump", "--format=custom", "--no-owner"]
-      ]
+      perform_sanity_check!
+
+      commands = [config.dump_command]
 
-      commands << ["pv", "-btra"] if system("which", "pv", out: File::NULL, err: File::NULL)
+      # Pipe pg_dump through pv if installed for a basic progress report
+      commands << ["pv", "-btra"] if config.report? && system("which", "pv", out: File::NULL, err: File::NULL)
 
       Open3.pipeline_r(*commands) do |last_stdout, wait_threads|
-        upload(last_stdout)
+        @sinks = config.stores.map do |store|
+          reader, writer = IO.pipe(binmode: true)
+
+          thread = Thread.new do
+            store.backup!(@key, reader)
+          rescue => e
+            # *always* return a Result, even if unexpected. Add store name for debugging.
+            NoopBackup::Stores::Result.new(success: false, error: e, store: store.class.name, key: @key)
+          ensure
+            reader.close
+          end
+
+          NoopBackup::Tee::Sink.new(store:, writer:, thread:)
+        end
 
-        raise "pipeline failed" unless wait_threads.all? { |t| t.value.success? }
-      end
+        sinks_fanout = NoopBackup::Tee.new(@sinks)
 
-      duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started
+        begin
+          IO.copy_stream(last_stdout, sinks_fanout)
+        rescue => e
+          raise NoopBackup::DumpFailedError, "streaming failed: #{e.message}"
+        ensure
+          @sinks.each(&:close)
+        end
 
-      if config.min_size && @bytes.to_i < config.min_size.to_i
-        raise "backup too small: #{human_size(@bytes)} < min_size (#{human_size(config.min_size)})"
+        @store_results = @sinks.map(&:collect)
+
+        raise NoopBackup::DumpFailedError, "pipeline failed" unless wait_threads.all? { |t| t.value.success? }
       end
 
-      config.notify(success_message(duration))
-    rescue => e
-      remove_partial_upload
+      CommandResult.new(store_results: @store_results)
+    rescue NoopBackup::DumpFailedError => error
+      # The dump stream itself failed, so uploads that finished are truncated copies of a bad stream — delete them.
+      # Collect the results first: if copy_stream raised, the collection line above never ran. Sinks are already
+      # closed on every DumpFailedError path, so collect can't block.
+      @store_results = @sinks.map(&:collect)
 
-      config.notify("❌ Backup failed: #{e.message}")
+      cleanup_uploaded_stores!
 
-      raise
+      CommandResult.new(error:, store_results: @store_results)
+    rescue => error
+      CommandResult.new(error:, store_results: @store_results)
     end
 
     private
 
-    def config
-      @config ||= NoopBackup.configuration
-    end
+    # 1. Check if any stores are registered.
+    # 2. Make sure **all** stores have a valid configuration
+    def perform_sanity_check!
+      raise NoopBackup::ConfigurationError, "No backup stores registered" if config.stores.empty?
+      raise NoopBackup::ConfigurationError, "Could not resolve PGDATABASE" if config.pg_env["PGDATABASE"].to_s.empty?
 
-    def s3_client
-      @_s3_client ||= Aws::S3::Client.new(**config.s3_config)
+      config.stores.each(&:validate!)
     end
 
-    # Prefer Aws::S3::TransferManager for streaming uploads if available.
-    # Aws::S3::Resource.upload_stream is deprecated in newer versions
-    def upload(stdout)
-      if defined?(Aws::S3::TransferManager)
-        manager = Aws::S3::TransferManager.new(client: s3_client)
-
-        manager.upload_stream(bucket: config.bucket, key: @key, part_size: 8 * 1024 * 1024, thread_count: 2) do |s3_stream|
-          @bytes = IO.copy_stream(stdout, s3_stream)
-        end
-      else
-        object = Aws::S3::Resource.new(region: config.region).bucket(config.bucket).object(@key)
-
-        object.upload_stream(part_size: 8 * 1024 * 1024, thread_count: 2) do |s3_stream|
-          @bytes = IO.copy_stream(stdout, s3_stream)
-        end
+    def cleanup_uploaded_stores!
+      @sinks.each do |sink|
+        sink.store.cleanup!(@key) if sink.collect&.success
+      rescue => e
+        NoopBackup.notify "⚠️ Cleanup failed for #{sink.store.class}: #{e.message}"
       end
     end
 
-    def human_size(bytes)
-      units = %w[B KB MB GB TB]
-      size = bytes.to_f
-      i = 0
-      while size >= 1024 && i < units.size - 1
-        size /= 1024
-        i += 1
-      end
-      format("%.1f %s", size, units[i])
+    def config
+      @config ||= NoopBackup.config
     end
 
-    def success_message(duration)
-      "✅ #{config.pg_env["PGDATABASE"]} backed up successfully — " \
-        "#{human_size(@bytes)} in #{duration.round(1)}s → #{config.bucket}/#{@key}"
-    end
+    # File name of the current backup. Example:
+    #
+    #  prefix   db_name            y    m  d  h  m  s  ms
+    # /database/db_name_production/2026/07/03-14-47-23-724.dump
+    def generate_key
+      now = Time.now.utc
 
-    def remove_partial_upload
-      s3_client.delete_object(bucket: config.bucket, key: @key)
-    rescue => e
-      warn "Failed to clean up partial upload #{@key}: #{e.message}"
+      [
+        config.prefix,
+        config.pg_env["PGDATABASE"],
+        now.strftime("%Y"),
+        now.strftime("%m"),
+        "#{now.strftime("%d-%H-%M-%S-%L")}.dump"
+      ].compact.join("/")
     end
   end
 end

data/lib/noop_backup/configuration.rb

@@ -1,22 +1,44 @@
 module NoopBackup
   class Configuration
-    # NOTE: These are all going to change a lot before version 1.0.0
-    attr_accessor :bucket,
-      :region,
-      :access_key_id,
-      :secret_access_key,
-      :prefix,
+    attr_accessor :prefix,
+      :min_size,
       :pg_host,
       :pg_port,
       :pg_user,
       :pg_password,
-      :pg_database,
-      :min_size
+      :pg_database
+
+    attr_reader :stores, :notifiers
+    attr_writer :report, :dump_command
 
     def initialize
-      @prefix = "database"
+      @prefix = ENV.fetch("NBU_PREFIX", "database")
+      @stores = []
+      @min_size = ENV.fetch("NBU_MIN_SIZE", 2048).to_i
       @notifiers = [NoopBackup::Notifiers::Stdout.new]
-      @min_size = ENV.fetch("NBU_MIN_SIZE", 1024).to_i
+      @report = true
+    end
+
+    def report?
+      @report
+    end
+
+    def dump_command
+      @dump_command || [pg_env, "pg_dump", "--format=custom", "--no-owner"]
+    end
+
+    def register(store_type)
+      raise NoopBackup::ConfigurationError, "`config.register` requires a block" unless block_given?
+
+      store =
+        case store_type.to_sym
+        when :s3 then NoopBackup::Stores::S3.new
+        else raise NoopBackup::ConfigurationError, "unknown store type: #{store_type}"
+        end
+
+      @stores << store
+
+      yield store
     end
 
     def notifier(type)
@@ -31,27 +53,13 @@ module NoopBackup
       end
     end
 
-    def notify(message)
-      @notifiers.each do |notifier|
-        notifier.notify(message)
-      end
-    end
-
     def pg_env
       {
-        "PGHOST" => pg_host || db_config[:host]&.to_s || "localhost",
-        "PGPORT" => pg_port || db_config[:port]&.to_s,
-        "PGUSER" => pg_user || db_config[:username]&.to_s,
-        "PGPASSWORD" => pg_password || db_config[:password]&.to_s,
-        "PGDATABASE" => pg_database || db_config[:database].to_s
-      }.compact
-    end
-
-    def s3_config
-      {
-        region: region,
-        access_key_id: access_key_id,
-        secret_access_key: secret_access_key
+        "PGHOST" => (pg_host || db_config[:host])&.to_s,
+        "PGPORT" => (pg_port || db_config[:port])&.to_s,
+        "PGUSER" => (pg_user || db_config[:username])&.to_s,
+        "PGPASSWORD" => (pg_password || db_config[:password])&.to_s,
+        "PGDATABASE" => (pg_database || db_config[:database])&.to_s
       }.compact
     end
 
@@ -59,9 +67,5 @@ module NoopBackup
       @db_config ||= defined?(::ActiveRecord) ?
         ::ActiveRecord::Base.connection_db_config.configuration_hash : {}
     end
-
-    # TODO: raise if config invalid (e.g. bucket missing)
-    def validate!
-    end
   end
 end

data/lib/noop_backup/jobs/backup_job.rb

@@ -1,9 +1,11 @@
 module NoopBackup
   class BackupJob < ActiveJob::Base
-    queue_as :backups
+    queue_as :default
 
     def perform
-      NoopBackup::Commands::Backup.execute
+      result = NoopBackup::Commands::Backup.execute
+
+      raise NoopBackup::BackupFailedError.new(result) unless result.success?
     end
   end
 end

data/lib/noop_backup/stores/result.rb

@@ -0,0 +1,11 @@
+module NoopBackup::Stores
+  Result = Struct.new(:success, :error, :store, :key, :bytes, :duration, keyword_init: true) do
+    def message
+      if success
+        "✅ [#{store}] Backup successful — #{NoopBackup.utils.human_size(bytes)} in #{duration.round(1)}s → /#{key}"
+      else
+        "❌ [#{store}] Backup failed: #{error.message}"
+      end
+    end
+  end
+end

data/lib/noop_backup/stores/s3.rb

@@ -0,0 +1,82 @@
+require "aws-sdk-s3"
+
+module NoopBackup::Stores
+  class S3 < Store
+    attr_accessor :bucket, :region, :access_key_id, :secret_access_key
+
+    def initialize
+      @bucket = ENV["AWS_S3_BUCKET"]
+      @region = ENV["AWS_REGION"]
+      @access_key_id = ENV["AWS_ACCESS_KEY_ID"]
+      @secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
+    end
+
+    # Prefer Aws::S3::TransferManager for streaming uploads if available.
+    # Aws::S3::Resource.upload_stream is deprecated in newer versions
+    def backup!(key, stream)
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      bytes = 0
+
+      validate!
+
+      upload_attempted = true
+
+      if defined?(Aws::S3::TransferManager)
+        manager = Aws::S3::TransferManager.new(client: s3_client)
+
+        manager.upload_stream(bucket:, key: key, part_size: 8 * 1024 * 1024, thread_count: 2) do |s3_stream|
+          bytes = IO.copy_stream(stream, s3_stream)
+        end
+      else
+        object = Aws::S3::Resource.new(client: s3_client).bucket(bucket).object(key)
+
+        object.upload_stream(part_size: 8 * 1024 * 1024, thread_count: 2) do |s3_stream|
+          bytes = IO.copy_stream(stream, s3_stream)
+        end
+      end
+
+      duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+
+      if config.min_size && bytes < config.min_size.to_i
+        raise NoopBackup::DumpTooSmallError, "backup too small: #{NoopBackup.utils.human_size(bytes)} < min_size (#{NoopBackup.utils.human_size(config.min_size)})"
+      end
+
+      Result.new(success: true, store: :s3, bytes:, key:, duration:)
+    rescue => e
+      duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+
+      cleanup!(key) if upload_attempted
+
+      Result.new(success: false, error: e, store: :s3, bytes:, key:, duration:)
+    end
+
+    def validate!
+      raise NoopBackup::ConfigurationError, "bucket is not configured" if bucket.to_s.empty?
+
+      if access_key_id.to_s.empty? != secret_access_key.to_s.empty?
+        raise NoopBackup::ConfigurationError,
+          "access_key_id and secret_access_key must both be set, or both left blank to use the default AWS credential chain"
+      end
+    end
+
+    def cleanup!(key)
+      s3_client.delete_object(bucket:, key: key)
+    rescue => e
+      warn "Failed to clean up partial upload #{key}: #{e.message}"
+    end
+
+    private
+
+    def s3_client
+      @_s3_client ||= Aws::S3::Client.new(**s3_config)
+    end
+
+    def s3_config
+      {
+        region: region,
+        access_key_id: access_key_id,
+        secret_access_key: secret_access_key
+      }.compact
+    end
+  end
+end

data/lib/noop_backup/stores/store.rb

@@ -0,0 +1,9 @@
+module NoopBackup::Stores
+  class Store
+    private
+
+    def config
+      @config ||= NoopBackup.config
+    end
+  end
+end

data/lib/noop_backup/stores.rb

@@ -0,0 +1,3 @@
+require_relative "stores/store"
+require_relative "stores/result"
+require_relative "stores/s3" # TODO: load conditionally

data/lib/noop_backup/tee.rb

@@ -0,0 +1,33 @@
+module NoopBackup
+  class Tee
+    Sink = Struct.new(:store, :writer, :thread, keyword_init: true) do
+      def write(chunk)
+        return if @error
+
+        writer.write(chunk)
+      rescue Errno::EPIPE, IOError => e
+        @error = e
+
+        close unless writer.closed?
+      end
+
+      def close
+        writer.close
+      end
+
+      def collect
+        thread.value
+      rescue
+        nil
+      end
+    end
+
+    def initialize(sinks) = @sinks = sinks
+
+    def write(chunk)
+      @sinks.each { |sink| sink.write(chunk) }
+
+      chunk.bytesize
+    end
+  end
+end

data/lib/noop_backup/utils.rb

@@ -0,0 +1,14 @@
+module NoopBackup::Utils
+  extend self
+
+  def human_size(bytes)
+    units = %w[B KB MB GB TB]
+    size = bytes.to_f
+    i = 0
+    while size >= 1024 && i < units.size - 1
+      size /= 1024
+      i += 1
+    end
+    format("%.1f %s", size, units[i])
+  end
+end

data/lib/noop_backup/version.rb

@@ -1,3 +1,3 @@
 module NoopBackup
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end

data/lib/noop_backup.rb

@@ -1,5 +1,8 @@
 require_relative "noop_backup/version"
 require_relative "noop_backup/configuration"
+require_relative "noop_backup/tee"
+require_relative "noop_backup/utils"
+require_relative "noop_backup/stores"
 require_relative "noop_backup/commands/backup"
 require_relative "noop_backup/notifiers/slack"
 require_relative "noop_backup/notifiers/stdout"
@@ -7,6 +10,26 @@ require_relative "noop_backup/notifiers/stdout"
 module NoopBackup
   class Error < StandardError; end
 
+  class DumpTooSmallError < Error; end
+  class DumpFailedError < Error; end
+  class ConfigurationError < Error; end
+
+  class BackupFailedError < Error
+    attr_reader :result
+
+    def initialize(result)
+      @result = result
+
+      super("Backup failed (#{result.status}): #{result.error&.message || failed_stores(result)}")
+    end
+
+    private
+
+    def failed_stores(result)
+      result.store_results.reject(&:success).map(&:store).join(", ")
+    end
+  end
+
   class << self
     def configuration
       @configuration ||= Configuration.new
@@ -14,6 +37,10 @@ module NoopBackup
 
     alias_method :config, :configuration
 
+    def reset!
+      @configuration = nil
+    end
+
     def configure
       yield(configuration)
     end
@@ -24,6 +51,16 @@ module NoopBackup
 
       require env_file if File.exist?(env_file)
     end
+
+    def notify(message)
+      config.notifiers.each do |notifier|
+        notifier.notify(message)
+      end
+    end
+
+    def utils
+      NoopBackup::Utils
+    end
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: noop-backup
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Georgi Mitrev
@@ -60,6 +60,12 @@ files:
 - lib/noop_backup/notifiers/slack.rb
 - lib/noop_backup/notifiers/stdout.rb
 - lib/noop_backup/plugins/rails.rb
+- lib/noop_backup/stores.rb
+- lib/noop_backup/stores/result.rb
+- lib/noop_backup/stores/s3.rb
+- lib/noop_backup/stores/store.rb
+- lib/noop_backup/tee.rb
+- lib/noop_backup/utils.rb
 - lib/noop_backup/version.rb
 homepage: https://github.com/gmitrev/noop-backup
 licenses: