martilla 0.2.9 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e7aa68d0f99f6f614717b71f8952b505471eb1cff84a99808eb319de33daf57
-  data.tar.gz: eda97326ca0b55f78e2240ab7596eca8b8721700854930080dda9b544254cc3e
+  metadata.gz: aaf89471f6777ba461ddf9769030ef39b30a019c76e10dc35236cf8bcca03a53
+  data.tar.gz: 192c63a35a2e7d0381c45643d86defe699bcdaadc750ccbf8c782aeadf52fbbb
 SHA512:
-  metadata.gz: 45943b19f5fa5d9827dab8279fbc2dd610eb04f7e146674618ff941f94fd8ee31696243cdccaa9f8e7f7310be55c4ee135264457973c3c85c45e447fa659bb32
-  data.tar.gz: b78c7f5916337c4f8e382547a3721cd08f0b85944198ea4dd195f00a78a21cd4dc74dc343ba7d7f42a32232cbe94d47bb619dffb0fd44716f4c4a51cccde5e67
+  metadata.gz: 596ccdd75c799206a00f363ccd9c203eb93671c70296127efb197087d621e2209fa584f4ef73b1f5fe2fa9ff5d895807e926bf32764e752803eb02eadff320b3
+  data.tar.gz: dbada619ce14429929942cd842396d5aae9551bcc177e2438b48dabdc79d5623adb63d73db752e23fef0f575579412f5eac4014c0e20a26f153f02b4d8f66efc

data/.gitignore

@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/backups/
 
 # rspec failure tracking
 .rspec_status
@@ -14,3 +15,6 @@
 martilla.yml
 backup.sql
 backup.sql.gz
+
+test-database-dump.sql
+test-database-dump.sql.gz

data/.travis.yml

@@ -3,5 +3,8 @@ sudo: false
 language: ruby
 cache: bundler
 rvm:
-  - 2.6.3
+  - 2.6.5
+  - 2.5.7
+  - 2.4.9
 before_install: gem install bundler -v 2.0.2
+script: bundle exec rspec

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    martilla (0.2.9)
+    martilla (0.3.0)
       aws-sdk-s3 (~> 1.49)
       aws-sdk-ses (~> 1.26)
       memoist (~> 0.16.0)
@@ -13,8 +13,8 @@ GEM
   remote: https://rubygems.org/
   specs:
     aws-eventstream (1.0.3)
-    aws-partitions (1.231.0)
-    aws-sdk-core (3.72.0)
+    aws-partitions (1.237.0)
+    aws-sdk-core (3.76.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1, >= 1.228.0)
       aws-sigv4 (~> 1.1)
@@ -22,7 +22,7 @@ GEM
     aws-sdk-kms (1.25.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.52.0)
+    aws-sdk-s3 (1.53.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
@@ -39,7 +39,7 @@ GEM
     json (2.2.0)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
-    memoist (0.16.0)
+    memoist (0.16.1)
     method_source (0.9.2)
     mini_mime (1.0.2)
     pony (1.13.1)

data/README.md

@@ -72,10 +72,12 @@ Currently available DB types to choose from are **postgres** & **mysql**. They b
 
 ### Storages
 
-The available Storages types are 'local', 'S3' & 'SCP'. They each have different available options:
+The available Storages types are **local**, **S3**& **SCP**. They each have different available options:
  - options for type: **local**
    - `filename`
      - The location to where the backup will be stored
+   - `retention`
+     - An integer that defines the max number of backups stored at the defined location
  - options for type: **s3**
    - `filename`
      - The location to where the backup will be stored within the S3 bucket
@@ -85,51 +87,61 @@ The available Storages types are 'local', 'S3' & 'SCP'. They each have different
      - can be specified with the usual ENV variables or IAM roles
    - `secret_access_key`
      - can be specified with the usual ENV variables or IAM roles
+   - `retention`
+     - An integer that defines the max number of backups stored at the defined location
  - options for type: **scp**
    - `filename`
      - The location to where the backup will be stored within remote server
    - `host`
    - `user`
    - `identity_file`
+   - `retention`
+     - Not implemented for this storage ([see #12](https://github.com/fdoxyz/martilla/issues/12))
 
 All storage types also accept 'suffix' as a boolean that enables or disables a timestamp to be added as a suffix to the backup 'filename', it defaults as `true`.
 
 ### Notifiers
 
-The available Notifiers are 'ses', 'sendmail' & 'smtp'. They each have different available options:
-- options for type: **ses** (email notifier)
-  - `aws_region`
-  - `access_key_id`
-    - can be specified with the usual ENV variables or IAM role
-  - `secret_access_key`
-    - can be specified with the usual ENV variables or IAM roles
-- options for type: **sendmail** (email notifier)
-  - no custom options
-- options for type: **smtp** (email notifier)
-  - `address`
-  - `domain`
-  - `user_name`
-  - `password`
-- options for type: **slack**
-  - `slack_webhook_url`
-    - required
-    - more info [here](https://api.slack.com/messaging/webhooks)
-  - `slack_channel`
-    - defaults to `#general`
-    - the channel to post the backup notifications
-  - `slack_username`
-    - defaults to `Martilla`
-    - the username which backup notifications will be posted with
+The available Notifiers are **ses**, **sendmail** & **smtp**. They each have different available options:
+  - options for type: **ses** (email notifier)
+    - `aws_region`
+    - `access_key_id`
+      - can be specified with the usual ENV variables or IAM role
+    - `secret_access_key`
+      - can be specified with the usual ENV variables or IAM roles
+  - options for type: **sendmail** (email notifier)
+    - no custom options
+  - options for type: **smtp** (email notifier)
+    - `address`
+    - `domain`
+    - `user_name`
+    - `password`
+  - options for type: **slack**
+    - `slack_webhook_url`
+      - required
+      - more info [here](https://api.slack.com/messaging/webhooks)
+    - `slack_channel`
+      - defaults to `#general`
+      - the channel to post the backup notifications
+    - `slack_username`
+      - defaults to `Martilla`
+      - the username which backup notifications will be posted with
 
 All of the previous **email notifiers** also have the following options that can be customized:
- - `to`
-   - a list of comma separated emails to be notified
- - `from`
-   - defaults to 'martilla@no-reply.com'
- - `success_subject`
-   - the subject of the success email
- - `failure_subject`
-   - the subject of the failure email
+  - `to`
+    - a list of comma separated emails to be notified
+  - `from`
+    - defaults to 'martilla@no-reply.com'
+  - `success_subject`
+    - the subject of the success email
+  - `failure_subject`
+    - the subject of the failure email
+
+Also **ALL** notifiers have the following two options
+  - `send_success`
+    - `Boolean` value that will disable notifications on success when set to false. Defaults to `true`
+  - `send_failure`
+    - `Boolean` value that will disable notifications on failure when set to false. Defaults to `true`
 
 It's **HIGHLY RECOMMENDED** to test and make sure emails are being delivered correctly to each target inbox. Emails with standard messages like these automated backup notifications tend to be easily marked as spam.
 

data/lib/martilla/backup.rb

@@ -45,14 +45,15 @@ module Martilla
 
         # Persist the backup
         @storage.persist(tmp_file: tmp_file, gzip: gzip?)
+        @storage.enfore_retention!(gzip: gzip?)
       rescue Exception => e
         @notifiers.each do |notifier|
-          notifier.error(e.message, metadata)
+          notifier.error(e.message, metadata) if notifier.send_failure?
         end
         puts "An error occurred: #{e.inspect}"
       else
         @notifiers.each do |notifier|
-          notifier.success(metadata)
+          notifier.success(metadata) if notifier.send_success?
         end
         puts "Backup created and persisted successfully"
       end
@@ -101,7 +102,8 @@ module Martilla
         'storage' => {
           'type' => 'local',
           'options' => {
-            'filename' => 'database-backup.sql'
+            'filename' => 'database-backup.sql',
+            'retention' => 0
           }
         },
         'notifiers' => [

data/lib/martilla/databases/mysql.rb

@@ -14,7 +14,7 @@ module Martilla
     private
 
     def connection_arguments
-      "-u #{user} --password=#{password} -P #{port} #{db}"
+      "-u #{user} --password=#{password} --host=#{host} -P #{port} #{db}"
     end
 
     def host

data/lib/martilla/notifier.rb

@@ -1,5 +1,3 @@
-require 'forwardable'
-
 module Martilla
   class Notifier < Component
     attr_reader :options
@@ -21,6 +19,18 @@ module Martilla
       'Notifier configuration is invalid. Details here: https://github.com/fdoxyz/martilla'
     end
 
+    def send_success?
+      value = @options['send_success']
+      return true if value.nil?
+      value
+    end
+
+    def send_failure?
+      value = @options['send_failure']
+      return true if value.nil?
+      value
+    end
+
     # When a new Notifier is supported it needs to go here
     def self.create(config = {})
       case config['type'].downcase

data/lib/martilla/notifiers/ses.rb

@@ -35,9 +35,7 @@ module Martilla
       begin
         ses_client.send_email(
           destination: {
-            to_addresses: [
-              to_email.split(',')
-            ]
+            to_addresses: to_email.split(',')
           },
           message: {
             body: {

data/lib/martilla/notifiers/slack.rb

@@ -25,8 +25,9 @@ module Martilla
     end
 
     def slack_webhook_url
-      raise config_error('slack_webhook_url') if @options['slack_webhook_url'].nil?
-      @options['slack_webhook_url']
+      webhook_url = @options['slack_webhook_url'] || ENV['SLACK_WEBHOOK_URL']
+      raise config_error('slack_webhook_url') if webhook_url.nil?
+      webhook_url
     end
 
     def slack_channel

data/lib/martilla/storage.rb

@@ -13,6 +13,10 @@ module Martilla
       raise NotImplementedError, 'You must implement the persist method'
     end
 
+    def enforce_retention!
+      raise NotImplementedError, 'You must implement the enforce_retention! method'
+    end
+
     def invalid_options_msg
       'Storage configuration is invalid. Details here: https://github.com/fdoxyz/martilla'
     end
@@ -42,6 +46,14 @@ module Martilla
       "#{dirname}/#{basename}_#{timestamp}#{extension}"
     end
 
+    def retention_limit
+      @options['retention'].to_i
+    end
+
+    def timestamp_regex
+      /\d{4}-\d{2}-\d{2}T\d{6}/
+    end
+
     def config_error(config_name)
       Error.new("Storage adapter configuration requires #{config_name}. Details here: https://github.com/fdoxyz/martilla")
     end

data/lib/martilla/storages/local.rb

@@ -5,5 +5,34 @@ module Martilla
       return nil if $?.success?
       raise Error.new("Local storage failed with code #{$?.exitstatus}")
     end
+
+    def enfore_retention!(gzip:)
+      return if retention_limit < 1
+      files = backup_file_list(output_filename(gzip))
+
+      while files.count > retention_limit do
+        File.delete(files.first)
+        puts "Retention limit met. Removed the backup file: #{files.shift}"
+      end
+    end
+
+    private
+
+    # Oldest first & most recent last
+    def backup_file_list(sample_filename)
+      dirname = File.dirname(sample_filename)
+      basename = File.basename(sample_filename, '.*')
+
+      if suffix?
+        # Replaces file's basename timestamp for wildcards to match againts
+        # what exist in the directory. Ex: "backup_2019-11-10T114342" will be
+        # replaced with "backup_*-*-*T*". This means all backups will match
+        # using `Dir.glob` below
+        sections = basename.split('_').reject { |str| timestamp_regex =~ str }
+        basename = "#{sections.join('_')}_*-*-*T*"
+      end
+
+      Dir["#{dirname}/#{basename}.*"].sort_by { |f| File.mtime(f) }
+    end
   end
 end

data/lib/martilla/storages/s3.rb

@@ -11,6 +11,16 @@ module Martilla
       raise Error.new('Error uploading backup to bucket')
     end
 
+    def enfore_retention!(gzip:)
+      return if retention_limit < 1
+
+      objs = s3_resource.bucket(bucket_name).objects.sort_by(&:last_modified)
+      while objs.count > retention_limit do
+        objs.first.delete
+        puts "Retention limit met. Removed the backup file: #{objs.shift.key}"
+      end
+    end
+
     private
 
     def s3_resource

data/lib/martilla/storages/scp.rb

@@ -6,6 +6,10 @@ module Martilla
       raise Error.new("SCP storage failed with code #{$?.exitstatus}")
     end
 
+    def enfore_retention!(gzip:)
+      puts 'WARNING: Retention is not implemented for SCP storage. More details: https://github.com/fdoxyz/martilla/issues/12'
+    end
+
     private
 
     def host

data/lib/martilla/version.rb

@@ -1,3 +1,3 @@
 module Martilla
-  VERSION = '0.2.9'
+  VERSION = '0.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: martilla
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: 0.3.0
 platform: ruby
 authors:
 - Fernando Valverde
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-31 00:00:00.000000000 Z
+date: 2019-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor