martilla 0.0.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1298cbc9a2e46470a324413da4f9e0c478d810f166c9088a8a512c1351d634a0
-  data.tar.gz: 04ff757ae6fd6bca7b61803315f4938c9788edce5b3418d1a347529dd670b626
+  metadata.gz: 14bd7c780f6612c6adfbd67e932bf30db058e6af7148a7889d18ab0249e4d768
+  data.tar.gz: a4cd222e53ac9eefb6fdef35bbe91ff3a2b16f49ddbfefe221a8d5ab244ae2e7
 SHA512:
-  metadata.gz: ce4b0564bd205c4d3dc51170f5f1715d67516a3676fbeca4d1baa72288300c1acab2c40c5f535c3cdb3b92042bcc0080b7995d66c8065fdb63e3fef942afb520
-  data.tar.gz: 5f6f19da462b301af362e175642075edb456412eedbaae3154f62f9225e7f436b679d4b1c56ac98a40c564d9727a354fd14cb3446eb4803dbf045621796e5a21
+  metadata.gz: 9c6fa4a885951c8a07c88cf62973fa82c63e889b2a4b63080a21888f9a4d8363fdcec841b65b4117af32f8821cccd62798ee1b36271c32a3cf8a6f7d09694a82
+  data.tar.gz: 612b5bf6139d30935919e2989f9a87b2ebc58cc9f874b9e9a1b3b993f29e2069f1e13001d12696a563c0962cac17b6cba8f36c6b8bf57b404908bab9469adfaf

data/.gitignore

@@ -9,3 +9,8 @@
 
 # rspec failure tracking
 .rspec_status
+.byebug_history
+
+martilla.yml
+backup.sql
+backup.sql.gz

data/Gemfile.lock

@@ -1,29 +1,54 @@
 PATH
   remote: .
   specs:
-    martilla (0.0.1)
-      aws-ses (~> 0.6.0)
+    martilla (0.2.0)
+      aws-sdk-s3 (~> 1.49)
+      aws-sdk-ses (~> 1.26)
+      memoist (~> 0.16.0)
       pony (~> 1.13)
       thor (~> 0.20.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-ses (0.6.0)
-      builder
-      mail (> 2.2.5)
-      mime-types
-      xml-simple
-    builder (3.2.3)
+    aws-eventstream (1.0.3)
+    aws-partitions (1.226.0)
+    aws-sdk-core (3.69.1)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-kms (1.24.0)
+      aws-sdk-core (~> 3, >= 3.61.1)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-s3 (1.50.0)
+      aws-sdk-core (~> 3, >= 3.61.1)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-ses (1.26.0)
+      aws-sdk-core (~> 3, >= 3.61.1)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.1.0)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+    byebug (11.0.1)
+    coderay (1.1.2)
     diff-lcs (1.3)
+    docile (1.3.2)
+    jmespath (1.4.0)
+    json (2.2.0)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
-    mime-types (3.3)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    memoist (0.16.0)
+    method_source (0.9.2)
     mini_mime (1.0.2)
     pony (1.13.1)
       mail (>= 2.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    pry-byebug (3.7.0)
+      byebug (~> 11.0)
+      pry (~> 0.10)
     rake (13.0.0)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -38,17 +63,24 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.0)
+    simplecov (0.17.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
     thor (0.20.3)
-    xml-simple (1.1.5)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 2.0)
+  byebug (~> 11.0)
   martilla!
+  pry-byebug (~> 3.7)
   rake (~> 13.0)
   rspec (~> 3.9)
+  simplecov (~> 0.17.1)
 
 BUNDLED WITH
    2.0.2

data/README.md

@@ -1,36 +1,145 @@
 # Martilla
 
-This is the first commit at the backup tool named Martilla. More to come soon.
+Martilla is a tool to automate your backups. With simple but flexible configuration options you can have a database backup configured to run (using cron jobs or similar). Receive a notification whenever a backup fails, choose multiple ways of getting notified (i.e. email + slack).
+
+The name Martilla comes from a local name for the [Kinkajou](https://en.wikipedia.org/wiki/Kinkajou). This nocturnal animal is goes fairly unnoticed, just like we hope database backups should remain.
 
 ## Installation
 
-Add this line to your application's Gemfile:
+To use as a CLI tool
+
+    $ gem install martilla
+
+Or add this line to your application's Gemfile:
 
 ```ruby
 gem 'martilla'
 ```
 
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install martilla
-
 ## Usage
 
-TODO: Write usage instructions here
+Martilla uses a YAML configuration file that specifies the backup to be performed. The gem works by making three main concepts work together, they're listed out with details that should generally be specified in the config file:
+
+ - Database
+   - What database are we going to backup
+   - How can we connect to the database
+ - Storage
+   - Where is this backup going to be stored
+   - Credentials needed to persist the backup
+ - Notifiers
+   - How will you get notified of the backup result
+   - Can be a list of multiple ways to get notified
+
+Execute `martilla setup backup-config.yml` and you'll have your first (default) config file that looks like the following:
+
+```yaml
+---
+db:
+  type: postgres
+  options:
+    host: localhost
+    user: username
+    password: password
+    db: databasename
+storage:
+  type: local
+  options:
+    filename: database-backup.sql
+notifiers:
+- type: none
+```
+
+From here on you pick the building blocks that work for your specific case:
+
+### Databases
+
+Currently available DB types to choose from are 'postgres' & 'mysql'. They both have the same available options:
+ - `host`
+   - defaults to localhost
+   - can be set in ENV variable `PG_HOST` or `MYSQL_HOST`
+ - `user`
+   - required
+   - can be set in ENV variable `PG_USER` or `MYSQL_USER`
+ - `password`
+   - required
+   - can be set in ENV variable `PG_PASSWORD` or `MYSQL_PASSWORD`
+ - `db`
+   - required
+   - can be set in ENV variable `PG_DATABASE` or `MYSQL_DATABASE`
+ - `port`
+   - defaults to 5432 or 3306
+   - can be set in ENV variable `PG_USER` or `MYSQL_USER`
+
+### Storages
+
+The available Storages types are 'local', 'S3' & 'SCP'. They each have different available options:
+ - options for type: 'local'
+   - `filename`
+     - The location to where the backup will be stored
+ - options for type: 's3'
+   - `filename`
+     - The location to where the backup will be stored within the S3 bucket
+   - `bucket`
+   - `region`
+   - `access_key_id`
+     - can be specified with the usual ENV variables or IAM roles
+   - `secret_access_key`
+     - can be specified with the usual ENV variables or IAM roles
+ - options for type: 'scp'
+   - `filename`
+     - The location to where the backup will be stored within remote server
+   - `host`
+   - `user`
+   - `identity_file`
+
+All storage types also accept 'suffix' as a boolean that enables or disables a timestamp to be added as a suffix to the backup 'filename', it defaults as `true`.
+
+### Notifiers
+
+The available Notifiers are 'ses', 'sendmail' & 'smtp'. They each have different available options:
+- options for type: 'ses' (email notifier)
+  - `aws_region`
+  - `access_key_id`
+    - can be specified with the usual ENV variables or IAM role
+  - `secret_access_key`
+    - can be specified with the usual ENV variables or IAM roles
+- options for type: 'sendmail' (email notifier)
+  - no custom options
+- options for type: 'smtp' (email notifier)
+  - `address`
+  - `domain`
+  - `user_name`
+  - `password`
+
+All of the previous email notifiers also have the following options they all use:
+ - `to`
+   - a list of comma separated emails to be notified
+ - `from`
+   - defaults to 'martilla@no-reply.com'
+ - `success_subject`
+   - the subject of the success email
+ - `failure_subject`
+   - the subject of the failure email
+
+It's **HIGHLY RECOMMENDED** to test and make sure emails are being delivered correctly to each target inbox. Emails with standard messages like these automated backup notifications tend to be easily marked as spam.
+
+### Perform a backup
+
+As simple as running the `backup` command on the martilla CLI and passing as argument the configuration file you want to use
+
+    $ martilla backup backup-config.yml
+
+Help the help command help you
+
+    $ martilla help
 
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
-
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/martilla. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/fdoxyz/martilla. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## License
 
@@ -38,4 +147,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Martilla project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/martilla/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Martilla project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/fdoxyz/martilla/blob/master/CODE_OF_CONDUCT.md).

data/config.yml

@@ -0,0 +1,14 @@
+---
+db:
+  type: postgres
+  options:
+    host: localhost
+    user: username
+    password: password
+    db: databasename
+storage:
+  type: local
+  options:
+    filename: database-backup.sql
+notifiers:
+- type: none

data/lib/martilla/backup.rb

@@ -0,0 +1,115 @@
+require 'memoist'
+require 'martilla/utilities'
+
+module Martilla
+  class Backup
+    extend Memoist
+
+    include Utilities
+
+    attr_reader :file_size
+
+    def initialize(config)
+      @options = config['options']
+      @db = Database.create(config['db'])
+      @storage = Storage.create(config['storage'])
+      @notifiers = config['notifiers'].map { |c| Notifier.create(c) }.compact
+    end
+
+    def self.create(config)
+      backup = Backup.new(config)
+      backup.execute
+      backup
+    end
+
+    def gzip?
+      return true if @options['gzip'].nil?
+      @options['gzip']
+    end
+
+    def tmp_file
+      filename = @options.dig('tmp_file') || '/tmp/backup'
+      return "#{filename}.gz" if gzip?
+      filename
+    end
+
+    def execute
+      begin
+        # Perform DB dump
+        @ticks = [Time.now]
+        @db.dump(tmp_file: tmp_file, gzip: gzip?)
+
+        # Metadata capture
+        @file_size = File.size(tmp_file).to_f if File.exist?(tmp_file)
+        @ticks << Time.now
+
+        # Persist the backup
+        @storage.persist(tmp_file: tmp_file, gzip: gzip?)
+      rescue Exception => e
+        @notifiers.each do |notifier|
+          notifier.error(e.message, metadata)
+        end
+        puts "An error occurred: #{e.inspect}"
+      else
+        @notifiers.each do |notifier|
+          notifier.success(metadata)
+        end
+        puts "Backup created and persisted successfully"
+      end
+
+      File.delete(tmp_file) if File.exist?(tmp_file)
+    end
+
+    def metadata
+      @ticks << Time.now
+      data = []
+
+      # Total backup size
+      if @file_size.nil?
+        data << "No backup file was created"
+      else
+        data << "Total backup size: #{formatted_file_size}"
+      end
+
+      # Backup time measurements
+      if @ticks.count >= 2
+        time_diff = duration_format(@ticks[1] - @ticks[0])
+        data << "Backup 'dump' duration: #{time_diff}"
+      end
+
+      # Storage time measurements
+      if @ticks.count >= 3
+        time_diff = duration_format(@ticks[2] - @ticks[1])
+        data << "Backup storage duration: #{time_diff}"
+      end
+
+      data
+    end
+    memoize :metadata
+
+    def self.sample_config
+      {
+        'db' => {
+          'type' => 'postgres',
+          'options' => {
+            'host' => 'localhost',
+            'user' => 'username',
+            'password' => 'password',
+            'db' => 'databasename'
+          }
+        },
+        'storage' => {
+          'type' => 'local',
+          'options' => {
+            'filename' => 'database-backup.sql'
+          }
+        },
+        'notifiers' => [
+          {
+            'type' => 'none'
+          }
+        ]
+      }
+    end
+  end
+end

data/lib/martilla/cli.rb

@@ -1,3 +1,4 @@
+require 'byebug'
 require 'yaml'
 require 'thor'
 
@@ -15,13 +16,14 @@ module Martilla
       rescue Errno::ENOENT
         puts "Couldn't access non-existent file #{file_path}"
       else
-        execute_backup(backup_config)
+        backup = Backup.create(backup_config)
       end
     end
 
     desc "setup FILEPATH", "Generates a sample backup config file at FILEPATH"
-    def setup(filepath = './martilla.yml')
-      puts filepath
+    def setup(filename = 'martilla.yml')
+      file_path = File.join(Dir.pwd, filename)
+      File.write(file_path, Backup.sample_config.to_yaml)
     end
   end
 end

data/lib/martilla/database.rb

@@ -2,20 +2,31 @@ require 'forwardable'
 
 module Martilla
   class Database
-    extend Forwardable
+    attr_reader :options
 
     def initialize(config)
-      # When a new core target is added to the project include it here
+      @options = config
+      raise Error.new(invalid_options_msg) if @options.nil?
+    end
+
+    def dump(temp_file:, gzip:)
+      raise NotImplementedError, 'You must implement the dump method'
+    end
+
+    def invalid_options_msg
+      'DB configuration is invalid. Details here: https://github.com/fdoxyz/martilla'
+    end
+
+    # When a new DB is supported it needs to go here
+    def self.create(config = {})
       case config['type'].downcase
       when 'postgres'
-        @database = Databases::Postgres.new(config['options'])
+        Postgres.new(config['options'])
       when 'mysql'
-        @database = Databases::Mysql.new(config['options'])
+        Mysql.new(config['options'])
       else
         raise Error.new("Invalid Database type: #{config['type']}")
       end
     end
-
-    def_delegators :@database, :dump
   end
 end

data/lib/martilla/databases/mysql.rb

@@ -1,5 +1,40 @@
-class Martilla::Databases::Mysql < Martilla::Databases::Base
-  def dump
+module Martilla
+  class Mysql < Database
+    def dump(tmp_file:, gzip:)
+      if gzip
+        `mysqldump #{connection_arguments} | gzip -c > #{tmp_file}`
+      else
+        `mysqldump #{connection_arguments} > #{tmp_file}`
+      end
 
+      return if $?.success?
+      raise Error.new("Database dump failed with code #{$?.exitstatus}")
+    end
+
+    private
+
+    def connection_arguments
+      "-u #{user} -p #{password} -P #{port} #{db}"
+    end
+
+    def host
+      @options['host'] || ENV['MYSQL_HOST'] || 'localhost'
+    end
+
+    def port
+      @options['port'] || ENV['MYSQL_PORT'] || '3306'
+    end
+
+    def user
+      @options['user'] || ENV['MYSQL_USER']
+    end
+
+    def password
+      @options['password'] || ENV['MYSQL_PASSWORD']
+    end
+
+    def db
+      @options['db'] || ENV['MYSQL_DATABASE']
+    end
   end
 end

data/lib/martilla/databases/postgres.rb

@@ -1,4 +1,40 @@
-class Martilla::Databases::Postgres < Martilla::Databases::Base
-  def dump
+module Martilla
+  class Postgres < Database
+    def dump(tmp_file:, gzip:)
+      if gzip
+        `pg_dump #{connection_string} | gzip -c > #{tmp_file}`
+      else
+        `pg_dump #{connection_string} > #{tmp_file}`
+      end
+
+      return if $?.success?
+      raise Error.new("Database dump failed with code #{$?.exitstatus}")
+    end
+
+    private
+
+    def connection_string
+      "postgres://#{user}:#{password}@#{host}:#{port}/#{db}"
+    end
+
+    def host
+      @options['host'] || ENV['PG_HOST'] || 'localhost'
+    end
+
+    def port
+      @options['port'] || ENV['PG_PORT'] || '5432'
+    end
+
+    def user
+      @options['user'] || ENV['PG_USER']
+    end
+
+    def password
+      @options['password'] || ENV['PG_PASSWORD']
+    end
+
+    def db
+      @options['db'] || ENV['PG_DATABASE']
+    end
   end
 end

data/lib/martilla/notifier.rb

@@ -2,22 +2,41 @@ require 'forwardable'
 
 module Martilla
   class Notifier
-    extend Forwardable
+    attr_reader :options
 
     def initialize(config)
-      # When a new core target is added to the project include it here
+      @options = config
+      raise Error.new(invalid_options_msg) if @options.nil?
+    end
+
+    def success(data)
+      raise NotImplementedError, 'You must implement the success method'
+    end
+
+    def error(msg, data)
+      raise NotImplementedError, 'You must implement the error method'
+    end
+
+    def invalid_options_msg
+      'Notifier configuration is invalid. Details here: https://github.com/fdoxyz/martilla'
+    end
+
+    # When a new Notifier is supported it needs to go here
+    def self.create(config = {})
       case config['type'].downcase
-      when 'email'
-        @notifier = Notifiers::Email.new(config['options'])
+      when 'sendmail'
+        Sendmail.new(config['options'])
+      when 'smtp'
+        Smtp.new(config['options'])
       when 'ses'
-        @notifier = Notifiers::Ses.new(config['options'])
+        Ses.new(config['options'])
       when 'slack'
-        @notifier = Notifiers::Slack.new(config['options'])
+        Slack.new(config['options'])
+      when 'none'
+        nil
       else
         raise Error.new("Invalid Notifier type: #{config['type']}")
       end
     end
-
-    def_delegators :@notifier, :success, :error
   end
 end

data/lib/martilla/notifiers/email_notifier.rb

@@ -0,0 +1,65 @@
+require 'pony'
+
+module Martilla
+  class EmailNotifier < Notifier
+    private
+
+    def success_html(data)
+      data_list = data.map { |d| "<li>#{d}</li>" }.join("\n")
+      <<~HTML
+        <h2>The backup was created successfully</h2>
+        <ul>
+          #{data_list}
+        </ul>
+      HTML
+    end
+
+    def success_txt(data)
+      data_list = data.map { |d| "- #{d}" }.join("\n")
+      <<~TXT
+        The backup was created successfully
+
+        #{data_list}
+      TXT
+    end
+
+    def error_html(msg, data)
+      data_list = data.map { |d| "<li>#{d}</li>" }.join("\n")
+      <<~HTML
+        <h2>The backup attempt failed with the following error</h2>
+        <p><strong>#{msg}</strong></p>
+        <ul>
+          #{data_list}
+        </ul>
+      HTML
+    end
+
+    def error_txt(msg, data)
+      data_list = data.map { |d| "- #{d}" }.join("\n")
+      <<~TXT
+        The backup attempt failed with the following error:
+        #{msg}
+
+        #{data_list}
+      TXT
+    end
+
+    def to_email
+      email = @options['to']
+      raise config_error('to') if email.nil?
+      email
+    end
+
+    def from_email
+      @options['from'] || 'martilla@no-reply.com'
+    end
+
+    def success_subject
+      @options['success_subject'] || '[SUCCESS] Backup created'
+    end
+
+    def failure_subject
+      @options['failure_subject'] || '[FAILURE] Backup failed'
+    end
+  end
+end