stratocumulus 0.0.2 → 0.0.3

data/lib/stratocumulus.rb

@@ -3,4 +3,5 @@ require 'stratocumulus/version'
 require 'stratocumulus/database'
 require 'stratocumulus/storage'
 require 'stratocumulus/runner'
+require 'stratocumulus/retention'
 require 'stratocumulus/cli'

data/lib/stratocumulus/database.rb

@@ -7,8 +7,8 @@ module Stratocumulus
       @password = options['password']
       @name = options['name']
 
-      @host = options['host'] || 'localhost'
-      @port = options['port'] || 3306
+      @host = options['host']
+      @port = options['port']
 
       check_dependencies(options['type'])
     end
@@ -18,7 +18,7 @@ module Stratocumulus
     end
 
     def filename
-      "#{@name}/#{@name}.#{Time.now.utc.strftime('%Y%m%d%H%M')}.sql.gz"
+      @filename ||= Time.now.utc.strftime("#{@name}/#{@name}.%Y%m%d%H%M.sql.gz")
     end
 
     private
@@ -27,12 +27,24 @@ module Stratocumulus
       command = 'mysqldump '
       command << '--single-transaction '
       command << "-u#{@username} "
-      command << "-h#{@host} "
-      command << "-P#{@port} "
+      command << "-h#{host} " unless socket?
+      command << "-P#{port} " unless socket?
       command << "-p#{@password} " if @password
       command << @name
     end
 
+    def host
+      @host || 'localhost'
+    end
+
+    def port
+      @port || 3306
+    end
+
+    def socket?
+      !@host && !@port
+    end
+
     def pipefail
       'set -o pipefail;'
     end

data/lib/stratocumulus/retention.rb

@@ -0,0 +1,38 @@
+# encoding: UTF-8
+#
+module Stratocumulus
+  class Retention
+    def initialize(schedule)
+      @schedule = schedule
+    end
+
+    def rule(key)
+      return unless expires_in_days
+
+      {
+        'ID' => key,
+        'Prefix' => key,
+        'Enabled' => true,
+        'Days' => expires_in_days
+      }
+    end
+
+    def upload_today?
+      !@schedule || relevent_period
+    end
+
+    private
+
+    def expires_in_days
+      return unless @schedule
+
+      relevent_period * @schedule[relevent_period]
+    end
+
+    def relevent_period
+      @schedule.keys.sort.reverse.find do |period|
+        0 == Date.today.yday % period
+      end
+    end
+  end
+end

data/lib/stratocumulus/runner.rb

@@ -21,7 +21,7 @@ module Stratocumulus
     end
 
     def storage
-      @storage ||= Storage.new(@config['s3'])
+      Storage.new(@config['s3'])
     end
   end
 end

data/lib/stratocumulus/storage.rb

@@ -9,9 +9,12 @@ module Stratocumulus
       @secret_access_key = options.fetch('secret_access_key')
       @region = options['region']
       @bucket = options.fetch('bucket')
+      @retention = Retention.new(options['retention'])
     end
 
     def upload(database)
+      return unless @retention.upload_today?
+      add_expiry_rule(database.filename)
       files.create(
         key: database.filename,
         body: database.dump,
@@ -32,7 +35,32 @@ module Stratocumulus
     end
 
     def files
-      connection.directories.get(@bucket).files
+      @files ||= directories.get(@bucket).files
+    end
+
+    def directories
+      connection.directories
+    end
+
+    def add_expiry_rule(key)
+      new_rule = @retention.rule(key)
+      return unless new_rule
+      directories.service.put_bucket_lifecycle(
+        @bucket,
+        'Rules' => current_rules << new_rule
+      )
+    end
+
+    def current_rules
+      existing_rules.select do |rule|
+        files.find { |file| file.key == rule['ID'] }
+      end
+    end
+
+    def existing_rules
+      directories.service.get_bucket_lifecycle(@bucket).data[:body]['Rules']
+    rescue Excon::Errors::NotFound
+      []
     end
   end
 end

data/lib/stratocumulus/version.rb

@@ -1,4 +1,4 @@
 # encoding: UTF-8
 module Stratocumulus
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

data/spec/spec_helper.rb

@@ -30,6 +30,5 @@ RSpec.configure do |config|
 
   config.mock_with :rspec do |mocks|
     mocks.syntax = :expect
-    mocks.verify_partial_doubles = true
   end
 end

data/spec/support/test_config_file.yml

@@ -3,6 +3,9 @@ s3:
   secret_access_key: IamTHESekret
   bucket: stratocumulus-test 
   region: eu-west1
+  retention:
+    1: 30
+    30: 12
 databases:
   -
     type: mysql

data/spec/unit/database_spec.rb

@@ -115,7 +115,9 @@ describe Stratocumulus::Database do
     end
 
     describe 'host' do
-      context 'default' do
+      context 'default with the port set' do
+        let(:config) { base_config.merge('port' => '3306') }
+
         it 'uses localhost' do
           expect(IO).to receive(:popen) do |command|
             expect(command).to include(' -hlocalhost ')
@@ -123,6 +125,14 @@ describe Stratocumulus::Database do
         end
       end
 
+      context 'default' do
+        it 'uses the default socket' do
+          expect(IO).to receive(:popen) do |command|
+            expect(command).to_not include(' -hlocalhost ')
+          end
+        end
+      end
+
       context 'setting the host' do
         let(:config) { base_config.merge('host' => 'db.awesome-server.net') }
 
@@ -135,7 +145,9 @@ describe Stratocumulus::Database do
     end
 
     describe 'port' do
-      context 'default' do
+      context 'default with the host set' do
+        let(:config) { base_config.merge('host' => 'db.awesome-server.net') }
+
         it 'uses 3306' do
           expect(IO).to receive(:popen) do |command|
             expect(command).to include(' -P3306 ')
@@ -143,6 +155,14 @@ describe Stratocumulus::Database do
         end
       end
 
+      context 'default' do
+        it 'uses the default socket' do
+          expect(IO).to receive(:popen) do |command|
+            expect(command).to_not include(' -P3306 ')
+          end
+        end
+      end
+
       context 'setting the port' do
         let(:config) { base_config.merge('port' => '4306') }
 
@@ -181,5 +201,11 @@ describe Stratocumulus::Database do
       filename = "stratocumulus_test/stratocumulus_test.#{timestamp}.sql.gz"
       expect(subject.filename).to eq filename
     end
+
+    it 'stays the same for an instance even if time moves on' do
+      filename = subject.filename
+      allow(Time).to receive(:now).and_return(Time.now + 60 * 60 * 26)
+      expect(subject.filename).to eq filename
+    end
   end
 end

data/spec/unit/retention_spec.rb

@@ -0,0 +1,110 @@
+# encoding: UTF-8
+require 'spec_helper'
+
+describe Stratocumulus::Retention do
+  subject { described_class.new(schedule) }
+
+  let(:schedule) do
+    {
+      1 => 30,
+      7 => 8,
+      30 => 12
+    }
+  end
+
+  let(:keep_for_a_month) do
+    (1..365).to_a - keep_for_two_months - keep_for_a_year
+  end
+
+  let(:keep_for_two_months) do
+    (1..56).map { |i| i * 7 } - keep_for_a_year
+  end
+
+  let(:keep_for_a_year) do
+    (1..12).map { |i| i * 30 }
+  end
+
+  let(:key) { 'testdb/testdbTIMESTAMP.sql.gz' }
+
+  let(:expires_in_days) { subject.rule(key)['Days'] }
+
+  describe '#rule' do
+
+    it 'returns a rule with the key and an expiry' do
+      stub_yday(30)
+      expect(subject.rule(key)).to eq(
+        'Days' => 360,
+        'Enabled' => true,
+        'ID' => key,
+        'Prefix' => key
+      )
+    end
+
+    it 'it keeps a daily backup for a month' do
+      keep_for_a_month.each do |day|
+        stub_yday(day)
+        expect(expires_in_days).to eq 30
+      end
+    end
+
+    it 'keeps a weekly backup for 8 weeks' do
+      keep_for_two_months.each do |day|
+        stub_yday(day)
+        expect(expires_in_days).to eq 56
+      end
+    end
+
+    it 'keeps a monthly backup for 12 months' do
+      keep_for_a_year.each do |day|
+        stub_yday(day)
+        expect(expires_in_days).to eq 360
+      end
+    end
+
+    context 'a non daily schedule' do
+      let(:schedule) do
+        { 14 => 2 }
+      end
+
+      let(:days_to_keep) do
+        (1..28).map { |i| i * 14 }
+      end
+
+      let(:days_not_to_keep) do
+        (1..365).to_a - days_to_keep
+      end
+
+      it 'keeps fortightly backups for 2 fortnights' do
+        days_to_keep.each do |day|
+          stub_yday(day)
+          expect(subject.upload_today?).to be_truthy
+          expect(expires_in_days).to eq 28
+        end
+      end
+
+      it 'keeps nothing on the other days' do
+        days_not_to_keep.each do |day|
+          stub_yday(day)
+          expect(subject.upload_today?).to be_falsy
+        end
+      end
+    end
+
+    context 'with no schedule' do
+      let(:schedule) { nil }
+
+      it 'runs the backup' do
+        expect(subject.upload_today?).to eq true
+      end
+
+      it 'keeps the backup forever' do
+        expect(subject.rule(key)).to be_nil
+      end
+    end
+
+    def stub_yday(day)
+      day = double(yday: day)
+      allow(Date).to receive(:today).and_return(day)
+    end
+  end
+end

data/spec/unit/runner_spec.rb

@@ -16,11 +16,12 @@ describe Stratocumulus::Runner do
   end
 
   it 'passes the correct config to Storage' do
-    expect(Stratocumulus::Storage).to receive(:new).once.with(
+    expect(Stratocumulus::Storage).to receive(:new).twice.with(
       'access_key_id' => 'I_AM_THE_KEY_ID',
       'secret_access_key' => 'IamTHESekret',
       'bucket' => 'stratocumulus-test',
-      'region' => 'eu-west1'
+      'region' => 'eu-west1',
+      'retention' => { 1 => 30, 30 => 12 }
     ).and_return(storage)
   end
 

data/spec/unit/storage_spec.rb

@@ -2,7 +2,7 @@
 require 'spec_helper'
 
 describe Stratocumulus::Storage do
-  let(:config) do
+  let(:base_config) do
     {
       'access_key_id' => 'IM_A_ID',
       'secret_access_key' => 'IM_A_SEKRET_KEY',
@@ -11,6 +11,8 @@ describe Stratocumulus::Storage do
     }
   end
 
+  let(:config) { base_config }
+
   subject { described_class.new(config) }
 
   describe '#upload' do
@@ -54,6 +56,100 @@ describe Stratocumulus::Storage do
         ).and_return(connection)
       end
     end
-  end
 
+    context 'with a schedule' do
+      let(:config) do
+        base_config.merge(
+          'retention' => {
+            1 => 30
+          }
+        )
+      end
+
+      let(:directories) do
+        double(:fog_dirs, get: double(files: files), service: service)
+      end
+
+      let(:service) { double(:fog_service) }
+
+      context 'no rules set on the bucket yet' do
+        before do
+          allow(service).to receive(:get_bucket_lifecycle)
+            .with('stratocumulus-test')
+            .and_raise(Excon::Errors::NotFound, '404 No rules set yet')
+        end
+
+        it 'puts a rule for the uploaded file' do
+          expect(service).to receive(:put_bucket_lifecycle)
+            .with(
+              'stratocumulus-test',
+              'Rules' => [
+                {
+                  'ID' => 'foo.sql.gz',
+                  'Prefix' => 'foo.sql.gz',
+                  'Enabled' => true,
+                  'Days' => 30
+                }
+              ]
+            )
+        end
+      end
+
+      context 'rules allready set on the bucket' do
+        let(:files) { [existing_file] }
+        let(:existing_file) { double(:fog_file, key: 'bar.sql.gz') }
+        let(:existing_rules) do
+          [
+            {
+              'ID' => 'bar.sql.gz',
+              'Prefix' => 'bar.sql.gz',
+              'Enabled' => true,
+              'Days' => 30
+            },
+            {
+              'ID' => 'baz.sql.gz',
+              'Prefix' => 'baz.sql.gz',
+              'Enabled' => true,
+              'Days' => 30
+            }
+          ]
+        end
+
+        before do
+          allow(service).to receive(:get_bucket_lifecycle)
+            .with('stratocumulus-test')
+            .and_return(
+              double(
+                data: {
+                  body: {
+                    'Rules' => existing_rules
+                  }
+                }
+              )
+            )
+          allow(files).to receive(:create).and_return(:true)
+        end
+
+        it 'adds the rule to the rules for existing files' do
+          expect(service).to receive(:put_bucket_lifecycle).with(
+            'stratocumulus-test',
+            'Rules' => [
+              {
+                'ID' => 'bar.sql.gz',
+                'Prefix' => 'bar.sql.gz',
+                'Enabled' => true,
+                'Days' => 30
+              },
+              {
+                'ID' => 'foo.sql.gz',
+                'Prefix' => 'foo.sql.gz',
+                'Enabled' => true,
+                'Days' => 30
+              }
+            ]
+          )
+        end
+      end
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stratocumulus
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-17 00:00:00.000000000 Z
+date: 2014-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -161,6 +161,7 @@ files:
 - lib/stratocumulus/cli.rb
 - lib/stratocumulus/database.rb
 - lib/stratocumulus/ext/io.rb
+- lib/stratocumulus/retention.rb
 - lib/stratocumulus/runner.rb
 - lib/stratocumulus/storage.rb
 - lib/stratocumulus/version.rb
@@ -171,6 +172,7 @@ files:
 - spec/support/test_config_file.yml
 - spec/unit/cli_spec.rb
 - spec/unit/database_spec.rb
+- spec/unit/retention_spec.rb
 - spec/unit/runner_spec.rb
 - spec/unit/storage_spec.rb
 - stratocumulus.gemspec
@@ -189,7 +191,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3829672677026677809
+      hash: -4542051147236532735
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -198,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3829672677026677809
+      hash: -4542051147236532735
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25
@@ -213,5 +215,6 @@ test_files:
 - spec/support/test_config_file.yml
 - spec/unit/cli_spec.rb
 - spec/unit/database_spec.rb
+- spec/unit/retention_spec.rb
 - spec/unit/runner_spec.rb
 - spec/unit/storage_spec.rb