stratocumulus 0.0.2 → 0.0.3

Sign up to get free protection for your applications and to get access to all the features.
@@ -3,4 +3,5 @@ require 'stratocumulus/version'
3
3
  require 'stratocumulus/database'
4
4
  require 'stratocumulus/storage'
5
5
  require 'stratocumulus/runner'
6
+ require 'stratocumulus/retention'
6
7
  require 'stratocumulus/cli'
@@ -7,8 +7,8 @@ module Stratocumulus
7
7
  @password = options['password']
8
8
  @name = options['name']
9
9
 
10
- @host = options['host'] || 'localhost'
11
- @port = options['port'] || 3306
10
+ @host = options['host']
11
+ @port = options['port']
12
12
 
13
13
  check_dependencies(options['type'])
14
14
  end
@@ -18,7 +18,7 @@ module Stratocumulus
18
18
  end
19
19
 
20
20
  def filename
21
- "#{@name}/#{@name}.#{Time.now.utc.strftime('%Y%m%d%H%M')}.sql.gz"
21
+ @filename ||= Time.now.utc.strftime("#{@name}/#{@name}.%Y%m%d%H%M.sql.gz")
22
22
  end
23
23
 
24
24
  private
@@ -27,12 +27,24 @@ module Stratocumulus
27
27
  command = 'mysqldump '
28
28
  command << '--single-transaction '
29
29
  command << "-u#{@username} "
30
- command << "-h#{@host} "
31
- command << "-P#{@port} "
30
+ command << "-h#{host} " unless socket?
31
+ command << "-P#{port} " unless socket?
32
32
  command << "-p#{@password} " if @password
33
33
  command << @name
34
34
  end
35
35
 
36
+ def host
37
+ @host || 'localhost'
38
+ end
39
+
40
+ def port
41
+ @port || 3306
42
+ end
43
+
44
+ def socket?
45
+ !@host && !@port
46
+ end
47
+
36
48
  def pipefail
37
49
  'set -o pipefail;'
38
50
  end
@@ -0,0 +1,38 @@
1
+ # encoding: UTF-8
2
+ #
3
+ module Stratocumulus
4
+ class Retention
5
+ def initialize(schedule)
6
+ @schedule = schedule
7
+ end
8
+
9
+ def rule(key)
10
+ return unless expires_in_days
11
+
12
+ {
13
+ 'ID' => key,
14
+ 'Prefix' => key,
15
+ 'Enabled' => true,
16
+ 'Days' => expires_in_days
17
+ }
18
+ end
19
+
20
+ def upload_today?
21
+ !@schedule || relevent_period
22
+ end
23
+
24
+ private
25
+
26
+ def expires_in_days
27
+ return unless @schedule
28
+
29
+ relevent_period * @schedule[relevent_period]
30
+ end
31
+
32
+ def relevent_period
33
+ @schedule.keys.sort.reverse.find do |period|
34
+ 0 == Date.today.yday % period
35
+ end
36
+ end
37
+ end
38
+ end
@@ -21,7 +21,7 @@ module Stratocumulus
21
21
  end
22
22
 
23
23
  def storage
24
- @storage ||= Storage.new(@config['s3'])
24
+ Storage.new(@config['s3'])
25
25
  end
26
26
  end
27
27
  end
@@ -9,9 +9,12 @@ module Stratocumulus
9
9
  @secret_access_key = options.fetch('secret_access_key')
10
10
  @region = options['region']
11
11
  @bucket = options.fetch('bucket')
12
+ @retention = Retention.new(options['retention'])
12
13
  end
13
14
 
14
15
  def upload(database)
16
+ return unless @retention.upload_today?
17
+ add_expiry_rule(database.filename)
15
18
  files.create(
16
19
  key: database.filename,
17
20
  body: database.dump,
@@ -32,7 +35,32 @@ module Stratocumulus
32
35
  end
33
36
 
34
37
  def files
35
- connection.directories.get(@bucket).files
38
+ @files ||= directories.get(@bucket).files
39
+ end
40
+
41
+ def directories
42
+ connection.directories
43
+ end
44
+
45
+ def add_expiry_rule(key)
46
+ new_rule = @retention.rule(key)
47
+ return unless new_rule
48
+ directories.service.put_bucket_lifecycle(
49
+ @bucket,
50
+ 'Rules' => current_rules << new_rule
51
+ )
52
+ end
53
+
54
+ def current_rules
55
+ existing_rules.select do |rule|
56
+ files.find { |file| file.key == rule['ID'] }
57
+ end
58
+ end
59
+
60
+ def existing_rules
61
+ directories.service.get_bucket_lifecycle(@bucket).data[:body]['Rules']
62
+ rescue Excon::Errors::NotFound
63
+ []
36
64
  end
37
65
  end
38
66
  end
@@ -1,4 +1,4 @@
1
1
  # encoding: UTF-8
2
2
  module Stratocumulus
3
- VERSION = '0.0.2'
3
+ VERSION = '0.0.3'
4
4
  end
@@ -30,6 +30,5 @@ RSpec.configure do |config|
30
30
 
31
31
  config.mock_with :rspec do |mocks|
32
32
  mocks.syntax = :expect
33
- mocks.verify_partial_doubles = true
34
33
  end
35
34
  end
@@ -3,6 +3,9 @@ s3:
3
3
  secret_access_key: IamTHESekret
4
4
  bucket: stratocumulus-test
5
5
  region: eu-west1
6
+ retention:
7
+ 1: 30
8
+ 30: 12
6
9
  databases:
7
10
  -
8
11
  type: mysql
@@ -115,7 +115,9 @@ describe Stratocumulus::Database do
115
115
  end
116
116
 
117
117
  describe 'host' do
118
- context 'default' do
118
+ context 'default with the port set' do
119
+ let(:config) { base_config.merge('port' => '3306') }
120
+
119
121
  it 'uses localhost' do
120
122
  expect(IO).to receive(:popen) do |command|
121
123
  expect(command).to include(' -hlocalhost ')
@@ -123,6 +125,14 @@ describe Stratocumulus::Database do
123
125
  end
124
126
  end
125
127
 
128
+ context 'default' do
129
+ it 'uses the default socket' do
130
+ expect(IO).to receive(:popen) do |command|
131
+ expect(command).to_not include(' -hlocalhost ')
132
+ end
133
+ end
134
+ end
135
+
126
136
  context 'setting the host' do
127
137
  let(:config) { base_config.merge('host' => 'db.awesome-server.net') }
128
138
 
@@ -135,7 +145,9 @@ describe Stratocumulus::Database do
135
145
  end
136
146
 
137
147
  describe 'port' do
138
- context 'default' do
148
+ context 'default with the host set' do
149
+ let(:config) { base_config.merge('host' => 'db.awesome-server.net') }
150
+
139
151
  it 'uses 3306' do
140
152
  expect(IO).to receive(:popen) do |command|
141
153
  expect(command).to include(' -P3306 ')
@@ -143,6 +155,14 @@ describe Stratocumulus::Database do
143
155
  end
144
156
  end
145
157
 
158
+ context 'default' do
159
+ it 'uses the default socket' do
160
+ expect(IO).to receive(:popen) do |command|
161
+ expect(command).to_not include(' -P3306 ')
162
+ end
163
+ end
164
+ end
165
+
146
166
  context 'setting the port' do
147
167
  let(:config) { base_config.merge('port' => '4306') }
148
168
 
@@ -181,5 +201,11 @@ describe Stratocumulus::Database do
181
201
  filename = "stratocumulus_test/stratocumulus_test.#{timestamp}.sql.gz"
182
202
  expect(subject.filename).to eq filename
183
203
  end
204
+
205
+ it 'stays the same for an instance even if time moves on' do
206
+ filename = subject.filename
207
+ allow(Time).to receive(:now).and_return(Time.now + 60 * 60 * 26)
208
+ expect(subject.filename).to eq filename
209
+ end
184
210
  end
185
211
  end
@@ -0,0 +1,110 @@
1
+ # encoding: UTF-8
2
+ require 'spec_helper'
3
+
4
+ describe Stratocumulus::Retention do
5
+ subject { described_class.new(schedule) }
6
+
7
+ let(:schedule) do
8
+ {
9
+ 1 => 30,
10
+ 7 => 8,
11
+ 30 => 12
12
+ }
13
+ end
14
+
15
+ let(:keep_for_a_month) do
16
+ (1..365).to_a - keep_for_two_months - keep_for_a_year
17
+ end
18
+
19
+ let(:keep_for_two_months) do
20
+ (1..56).map { |i| i * 7 } - keep_for_a_year
21
+ end
22
+
23
+ let(:keep_for_a_year) do
24
+ (1..12).map { |i| i * 30 }
25
+ end
26
+
27
+ let(:key) { 'testdb/testdbTIMESTAMP.sql.gz' }
28
+
29
+ let(:expires_in_days) { subject.rule(key)['Days'] }
30
+
31
+ describe '#rule' do
32
+
33
+ it 'returns a rule with the key and an expiry' do
34
+ stub_yday(30)
35
+ expect(subject.rule(key)).to eq(
36
+ 'Days' => 360,
37
+ 'Enabled' => true,
38
+ 'ID' => key,
39
+ 'Prefix' => key
40
+ )
41
+ end
42
+
43
+ it 'it keeps a daily backup for a month' do
44
+ keep_for_a_month.each do |day|
45
+ stub_yday(day)
46
+ expect(expires_in_days).to eq 30
47
+ end
48
+ end
49
+
50
+ it 'keeps a weekly backup for 8 weeks' do
51
+ keep_for_two_months.each do |day|
52
+ stub_yday(day)
53
+ expect(expires_in_days).to eq 56
54
+ end
55
+ end
56
+
57
+ it 'keeps a monthly backup for 12 months' do
58
+ keep_for_a_year.each do |day|
59
+ stub_yday(day)
60
+ expect(expires_in_days).to eq 360
61
+ end
62
+ end
63
+
64
+ context 'a non daily schedule' do
65
+ let(:schedule) do
66
+ { 14 => 2 }
67
+ end
68
+
69
+ let(:days_to_keep) do
70
+ (1..28).map { |i| i * 14 }
71
+ end
72
+
73
+ let(:days_not_to_keep) do
74
+ (1..365).to_a - days_to_keep
75
+ end
76
+
77
+ it 'keeps fortightly backups for 2 fortnights' do
78
+ days_to_keep.each do |day|
79
+ stub_yday(day)
80
+ expect(subject.upload_today?).to be_truthy
81
+ expect(expires_in_days).to eq 28
82
+ end
83
+ end
84
+
85
+ it 'keeps nothing on the other days' do
86
+ days_not_to_keep.each do |day|
87
+ stub_yday(day)
88
+ expect(subject.upload_today?).to be_falsy
89
+ end
90
+ end
91
+ end
92
+
93
+ context 'with no schedule' do
94
+ let(:schedule) { nil }
95
+
96
+ it 'runs the backup' do
97
+ expect(subject.upload_today?).to eq true
98
+ end
99
+
100
+ it 'keeps the backup forever' do
101
+ expect(subject.rule(key)).to be_nil
102
+ end
103
+ end
104
+
105
+ def stub_yday(day)
106
+ day = double(yday: day)
107
+ allow(Date).to receive(:today).and_return(day)
108
+ end
109
+ end
110
+ end
@@ -16,11 +16,12 @@ describe Stratocumulus::Runner do
16
16
  end
17
17
 
18
18
  it 'passes the correct config to Storage' do
19
- expect(Stratocumulus::Storage).to receive(:new).once.with(
19
+ expect(Stratocumulus::Storage).to receive(:new).twice.with(
20
20
  'access_key_id' => 'I_AM_THE_KEY_ID',
21
21
  'secret_access_key' => 'IamTHESekret',
22
22
  'bucket' => 'stratocumulus-test',
23
- 'region' => 'eu-west1'
23
+ 'region' => 'eu-west1',
24
+ 'retention' => { 1 => 30, 30 => 12 }
24
25
  ).and_return(storage)
25
26
  end
26
27
 
@@ -2,7 +2,7 @@
2
2
  require 'spec_helper'
3
3
 
4
4
  describe Stratocumulus::Storage do
5
- let(:config) do
5
+ let(:base_config) do
6
6
  {
7
7
  'access_key_id' => 'IM_A_ID',
8
8
  'secret_access_key' => 'IM_A_SEKRET_KEY',
@@ -11,6 +11,8 @@ describe Stratocumulus::Storage do
11
11
  }
12
12
  end
13
13
 
14
+ let(:config) { base_config }
15
+
14
16
  subject { described_class.new(config) }
15
17
 
16
18
  describe '#upload' do
@@ -54,6 +56,100 @@ describe Stratocumulus::Storage do
54
56
  ).and_return(connection)
55
57
  end
56
58
  end
57
- end
58
59
 
60
+ context 'with a schedule' do
61
+ let(:config) do
62
+ base_config.merge(
63
+ 'retention' => {
64
+ 1 => 30
65
+ }
66
+ )
67
+ end
68
+
69
+ let(:directories) do
70
+ double(:fog_dirs, get: double(files: files), service: service)
71
+ end
72
+
73
+ let(:service) { double(:fog_service) }
74
+
75
+ context 'no rules set on the bucket yet' do
76
+ before do
77
+ allow(service).to receive(:get_bucket_lifecycle)
78
+ .with('stratocumulus-test')
79
+ .and_raise(Excon::Errors::NotFound, '404 No rules set yet')
80
+ end
81
+
82
+ it 'puts a rule for the uploaded file' do
83
+ expect(service).to receive(:put_bucket_lifecycle)
84
+ .with(
85
+ 'stratocumulus-test',
86
+ 'Rules' => [
87
+ {
88
+ 'ID' => 'foo.sql.gz',
89
+ 'Prefix' => 'foo.sql.gz',
90
+ 'Enabled' => true,
91
+ 'Days' => 30
92
+ }
93
+ ]
94
+ )
95
+ end
96
+ end
97
+
98
+ context 'rules allready set on the bucket' do
99
+ let(:files) { [existing_file] }
100
+ let(:existing_file) { double(:fog_file, key: 'bar.sql.gz') }
101
+ let(:existing_rules) do
102
+ [
103
+ {
104
+ 'ID' => 'bar.sql.gz',
105
+ 'Prefix' => 'bar.sql.gz',
106
+ 'Enabled' => true,
107
+ 'Days' => 30
108
+ },
109
+ {
110
+ 'ID' => 'baz.sql.gz',
111
+ 'Prefix' => 'baz.sql.gz',
112
+ 'Enabled' => true,
113
+ 'Days' => 30
114
+ }
115
+ ]
116
+ end
117
+
118
+ before do
119
+ allow(service).to receive(:get_bucket_lifecycle)
120
+ .with('stratocumulus-test')
121
+ .and_return(
122
+ double(
123
+ data: {
124
+ body: {
125
+ 'Rules' => existing_rules
126
+ }
127
+ }
128
+ )
129
+ )
130
+ allow(files).to receive(:create).and_return(:true)
131
+ end
132
+
133
+ it 'adds the rule to the rules for existing files' do
134
+ expect(service).to receive(:put_bucket_lifecycle).with(
135
+ 'stratocumulus-test',
136
+ 'Rules' => [
137
+ {
138
+ 'ID' => 'bar.sql.gz',
139
+ 'Prefix' => 'bar.sql.gz',
140
+ 'Enabled' => true,
141
+ 'Days' => 30
142
+ },
143
+ {
144
+ 'ID' => 'foo.sql.gz',
145
+ 'Prefix' => 'foo.sql.gz',
146
+ 'Enabled' => true,
147
+ 'Days' => 30
148
+ }
149
+ ]
150
+ )
151
+ end
152
+ end
153
+ end
154
+ end
59
155
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: stratocumulus
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-06-17 00:00:00.000000000 Z
12
+ date: 2014-06-18 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fog
@@ -161,6 +161,7 @@ files:
161
161
  - lib/stratocumulus/cli.rb
162
162
  - lib/stratocumulus/database.rb
163
163
  - lib/stratocumulus/ext/io.rb
164
+ - lib/stratocumulus/retention.rb
164
165
  - lib/stratocumulus/runner.rb
165
166
  - lib/stratocumulus/storage.rb
166
167
  - lib/stratocumulus/version.rb
@@ -171,6 +172,7 @@ files:
171
172
  - spec/support/test_config_file.yml
172
173
  - spec/unit/cli_spec.rb
173
174
  - spec/unit/database_spec.rb
175
+ - spec/unit/retention_spec.rb
174
176
  - spec/unit/runner_spec.rb
175
177
  - spec/unit/storage_spec.rb
176
178
  - stratocumulus.gemspec
@@ -189,7 +191,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
189
191
  version: '0'
190
192
  segments:
191
193
  - 0
192
- hash: -3829672677026677809
194
+ hash: -4542051147236532735
193
195
  required_rubygems_version: !ruby/object:Gem::Requirement
194
196
  none: false
195
197
  requirements:
@@ -198,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
198
200
  version: '0'
199
201
  segments:
200
202
  - 0
201
- hash: -3829672677026677809
203
+ hash: -4542051147236532735
202
204
  requirements: []
203
205
  rubyforge_project:
204
206
  rubygems_version: 1.8.25
@@ -213,5 +215,6 @@ test_files:
213
215
  - spec/support/test_config_file.yml
214
216
  - spec/unit/cli_spec.rb
215
217
  - spec/unit/database_spec.rb
218
+ - spec/unit/retention_spec.rb
216
219
  - spec/unit/runner_spec.rb
217
220
  - spec/unit/storage_spec.rb