stratocumulus 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.travis.yml +8 -13
- data/Gemfile +1 -1
- data/README.md +3 -2
- data/Rakefile +5 -5
- data/bin/stratocumulus +1 -1
- data/lib/stratocumulus.rb +6 -6
- data/lib/stratocumulus/cli.rb +3 -3
- data/lib/stratocumulus/database.rb +18 -20
- data/lib/stratocumulus/database/mysql.rb +6 -6
- data/lib/stratocumulus/database/pipe_io.rb +1 -1
- data/lib/stratocumulus/database/postgresql.rb +7 -7
- data/lib/stratocumulus/retention.rb +4 -4
- data/lib/stratocumulus/runner.rb +4 -4
- data/lib/stratocumulus/storage.rb +20 -15
- data/lib/stratocumulus/version.rb +1 -1
- data/spec/intergration/database_spec.rb +17 -60
- data/spec/spec_helper.rb +9 -9
- data/spec/unit/cli_spec.rb +5 -5
- data/spec/unit/database_spec.rb +38 -41
- data/spec/unit/mysql_spec.rb +33 -33
- data/spec/unit/pipe_io_spec.rb +7 -7
- data/spec/unit/postgresql_spec.rb +33 -33
- data/spec/unit/retention_spec.rb +19 -20
- data/spec/unit/runner_spec.rb +22 -22
- data/spec/unit/storage_spec.rb +96 -77
- data/stratocumulus.gemspec +19 -19
- metadata +32 -58
- data/.coveralls.yml +0 -1
- data/lib/stratocumulus/database/rethinkdb.rb +0 -39
data/spec/unit/retention_spec.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# encoding: UTF-8
|
2
|
-
require
|
2
|
+
require "spec_helper"
|
3
3
|
|
4
4
|
describe Stratocumulus::Retention do
|
5
5
|
subject { described_class.new(schedule) }
|
@@ -8,7 +8,7 @@ describe Stratocumulus::Retention do
|
|
8
8
|
{
|
9
9
|
1 => 30,
|
10
10
|
7 => 8,
|
11
|
-
30 => 12
|
11
|
+
30 => 12,
|
12
12
|
}
|
13
13
|
end
|
14
14
|
|
@@ -24,44 +24,43 @@ describe Stratocumulus::Retention do
|
|
24
24
|
(1..12).map { |i| i * 30 }
|
25
25
|
end
|
26
26
|
|
27
|
-
let(:key) {
|
27
|
+
let(:key) { "testdb/testdbTIMESTAMP.sql.gz" }
|
28
28
|
|
29
|
-
let(:expires_in_days) { subject.rule(key)[
|
29
|
+
let(:expires_in_days) { subject.rule(key)["Days"] }
|
30
30
|
|
31
|
-
describe
|
32
|
-
|
33
|
-
it 'returns a rule with the key and an expiry' do
|
31
|
+
describe "#rule" do
|
32
|
+
it "returns a rule with the key and an expiry" do
|
34
33
|
stub_yday(30)
|
35
34
|
expect(subject.rule(key)).to eq(
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
35
|
+
"Days" => 360,
|
36
|
+
"Enabled" => true,
|
37
|
+
"ID" => key,
|
38
|
+
"Prefix" => key,
|
40
39
|
)
|
41
40
|
end
|
42
41
|
|
43
|
-
it
|
42
|
+
it "it keeps a daily backup for a month" do
|
44
43
|
keep_for_a_month.each do |day|
|
45
44
|
stub_yday(day)
|
46
45
|
expect(expires_in_days).to eq 30
|
47
46
|
end
|
48
47
|
end
|
49
48
|
|
50
|
-
it
|
49
|
+
it "keeps a weekly backup for 8 weeks" do
|
51
50
|
keep_for_two_months.each do |day|
|
52
51
|
stub_yday(day)
|
53
52
|
expect(expires_in_days).to eq 56
|
54
53
|
end
|
55
54
|
end
|
56
55
|
|
57
|
-
it
|
56
|
+
it "keeps a monthly backup for 12 months" do
|
58
57
|
keep_for_a_year.each do |day|
|
59
58
|
stub_yday(day)
|
60
59
|
expect(expires_in_days).to eq 360
|
61
60
|
end
|
62
61
|
end
|
63
62
|
|
64
|
-
context
|
63
|
+
context "a non daily schedule" do
|
65
64
|
let(:schedule) do
|
66
65
|
{ 14 => 2 }
|
67
66
|
end
|
@@ -74,7 +73,7 @@ describe Stratocumulus::Retention do
|
|
74
73
|
(1..365).to_a - days_to_keep
|
75
74
|
end
|
76
75
|
|
77
|
-
it
|
76
|
+
it "keeps fortightly backups for 2 fortnights" do
|
78
77
|
days_to_keep.each do |day|
|
79
78
|
stub_yday(day)
|
80
79
|
expect(subject.upload_today?).to be_truthy
|
@@ -82,7 +81,7 @@ describe Stratocumulus::Retention do
|
|
82
81
|
end
|
83
82
|
end
|
84
83
|
|
85
|
-
it
|
84
|
+
it "keeps nothing on the other days" do
|
86
85
|
days_not_to_keep.each do |day|
|
87
86
|
stub_yday(day)
|
88
87
|
expect(subject.upload_today?).to be_falsy
|
@@ -90,14 +89,14 @@ describe Stratocumulus::Retention do
|
|
90
89
|
end
|
91
90
|
end
|
92
91
|
|
93
|
-
context
|
92
|
+
context "with no schedule" do
|
94
93
|
let(:schedule) { nil }
|
95
94
|
|
96
|
-
it
|
95
|
+
it "runs the backup" do
|
97
96
|
expect(subject.upload_today?).to eq true
|
98
97
|
end
|
99
98
|
|
100
|
-
it
|
99
|
+
it "keeps the backup forever" do
|
101
100
|
expect(subject.rule(key)).to be_nil
|
102
101
|
end
|
103
102
|
end
|
data/spec/unit/runner_spec.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# encoding: UTF-8
|
2
|
-
require
|
2
|
+
require "spec_helper"
|
3
3
|
|
4
4
|
describe Stratocumulus::Runner do
|
5
|
-
subject { described_class.new(
|
5
|
+
subject { described_class.new("spec/support/test_config_file.yml") }
|
6
6
|
let(:storage) { double(upload: true) }
|
7
7
|
let(:database) { double }
|
8
8
|
let(:database2) { double }
|
@@ -15,41 +15,41 @@ describe Stratocumulus::Runner do
|
|
15
15
|
subject.run
|
16
16
|
end
|
17
17
|
|
18
|
-
it
|
18
|
+
it "passes the correct config to Storage" do
|
19
19
|
expect(Stratocumulus::Storage).to receive(:new).twice.with(
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
20
|
+
"access_key_id" => "I_AM_THE_KEY_ID",
|
21
|
+
"secret_access_key" => "IamTHESekret",
|
22
|
+
"bucket" => "stratocumulus-test",
|
23
|
+
"region" => "eu-west1",
|
24
|
+
"retention" => { 1 => 30, 30 => 12 },
|
25
25
|
).and_return(storage)
|
26
26
|
end
|
27
27
|
|
28
|
-
it
|
28
|
+
it "passes each database config to instances of Database" do
|
29
29
|
expect(Stratocumulus::Database).to receive(:new).once.with(
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
30
|
+
"type" => "mysql",
|
31
|
+
"name" => "stratocumulus_test",
|
32
|
+
"storage" => "s3",
|
33
|
+
"username" => "root",
|
34
|
+
"password" => "sekret",
|
35
|
+
"host" => "db1.example.com",
|
36
|
+
"port" => 3307,
|
37
37
|
)
|
38
38
|
|
39
39
|
expect(Stratocumulus::Database).to receive(:new).once.with(
|
40
|
-
|
41
|
-
|
42
|
-
|
40
|
+
"type" => "mysql",
|
41
|
+
"name" => "stratocumulus_test_2",
|
42
|
+
"storage" => "s3",
|
43
43
|
)
|
44
44
|
end
|
45
45
|
|
46
|
-
it
|
46
|
+
it "uploads each database to storage" do
|
47
47
|
allow(Stratocumulus::Database).to receive(:new).once.with(
|
48
|
-
hash_including(
|
48
|
+
hash_including("name" => "stratocumulus_test"),
|
49
49
|
).and_return(database)
|
50
50
|
|
51
51
|
allow(Stratocumulus::Database).to receive(:new).once.with(
|
52
|
-
hash_including(
|
52
|
+
hash_including("name" => "stratocumulus_test_2"),
|
53
53
|
).and_return(database2)
|
54
54
|
|
55
55
|
expect(storage).to receive(:upload).once.with(database)
|
data/spec/unit/storage_spec.rb
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
# encoding: UTF-8
|
2
|
-
require
|
2
|
+
require "spec_helper"
|
3
3
|
|
4
4
|
describe Stratocumulus::Storage do
|
5
5
|
let(:base_config) do
|
6
6
|
{
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
7
|
+
"access_key_id" => "IM_A_ID",
|
8
|
+
"secret_access_key" => "IM_A_SEKRET_KEY",
|
9
|
+
"region" => "eu-west-1",
|
10
|
+
"bucket" => "stratocumulus-test",
|
11
11
|
}
|
12
12
|
end
|
13
13
|
|
@@ -15,7 +15,7 @@ describe Stratocumulus::Storage do
|
|
15
15
|
|
16
16
|
subject { described_class.new(config) }
|
17
17
|
|
18
|
-
describe
|
18
|
+
describe "#upload" do
|
19
19
|
let(:connection) { double(:fog_conn, directories: directories) }
|
20
20
|
let(:directories) { double(:fog_dirs, get: double(files: files)) }
|
21
21
|
let(:files) { double(:fog_files, create: file) }
|
@@ -24,9 +24,9 @@ describe Stratocumulus::Storage do
|
|
24
24
|
let(:database) do
|
25
25
|
double(
|
26
26
|
:database,
|
27
|
-
filename:
|
27
|
+
filename: "foo.sql.gz",
|
28
28
|
dump: :database_dump,
|
29
|
-
success?: true
|
29
|
+
success?: true,
|
30
30
|
)
|
31
31
|
end
|
32
32
|
|
@@ -44,37 +44,56 @@ describe Stratocumulus::Storage do
|
|
44
44
|
stderr
|
45
45
|
end
|
46
46
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
47
|
+
context "without specified folder" do
|
48
|
+
it "uploads the dump to s3" do
|
49
|
+
expect(files).to receive(:create).with(
|
50
|
+
key: "foo.sql.gz",
|
51
|
+
body: :database_dump,
|
52
|
+
multipart_chunk_size: 104_857_600,
|
53
|
+
public: false,
|
54
|
+
)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
context "with specified folder" do
|
59
|
+
let(:config) do
|
60
|
+
base_config.merge(
|
61
|
+
"folder" => "folder",
|
62
|
+
)
|
63
|
+
end
|
64
|
+
|
65
|
+
it "uploads the dump to s3 in the right folder" do
|
66
|
+
expect(files).to receive(:create).with(
|
67
|
+
key: "folder/foo.sql.gz",
|
68
|
+
body: :database_dump,
|
69
|
+
multipart_chunk_size: 104_857_600,
|
70
|
+
public: false,
|
71
|
+
)
|
72
|
+
end
|
54
73
|
end
|
55
74
|
|
56
|
-
it
|
57
|
-
expect(directories).to receive(:get).with(
|
75
|
+
it "uploads to the correct s3 bucket" do
|
76
|
+
expect(directories).to receive(:get).with("stratocumulus-test")
|
58
77
|
end
|
59
78
|
|
60
|
-
describe
|
61
|
-
it
|
79
|
+
describe "the fog connection" do
|
80
|
+
it "is setup corectly" do
|
62
81
|
expect(Fog::Storage).to receive(:new)
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
82
|
+
.with(
|
83
|
+
provider: "AWS",
|
84
|
+
aws_access_key_id: "IM_A_ID",
|
85
|
+
aws_secret_access_key: "IM_A_SEKRET_KEY",
|
86
|
+
region: "eu-west-1",
|
87
|
+
).and_return(connection)
|
69
88
|
end
|
70
89
|
end
|
71
90
|
|
72
|
-
context
|
91
|
+
context "with a schedule" do
|
73
92
|
let(:config) do
|
74
93
|
base_config.merge(
|
75
|
-
|
76
|
-
1 => 30
|
77
|
-
}
|
94
|
+
"retention" => {
|
95
|
+
1 => 30,
|
96
|
+
},
|
78
97
|
)
|
79
98
|
end
|
80
99
|
|
@@ -84,36 +103,36 @@ describe Stratocumulus::Storage do
|
|
84
103
|
|
85
104
|
let(:service) { double(:fog_service) }
|
86
105
|
|
87
|
-
context
|
106
|
+
context "no rules set on the bucket yet" do
|
88
107
|
before do
|
89
108
|
allow(service).to receive(:get_bucket_lifecycle)
|
90
|
-
.with(
|
91
|
-
.and_raise(Excon::Errors::NotFound,
|
109
|
+
.with("stratocumulus-test")
|
110
|
+
.and_raise(Excon::Errors::NotFound, "404 No rules set yet")
|
92
111
|
end
|
93
112
|
|
94
|
-
it
|
113
|
+
it "puts a rule for the uploaded file" do
|
95
114
|
expect(service).to receive(:put_bucket_lifecycle)
|
96
115
|
.with(
|
97
|
-
|
98
|
-
|
116
|
+
"stratocumulus-test",
|
117
|
+
"Rules" => [
|
99
118
|
{
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
}
|
105
|
-
]
|
119
|
+
"ID" => "foo.sql.gz",
|
120
|
+
"Prefix" => "foo.sql.gz",
|
121
|
+
"Enabled" => true,
|
122
|
+
"Days" => 30,
|
123
|
+
},
|
124
|
+
],
|
106
125
|
)
|
107
126
|
end
|
108
127
|
end
|
109
128
|
|
110
|
-
context
|
129
|
+
context "when the database is not sucessfull" do
|
111
130
|
let(:database) do
|
112
131
|
double(
|
113
132
|
:database,
|
114
|
-
filename:
|
133
|
+
filename: "foo.sql.gz",
|
115
134
|
dump: :database_dump,
|
116
|
-
success?: false
|
135
|
+
success?: false,
|
117
136
|
)
|
118
137
|
end
|
119
138
|
|
@@ -121,73 +140,73 @@ describe Stratocumulus::Storage do
|
|
121
140
|
allow(file).to receive(:destroy)
|
122
141
|
end
|
123
142
|
|
124
|
-
it
|
143
|
+
it "destroys the failing dump" do
|
125
144
|
expect(file).to receive(:destroy)
|
126
145
|
end
|
127
146
|
|
128
|
-
it
|
147
|
+
it "does not create a expiry rule" do
|
129
148
|
expect(service).to_not receive(:put_bucket_lifecycle)
|
130
149
|
end
|
131
150
|
|
132
|
-
it
|
151
|
+
it "logs the error to stderr" do
|
133
152
|
expect(stderr).to include(
|
134
|
-
|
153
|
+
"ERROR -- : there was an error generating foo.sql.gz",
|
135
154
|
)
|
136
155
|
end
|
137
156
|
end
|
138
157
|
|
139
|
-
context
|
158
|
+
context "rules allready set on the bucket" do
|
140
159
|
let(:files) { [existing_file] }
|
141
|
-
let(:existing_file) { double(:fog_file, key:
|
160
|
+
let(:existing_file) { double(:fog_file, key: "bar.sql.gz") }
|
142
161
|
let(:existing_rules) do
|
143
162
|
[
|
144
163
|
{
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
164
|
+
"ID" => "bar.sql.gz",
|
165
|
+
"Prefix" => "bar.sql.gz",
|
166
|
+
"Enabled" => true,
|
167
|
+
"Days" => 30,
|
149
168
|
},
|
150
169
|
{
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
}
|
170
|
+
"ID" => "baz.sql.gz",
|
171
|
+
"Prefix" => "baz.sql.gz",
|
172
|
+
"Enabled" => true,
|
173
|
+
"Days" => 30,
|
174
|
+
},
|
156
175
|
]
|
157
176
|
end
|
158
177
|
|
159
178
|
before do
|
160
179
|
allow(service).to receive(:get_bucket_lifecycle)
|
161
|
-
.with(
|
180
|
+
.with("stratocumulus-test")
|
162
181
|
.and_return(
|
163
182
|
double(
|
164
183
|
data: {
|
165
184
|
body: {
|
166
|
-
|
167
|
-
}
|
168
|
-
}
|
169
|
-
)
|
185
|
+
"Rules" => existing_rules,
|
186
|
+
},
|
187
|
+
},
|
188
|
+
),
|
170
189
|
)
|
171
190
|
allow(files).to receive(:create).and_return(:true)
|
172
191
|
end
|
173
192
|
|
174
|
-
it
|
193
|
+
it "adds the rule to the rules for existing files" do
|
175
194
|
expect(service).to receive(:put_bucket_lifecycle).with(
|
176
|
-
|
177
|
-
|
195
|
+
"stratocumulus-test",
|
196
|
+
"Rules" => [
|
178
197
|
{
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
198
|
+
"ID" => "bar.sql.gz",
|
199
|
+
"Prefix" => "bar.sql.gz",
|
200
|
+
"Enabled" => true,
|
201
|
+
"Days" => 30,
|
183
202
|
},
|
184
203
|
{
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
}
|
190
|
-
]
|
204
|
+
"ID" => "foo.sql.gz",
|
205
|
+
"Prefix" => "foo.sql.gz",
|
206
|
+
"Enabled" => true,
|
207
|
+
"Days" => 30,
|
208
|
+
},
|
209
|
+
],
|
191
210
|
)
|
192
211
|
end
|
193
212
|
end
|