s3_rotate 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +4 -0
- data/lib/s3_rotate/core/backup_rotator.rb +6 -6
- data/lib/s3_rotate/utils/file_utils.rb +9 -2
- data/s3_rotate.gemspec +11 -4
- data/spec/s3_rotate/core/backup_rotator_spec.rb +92 -0
- metadata +24 -10
- data/Gemfile.lock +0 -55
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 16f04d3b048eedc89e3a4efaacdb46009bda296f842ace1d1a0c56f87bb88158
|
4
|
+
data.tar.gz: df1fd512ebb22a0800ea822364e76f69c0a5409d89e90b95ee7731dcd81f0bed
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 89caf936d757c6d2afd3bc2eba65361e9d79ed1547b1fed1f51317925af7ddd458dce94cdd6a4402cf07d35889bb3be105c047dbdc6c690bf9f3c71fb715a26e
|
7
|
+
data.tar.gz: 30e6a7117c71466ff2bc99dafe37d97edfc574d945c30c6b4886d06f2644a2dae68db59f23d1f6dc11c3e7f32928927ad11182a24e69f1e0749203aee090cc62
|
data/README.md
CHANGED
@@ -63,13 +63,13 @@ module S3Rotate
|
|
63
63
|
weekly_backups = @s3_client.remote_backups(backup_name, "weekly").files
|
64
64
|
|
65
65
|
# get most recent weekly file
|
66
|
-
recent_weekly_file = weekly_backups.last
|
66
|
+
recent_weekly_file = weekly_backups.last ? weekly_backups.last.key : nil
|
67
67
|
|
68
68
|
# look through daily backups to find which oness should be promoted
|
69
69
|
daily_backups.each do |backup|
|
70
70
|
# promote to weekly if applicable
|
71
|
-
if should_promote_daily_to_weekly?(backup.key, recent_weekly_file
|
72
|
-
recent_weekly_file = promote(backup_name, backup.key, backup.body, "weekly")
|
71
|
+
if should_promote_daily_to_weekly?(backup.key, recent_weekly_file)
|
72
|
+
recent_weekly_file = promote(backup_name, backup.key, backup.body, "weekly").key
|
73
73
|
end
|
74
74
|
end
|
75
75
|
|
@@ -100,13 +100,13 @@ module S3Rotate
|
|
100
100
|
monthly_backups = @s3_client.remote_backups(backup_name, "monthly").files
|
101
101
|
|
102
102
|
# get most recent monthly file
|
103
|
-
recent_monthly_file = monthly_backups.last
|
103
|
+
recent_monthly_file = monthly_backups.last ? monthly_backups.last.key : nil
|
104
104
|
|
105
105
|
# look through weekly backups to find which oness should be promoted
|
106
106
|
weekly_backups.each do |backup|
|
107
107
|
# promote to monthly if applicable
|
108
|
-
if should_promote_weekly_to_monthly?(backup.key, recent_monthly_file
|
109
|
-
recent_monthly_file = promote(backup_name, backup.key, backup.body, "monthly")
|
108
|
+
if should_promote_weekly_to_monthly?(backup.key, recent_monthly_file)
|
109
|
+
recent_monthly_file = promote(backup_name, backup.key, backup.body, "monthly").key
|
110
110
|
end
|
111
111
|
end
|
112
112
|
|
@@ -15,8 +15,15 @@ module S3Rotate
|
|
15
15
|
#
|
16
16
|
def FileUtils.date_from_filename(filename, date_regex=/\d{4}-\d{2}-\d{2}/)
|
17
17
|
# match the date in the filename
|
18
|
-
match
|
19
|
-
|
18
|
+
match = filename.match(date_regex)
|
19
|
+
|
20
|
+
if not match
|
21
|
+
date_str = nil
|
22
|
+
elsif not match.captures
|
23
|
+
date_str = match.to_s
|
24
|
+
else
|
25
|
+
date_str = match.captures.first || match.to_s
|
26
|
+
end
|
20
27
|
|
21
28
|
# if nothing could be match, immediately fail
|
22
29
|
raise "Invalid date_regex or filename format" if not date_str
|
data/s3_rotate.gemspec
CHANGED
@@ -2,14 +2,21 @@ require 'date'
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = 's3_rotate'
|
5
|
-
s.version = '1.0.0'
|
6
|
-
s.homepage = 'https://github.com/Whova/s3_rotate'
|
7
|
-
s.date = Date.today.to_s
|
8
5
|
s.summary = "AWS S3 upload with rotation mechanism"
|
9
6
|
s.description = s.summary
|
7
|
+
s.homepage = 'https://github.com/Whova/s3_rotate'
|
8
|
+
s.license = 'MIT'
|
9
|
+
|
10
|
+
s.version = '1.0.1'
|
11
|
+
s.date = Date.today.to_s
|
12
|
+
|
10
13
|
s.authors = ["Simon Ninon"]
|
11
14
|
s.email = 'simon.ninon@gmail.com'
|
15
|
+
|
12
16
|
s.files = `git ls-files`.split("\n")
|
13
17
|
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
14
|
-
|
18
|
+
|
19
|
+
s.required_ruby_version = '>= 2.0.0'
|
20
|
+
|
21
|
+
s.add_dependency 'fog-aws', '~> 3.5.2'
|
15
22
|
end
|
@@ -147,6 +147,48 @@ describe S3Rotate::BackupRotator do
|
|
147
147
|
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files[0].body).to eq 'some data'
|
148
148
|
end
|
149
149
|
|
150
|
+
it 'rotates multiples when relevant' do
|
151
|
+
# mock
|
152
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-12.tgz', body: 'some data')
|
153
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-13.tgz', body: 'some data')
|
154
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-14.tgz', body: 'some data')
|
155
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-16.tgz', body: 'some data')
|
156
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-20.tgz', body: 'some data')
|
157
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-21.tgz', body: 'some data')
|
158
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-24.tgz', body: 'some data')
|
159
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-27.tgz', body: 'some data')
|
160
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-29.tgz', body: 'some data')
|
161
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/weekly/2020-01-06.tgz', body: 'some data')
|
162
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/monthly/2020-01-06.tgz', body: 'some data')
|
163
|
+
@client.connection.directories.get('bucket').files.create(key: '/other_backup_name/daily/2020-01-07.tgz', body: 'some data')
|
164
|
+
|
165
|
+
# perform test
|
166
|
+
@rotator.rotate_daily('backup_name', 3)
|
167
|
+
|
168
|
+
# verify result
|
169
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files.length).to eq 3
|
170
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[0].key).to eq '/backup_name/daily/2020-01-24.tgz'
|
171
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[0].body).to eq 'some data'
|
172
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[1].key).to eq '/backup_name/daily/2020-01-27.tgz'
|
173
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[1].body).to eq 'some data'
|
174
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[2].key).to eq '/backup_name/daily/2020-01-29.tgz'
|
175
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[2].body).to eq 'some data'
|
176
|
+
|
177
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files.length).to eq 4
|
178
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[0].key).to eq '/backup_name/weekly/2020-01-06.tgz'
|
179
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[0].body).to eq 'some data'
|
180
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[1].key).to eq '/backup_name/weekly/2020-01-13.tgz'
|
181
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[1].body).to eq 'some data'
|
182
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[2].key).to eq '/backup_name/weekly/2020-01-20.tgz'
|
183
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[2].body).to eq 'some data'
|
184
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[3].key).to eq '/backup_name/weekly/2020-01-27.tgz'
|
185
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[3].body).to eq 'some data'
|
186
|
+
|
187
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files.length).to eq 1
|
188
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files[0].key).to eq '/backup_name/monthly/2020-01-06.tgz'
|
189
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files[0].body).to eq 'some data'
|
190
|
+
end
|
191
|
+
|
150
192
|
it 'always promote when there is no weekly backup' do
|
151
193
|
# mock
|
152
194
|
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-12.tgz', body: 'some data')
|
@@ -338,6 +380,56 @@ describe S3Rotate::BackupRotator do
|
|
338
380
|
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files[1].body).to eq 'some data'
|
339
381
|
end
|
340
382
|
|
383
|
+
it 'rotates multiple when relevant' do
|
384
|
+
# mock
|
385
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-12.tgz', body: 'some data')
|
386
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-13.tgz', body: 'some data')
|
387
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-14.tgz', body: 'some data')
|
388
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-15.tgz', body: 'some data')
|
389
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-16.tgz', body: 'some data')
|
390
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-17.tgz', body: 'some data')
|
391
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/weekly/2020-02-07.tgz', body: 'some data')
|
392
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/weekly/2020-02-14.tgz', body: 'some data')
|
393
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/weekly/2020-03-06.tgz', body: 'some data')
|
394
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/weekly/2020-03-13.tgz', body: 'some data')
|
395
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/weekly/2020-04-05.tgz', body: 'some data')
|
396
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/weekly/2020-04-12.tgz', body: 'some data')
|
397
|
+
@client.connection.directories.get('bucket').files.create(key: '/backup_name/monthly/2020-01-12.tgz', body: 'some data')
|
398
|
+
@client.connection.directories.get('bucket').files.create(key: '/other_backup_name/daily/2020-01-07.tgz', body: 'some data')
|
399
|
+
|
400
|
+
# perform test
|
401
|
+
@rotator.rotate_weekly('backup_name', 2)
|
402
|
+
|
403
|
+
# verify result
|
404
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files.length).to eq 6
|
405
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[0].key).to eq '/backup_name/daily/2020-01-12.tgz'
|
406
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[0].body).to eq 'some data'
|
407
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[1].key).to eq '/backup_name/daily/2020-01-13.tgz'
|
408
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[1].body).to eq 'some data'
|
409
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[2].key).to eq '/backup_name/daily/2020-01-14.tgz'
|
410
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[2].body).to eq 'some data'
|
411
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[3].key).to eq '/backup_name/daily/2020-01-15.tgz'
|
412
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[3].body).to eq 'some data'
|
413
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[4].key).to eq '/backup_name/daily/2020-01-16.tgz'
|
414
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[4].body).to eq 'some data'
|
415
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[5].key).to eq '/backup_name/daily/2020-01-17.tgz'
|
416
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[5].body).to eq 'some data'
|
417
|
+
|
418
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files.length).to eq 2
|
419
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[0].key).to eq '/backup_name/weekly/2020-04-05.tgz'
|
420
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[0].body).to eq 'some data'
|
421
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[1].key).to eq '/backup_name/weekly/2020-04-12.tgz'
|
422
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[1].body).to eq 'some data'
|
423
|
+
|
424
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files.length).to eq 3
|
425
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files[0].key).to eq '/backup_name/monthly/2020-01-12.tgz'
|
426
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files[0].body).to eq 'some data'
|
427
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files[1].key).to eq '/backup_name/monthly/2020-02-14.tgz'
|
428
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files[1].body).to eq 'some data'
|
429
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files[2].key).to eq '/backup_name/monthly/2020-04-05.tgz'
|
430
|
+
expect(@client.connection.directories.get('bucket', prefix: '/backup_name/monthly').files[2].body).to eq 'some data'
|
431
|
+
end
|
432
|
+
|
341
433
|
it 'always promote when there is no monthly backup' do
|
342
434
|
# mock
|
343
435
|
@client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-12.tgz', body: 'some data')
|
metadata
CHANGED
@@ -1,25 +1,38 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_rotate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Simon Ninon
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-01-
|
12
|
-
dependencies:
|
11
|
+
date: 2020-01-28 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fog-aws
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 3.5.2
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 3.5.2
|
13
27
|
description: AWS S3 upload with rotation mechanism
|
14
28
|
email: simon.ninon@gmail.com
|
15
29
|
executables: []
|
16
30
|
extensions: []
|
17
31
|
extra_rdoc_files: []
|
18
32
|
files:
|
19
|
-
-
|
20
|
-
-
|
33
|
+
- .gitignore
|
34
|
+
- .rspec
|
21
35
|
- Gemfile
|
22
|
-
- Gemfile.lock
|
23
36
|
- LICENSE
|
24
37
|
- README.md
|
25
38
|
- examples/example.rb
|
@@ -50,16 +63,17 @@ require_paths:
|
|
50
63
|
- lib
|
51
64
|
required_ruby_version: !ruby/object:Gem::Requirement
|
52
65
|
requirements:
|
53
|
-
- -
|
66
|
+
- - '>='
|
54
67
|
- !ruby/object:Gem::Version
|
55
|
-
version:
|
68
|
+
version: 2.0.0
|
56
69
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
57
70
|
requirements:
|
58
|
-
- -
|
71
|
+
- - '>='
|
59
72
|
- !ruby/object:Gem::Version
|
60
73
|
version: '0'
|
61
74
|
requirements: []
|
62
|
-
|
75
|
+
rubyforge_project:
|
76
|
+
rubygems_version: 2.7.10
|
63
77
|
signing_key:
|
64
78
|
specification_version: 4
|
65
79
|
summary: AWS S3 upload with rotation mechanism
|
data/Gemfile.lock
DELETED
@@ -1,55 +0,0 @@
|
|
1
|
-
GEM
|
2
|
-
remote: https://rubygems.org/
|
3
|
-
specs:
|
4
|
-
builder (3.2.4)
|
5
|
-
diff-lcs (1.3)
|
6
|
-
excon (0.71.1)
|
7
|
-
fog-aws (3.5.2)
|
8
|
-
fog-core (~> 2.1)
|
9
|
-
fog-json (~> 1.1)
|
10
|
-
fog-xml (~> 0.1)
|
11
|
-
ipaddress (~> 0.8)
|
12
|
-
fog-core (2.2.0)
|
13
|
-
builder
|
14
|
-
excon (~> 0.71)
|
15
|
-
formatador (~> 0.2)
|
16
|
-
mime-types
|
17
|
-
fog-json (1.2.0)
|
18
|
-
fog-core
|
19
|
-
multi_json (~> 1.10)
|
20
|
-
fog-xml (0.1.3)
|
21
|
-
fog-core
|
22
|
-
nokogiri (>= 1.5.11, < 2.0.0)
|
23
|
-
formatador (0.2.5)
|
24
|
-
ipaddress (0.8.3)
|
25
|
-
mime-types (3.3.1)
|
26
|
-
mime-types-data (~> 3.2015)
|
27
|
-
mime-types-data (3.2019.1009)
|
28
|
-
mini_portile2 (2.4.0)
|
29
|
-
multi_json (1.14.1)
|
30
|
-
nokogiri (1.10.7)
|
31
|
-
mini_portile2 (~> 2.4.0)
|
32
|
-
rspec (3.9.0)
|
33
|
-
rspec-core (~> 3.9.0)
|
34
|
-
rspec-expectations (~> 3.9.0)
|
35
|
-
rspec-mocks (~> 3.9.0)
|
36
|
-
rspec-core (3.9.1)
|
37
|
-
rspec-support (~> 3.9.1)
|
38
|
-
rspec-expectations (3.9.0)
|
39
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
40
|
-
rspec-support (~> 3.9.0)
|
41
|
-
rspec-mocks (3.9.1)
|
42
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
43
|
-
rspec-support (~> 3.9.0)
|
44
|
-
rspec-support (3.9.2)
|
45
|
-
|
46
|
-
PLATFORMS
|
47
|
-
ruby
|
48
|
-
|
49
|
-
DEPENDENCIES
|
50
|
-
fog-aws (~> 3.5.2)
|
51
|
-
rspec
|
52
|
-
rspec-mocks
|
53
|
-
|
54
|
-
BUNDLED WITH
|
55
|
-
1.17.2
|