webtranslateit-safe 0.4.2 → 0.4.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG +22 -9
  3. data/bin/webtranslateit-safe +13 -13
  4. data/lib/webtranslateit/safe/archive.rb +9 -4
  5. data/lib/webtranslateit/safe/backup.rb +9 -2
  6. data/lib/webtranslateit/safe/cloudfiles.rb +21 -15
  7. data/lib/webtranslateit/safe/config/builder.rb +17 -8
  8. data/lib/webtranslateit/safe/config/node.rb +17 -10
  9. data/lib/webtranslateit/safe/ftp.rb +25 -25
  10. data/lib/webtranslateit/safe/gpg.rb +8 -2
  11. data/lib/webtranslateit/safe/gzip.rb +5 -1
  12. data/lib/webtranslateit/safe/local.rb +15 -11
  13. data/lib/webtranslateit/safe/mongodump.rb +12 -6
  14. data/lib/webtranslateit/safe/mysqldump.rb +9 -5
  15. data/lib/webtranslateit/safe/pgdump.rb +9 -9
  16. data/lib/webtranslateit/safe/pipe.rb +6 -0
  17. data/lib/webtranslateit/safe/s3.rb +28 -22
  18. data/lib/webtranslateit/safe/sftp.rb +42 -25
  19. data/lib/webtranslateit/safe/sink.rb +7 -2
  20. data/lib/webtranslateit/safe/source.rb +13 -9
  21. data/lib/webtranslateit/safe/stream.rb +14 -6
  22. data/lib/webtranslateit/safe/svndump.rb +5 -1
  23. data/lib/webtranslateit/safe/tmp_file.rb +16 -11
  24. data/lib/webtranslateit/safe.rb +12 -13
  25. metadata +3 -36
  26. data/.autotest +0 -3
  27. data/.document +0 -5
  28. data/.github/dependabot.yml +0 -26
  29. data/.github/release-drafter.yml +0 -36
  30. data/.github/workflows/ci.yml +0 -51
  31. data/.github/workflows/release-drafter.yml +0 -29
  32. data/.gitignore +0 -18
  33. data/.rspec +0 -3
  34. data/.rubocop.yml +0 -8
  35. data/.rubocop_todo.yml +0 -965
  36. data/Gemfile +0 -11
  37. data/Gemfile.lock +0 -89
  38. data/LICENSE.txt +0 -22
  39. data/Rakefile +0 -8
  40. data/TODO +0 -31
  41. data/lib/extensions/mktmpdir.rb +0 -45
  42. data/lib/webtranslateit/safe/version.rb +0 -5
  43. data/spec/integration/archive_integration_spec.rb +0 -89
  44. data/spec/integration/cleanup_spec.rb +0 -62
  45. data/spec/spec_helper.rb +0 -7
  46. data/spec/webtranslateit/safe/archive_spec.rb +0 -67
  47. data/spec/webtranslateit/safe/cloudfiles_spec.rb +0 -175
  48. data/spec/webtranslateit/safe/config_spec.rb +0 -307
  49. data/spec/webtranslateit/safe/gpg_spec.rb +0 -148
  50. data/spec/webtranslateit/safe/gzip_spec.rb +0 -64
  51. data/spec/webtranslateit/safe/local_spec.rb +0 -109
  52. data/spec/webtranslateit/safe/mongodump_spec.rb +0 -54
  53. data/spec/webtranslateit/safe/mysqldump_spec.rb +0 -83
  54. data/spec/webtranslateit/safe/pgdump_spec.rb +0 -45
  55. data/spec/webtranslateit/safe/s3_spec.rb +0 -168
  56. data/spec/webtranslateit/safe/svndump_spec.rb +0 -39
  57. data/templates/script.rb +0 -183
  58. data/webtranslateit-safe.gemspec +0 -31
@@ -1,83 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe WebTranslateIt::Safe::Mysqldump do
4
-
5
- def def_config(extra = {})
6
- {
7
- :options => 'OPTS',
8
- :user => 'User',
9
- :password => 'pwd',
10
- :host => 'localhost',
11
- :port => 7777,
12
- :socket => 'socket',
13
- :skip_tables => [:bar, :baz]
14
- }.merge(extra)
15
- end
16
-
17
- def mysqldump(id = :foo, config = def_config)
18
- WebTranslateIt::Safe::Mysqldump.new(id, WebTranslateIt::Safe::Config::Node.new(nil, config))
19
- end
20
-
21
- before(:each) do
22
- stub(Time).now.stub!.strftime {'NOW'}
23
- end
24
-
25
- after(:each) { WebTranslateIt::Safe::TmpFile.cleanup }
26
-
27
- describe :backup do
28
- before(:each) do
29
- @mysql = mysqldump
30
- stub(@mysql).mysql_password_file {'/tmp/pwd'}
31
- end
32
-
33
- {
34
- :id => 'foo',
35
- :kind => 'mysqldump',
36
- :extension => '.sql',
37
- :filename => 'mysqldump-foo.NOW',
38
- :command => 'mysqldump --defaults-extra-file=/tmp/pwd OPTS --ignore-table=foo.bar --ignore-table=foo.baz foo',
39
- }.each do |k, v|
40
- it "should set #{k} to #{v}" do
41
- @mysql.backup.send(k).should == v
42
- end
43
- end
44
-
45
- end
46
-
47
- describe :mysql_skip_tables do
48
- it 'should return nil if no skip_tables' do
49
- config = def_config.dup
50
- config.delete(:skip_tables)
51
- m = mysqldump(:foo, WebTranslateIt::Safe::Config::Node.new(nil, config))
52
- stub(m).timestamp {'NOW'}
53
- m.send(:mysql_skip_tables).should be_nil
54
- m.backup.command.should_not match(/ignore-table/)
55
- end
56
-
57
- it "should return '' if skip_tables empty" do
58
- config = def_config.dup
59
- config[:skip_tables] = []
60
- m = mysqldump(:foo, WebTranslateIt::Safe::Config::Node.new(nil, config))
61
- stub(m).timestamp {'NOW'}
62
- m.send(:mysql_skip_tables).should == ''
63
- m.backup.command.should_not match(/ignore-table/)
64
- end
65
-
66
- end
67
-
68
- describe :mysql_password_file do
69
- it 'should create passwords file with quoted values' do
70
- m = mysqldump(:foo, def_config(:password => '#qwe"asd\'zxc'))
71
- file = m.send(:mysql_password_file)
72
- File.exist?(file).should == true
73
- File.read(file).should == <<-PWD
74
- [mysqldump]
75
- user = "User"
76
- password = "#qwe\\"asd'zxc"
77
- socket = "socket"
78
- host = "localhost"
79
- port = 7777
80
- PWD
81
- end
82
- end
83
- end
@@ -1,45 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe WebTranslateIt::Safe::Pgdump do
4
-
5
- def def_config
6
- {
7
- :options => 'OPTS',
8
- :user => 'User',
9
- :password => 'pwd',
10
- :host => 'localhost',
11
- :port => 7777,
12
- :skip_tables => [:bar, :baz]
13
- }
14
- end
15
-
16
- def pgdump(id = :foo, config = def_config)
17
- WebTranslateIt::Safe::Pgdump.new(id, WebTranslateIt::Safe::Config::Node.new(nil, config))
18
- end
19
-
20
- before(:each) do
21
- stub(Time).now.stub!.strftime {'NOW'}
22
- end
23
-
24
- after(:each) { WebTranslateIt::Safe::TmpFile.cleanup }
25
-
26
- describe :backup do
27
- before(:each) do
28
- @pg = pgdump
29
- end
30
-
31
- {
32
- :id => 'foo',
33
- :kind => 'pgdump',
34
- :extension => '.sql',
35
- :filename => 'pgdump-foo.NOW',
36
- :command => "pg_dump OPTS --username='User' --host='localhost' --port='7777' foo",
37
- }.each do |k, v|
38
- it "should set #{k} to #{v}" do
39
- @pg.backup.send(k).should == v
40
- end
41
- end
42
-
43
- end
44
-
45
- end
@@ -1,168 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe WebTranslateIt::Safe::S3 do
4
-
5
- def def_config
6
- {
7
- :s3 => {
8
- :bucket => '_bucket',
9
- :key => '_key',
10
- :secret => '_secret',
11
- },
12
- :keep => {
13
- :s3 => 2
14
- }
15
- }
16
- end
17
-
18
- def def_backup(extra = {})
19
- {
20
- :kind => '_kind',
21
- :filename => '/backup/somewhere/_kind-_id.NOW.bar',
22
- :extension => '.bar',
23
- :id => '_id',
24
- :timestamp => 'NOW'
25
- }.merge(extra)
26
- end
27
-
28
- def s3(config = def_config, backup = def_backup)
29
- WebTranslateIt::Safe::S3.new(
30
- WebTranslateIt::Safe::Config::Node.new.merge(config),
31
- WebTranslateIt::Safe::Backup.new(backup)
32
- )
33
- end
34
-
35
- describe :cleanup do
36
-
37
- before(:each) do
38
- @s3 = s3
39
-
40
- @files = [4,1,3,2].map { |i| stub(o = {}).key {"aaaaa#{i}"}; o }
41
-
42
- stub(AWS::S3::Bucket).objects('_bucket', :prefix => '_kind/_id/_kind-_id.', :max_keys => 4) {@files}
43
- stub(AWS::S3::Bucket).objects('_bucket', :prefix => anything).stub![0].stub!.delete
44
- end
45
-
46
- it 'should check [:keep, :s3]' do
47
- @s3.config[:keep].data['s3'] = nil
48
- dont_allow(@s3.backup).filename
49
- @s3.send :cleanup
50
- end
51
-
52
- it 'should delete extra files' do
53
- mock(AWS::S3::Bucket).objects('_bucket', :prefix => 'aaaaa1').mock![0].mock!.delete
54
- mock(AWS::S3::Bucket).objects('_bucket', :prefix => 'aaaaa2').mock![0].mock!.delete
55
- @s3.send :cleanup
56
- end
57
-
58
- end
59
-
60
- describe :active do
61
- before(:each) do
62
- @s3 = s3
63
- end
64
-
65
- it 'should be true when all params are set' do
66
- expect(@s3.active?).to be_truthy
67
- end
68
-
69
- it 'should be false if bucket is missing' do
70
- @s3.config[:s3].data['bucket'] = nil
71
- expect(@s3.active?).to be_falsy
72
- end
73
-
74
- it 'should be false if key is missing' do
75
- @s3.config[:s3].data['key'] = nil
76
- expect(@s3.active?).to be_falsy
77
- end
78
-
79
- it 'should be false if secret is missing' do
80
- @s3.config[:s3].data['secret'] = nil
81
- expect(@s3.active?).to be_falsy
82
- end
83
- end
84
-
85
- describe :path do
86
- before(:each) do
87
- @s3 = s3
88
- end
89
- it 'should use s3/path 1st' do
90
- @s3.config[:s3].data['path'] = 's3_path'
91
- @s3.config[:local] = {:path => 'local_path'}
92
- @s3.send(:path).should == 's3_path'
93
- end
94
-
95
- it 'should use local/path 2nd' do
96
- @s3.config.merge local: {path: 'local_path'}
97
- @s3.send(:path).should == 'local_path'
98
- end
99
-
100
- it 'should use constant 3rd' do
101
- @s3.send(:path).should == '_kind/_id'
102
- end
103
-
104
- end
105
-
106
- describe :save do
107
- def add_stubs(*stubs)
108
- stubs.each do |s|
109
- case s
110
- when :connection
111
- stub(AWS::S3::Base).establish_connection!(:access_key_id => '_key', :secret_access_key => '_secret', :use_ssl => true)
112
- when :stat
113
- stub(File).stat('foo').stub!.size {123}
114
- when :create_bucket
115
- stub(AWS::S3::Bucket).find('_bucket') { raise_error AWS::S3::NoSuchBucket }
116
- stub(AWS::S3::Bucket).create
117
- when :file_open
118
- stub(File).open('foo') {|f, block| block.call(:opened_file)}
119
- when :s3_store
120
- stub(AWS::S3::S3Object).store(@full_path, :opened_file, '_bucket')
121
- end
122
- end
123
- end
124
-
125
- before(:each) do
126
- @s3 = s3(def_config, def_backup(:path => 'foo'))
127
- @full_path = '_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar'
128
- end
129
-
130
- it 'should fail if no backup.file is set' do
131
- @s3.backup.path = nil
132
- proc {@s3.send(:save)}.should raise_error(RuntimeError)
133
- end
134
-
135
- it 'should establish s3 connection' do
136
- mock(AWS::S3::Base).establish_connection!(:access_key_id => '_key', :secret_access_key => '_secret', :use_ssl => true)
137
- add_stubs(:stat, :create_bucket, :file_open, :s3_store)
138
- @s3.send(:save)
139
- end
140
-
141
- it 'should open local file' do
142
- add_stubs(:connection, :stat, :create_bucket)
143
- mock(File).open('foo')
144
- @s3.send(:save)
145
- end
146
-
147
- it 'should upload file' do
148
- add_stubs(:connection, :stat, :create_bucket, :file_open)
149
- mock(AWS::S3::S3Object).store(@full_path, :opened_file, '_bucket')
150
- @s3.send(:save)
151
- end
152
-
153
- it 'should fail on files bigger then 5G' do
154
- add_stubs(:connection)
155
- mock(File).stat('foo').stub!.size {5*1024*1024*1024+1}
156
- mock(STDERR).puts(anything)
157
- dont_allow(Benchmark).realtime
158
- @s3.send(:save)
159
- end
160
-
161
- it 'should not create a bucket that already exists' do
162
- add_stubs(:connection, :stat, :file_open, :s3_store)
163
- stub(AWS::S3::Bucket).find('_bucket') { true }
164
- dont_allow(AWS::S3::Bucket).create
165
- @s3.send(:save)
166
- end
167
- end
168
- end
@@ -1,39 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe WebTranslateIt::Safe::Svndump do
4
- def def_config
5
- {
6
- :options => 'OPTS',
7
- :repo_path => 'bar/baz'
8
- }
9
- end
10
-
11
- def svndump(id = :foo, config = def_config)
12
- WebTranslateIt::Safe::Svndump.new(id, WebTranslateIt::Safe::Config::Node.new(nil, config))
13
- end
14
-
15
- before(:each) do
16
- stub(Time).now.stub!.strftime {'NOW'}
17
- end
18
-
19
- after(:each) { WebTranslateIt::Safe::TmpFile.cleanup }
20
-
21
- describe :backup do
22
- before(:each) do
23
- @svn = svndump
24
- end
25
-
26
- {
27
- :id => 'foo',
28
- :kind => 'svndump',
29
- :extension => '.svn',
30
- :filename => 'svndump-foo.NOW',
31
- :command => 'svnadmin dump OPTS bar/baz',
32
- }.each do |k, v|
33
- it "should set #{k} to #{v}" do
34
- @svn.backup.send(k).should == v
35
- end
36
- end
37
-
38
- end
39
- end
data/templates/script.rb DELETED
@@ -1,183 +0,0 @@
1
- safe do
2
- # same as --verbose on the command line
3
- # verbose true
4
-
5
- # same as --local on the command line
6
- # local_only true
7
-
8
- # same as --dry-run on the command line
9
- # dry_run true
10
-
11
- # backup file path (not including filename)
12
- # supported substitutions:
13
- # :kind -> backup 'engine' kind, e.g. "mysqldump" or "archive"
14
- # :id -> backup 'id', e.g. "blog", "production", etc.
15
- # :timestamp -> current run timestamp (same for all the backups in the same 'run')
16
- # you can set separate :path for all backups (or once globally here)
17
- local do
18
- path '/backup/:kind'
19
- end
20
-
21
- ## uncomment to enable uploads to Amazon S3
22
- ## Amazon S3 auth (optional)
23
- # s3 do
24
- # key YOUR_S3_KEY
25
- # secret YOUR_S3_SECRET
26
- # bucket S3_BUCKET
27
- # # path for uploads to S3. supports same substitution like :local/:path
28
- # path ":kind/" # this is default
29
- # end
30
-
31
- ## alternative style:
32
- # s3 :key => YOUR_S3_KEY, :secret => YOUR_S3_SECRET, :bucket => S3_BUCKET, :path => ":kind/"
33
-
34
- ## uncomment to enable uploads to Rackspace Cloud Files
35
- ## http://www.rackspacecloud.com/cloud_hosting_products/files
36
- ## Rackspace auth (optional)
37
- # cloudfiles do
38
- # user "YOUR_RACKSPACE_CLOUD_USERNAME"
39
- # api_key "YOUR_RACKSPACE_API_KEY"
40
- # container "YOUR_CONTAINER_NAME"
41
- # # path for uploads to Cloud Files, supports same substitution like :local/:path
42
- # path ":kind/" # this is default
43
- # # If you are running the backup from a system within the Rackspace/Slicehost network and would like
44
- # # to back up over the private (unbilled) service net, set this value to true.
45
- # # service_net true
46
- # end
47
-
48
- ## uncomment to enable uploads via SFTP
49
- # sftp do
50
- # host "YOUR_REMOTE_HOSTNAME"
51
- # user "YOUR_REMOTE_USERNAME"
52
- # # port "NON STANDARD SSH PORT"
53
- # password "YOUR_REMOTE_PASSWORD"
54
- # path ":kind/:id" # this is the default
55
- # end
56
-
57
- ## uncomment to enable uploads via FTP
58
- # ftp do
59
- # host "YOUR_REMOTE_HOSTNAME"
60
- # user "YOUR_REMOTE_USERNAME"
61
- # # port "NON STANDARD FTP PORT"
62
- # password "YOUR_REMOTE_PASSWORD"
63
- # path ":kind/:id" # this is the default
64
- # end
65
-
66
- ## uncomment to enable GPG encryption.
67
- ## Note: you can use public 'key' or symmetric password but not both!
68
- # gpg do
69
- # # you can specify your own gpg executable with the 'command' options
70
- # # this can be useful for example to choose b/w gpg and gpg2 if both are installed
71
- # # some gpg installations will automatically set 'use-agent' option in the
72
- # # config file on the 1st run. see README for more details
73
- # options "--no-use-agent"
74
- # # command "/usr/local/bin/gpg"
75
- # # key "backup@astrails.com"
76
- # password "astrails"
77
- # end
78
-
79
- ## uncomment to enable backup rotation. keep only given number of latest
80
- ## backups. remove the rest
81
- # keep do
82
- # local 4 # keep 4 local backups
83
- # s3 20 # keep 20 S3 backups
84
- # end
85
-
86
- # backup mysql databases with mysqldump
87
- mysqldump do
88
- # you can override any setting from parent in a child:
89
- options '-ceKq --single-transaction --create-options'
90
-
91
- user 'astrails'
92
- password ''
93
- # host "localhost"
94
- # port 3306
95
- socket '/var/run/mysqld/mysqld.sock'
96
-
97
- # database is a 'collection' element. it must have a hash or block parameter
98
- # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key
99
- # the following code will create mysqldump/databases/blog and mysqldump/databases/mysql configuration 'nodes'
100
-
101
- # backup database with default values
102
- # database :blog
103
-
104
- # backup overriding some values
105
- # database :production do
106
- # # you can override 'partially'
107
- # keep :local => 3
108
- # # keep/local is 3, and keep/s3 is 20 (from parent)
109
-
110
- # # local override for gpg password
111
- # gpg do
112
- # password "custom-production-pass"
113
- # end
114
- # # skip those tables during backup
115
- # # you can pass an array
116
- # skip_tables [:logger_exceptions, :request_logs]
117
- # # or pass them all separately
118
- # skip_tables :test1
119
- # skip_tables :test2
120
- # end
121
-
122
- end
123
-
124
- # # uncomment to enable
125
- # # backup PostgreSQL databases with pg_dump
126
- # pgdump do
127
- # options "-i -x -O"
128
- #
129
- # user "markmansour"
130
- # # password "" - leave this out if you have ident setup
131
- #
132
- # # database is a 'collection' element. it must have a hash or block parameter
133
- # # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key
134
- # database :blog
135
- # database :production
136
- # end
137
-
138
- tar do
139
- # options "-h" # uncomment this to dereference symbolic links
140
-
141
- # 'archive' is a collection item, just like 'database'
142
- # archive "git-repositories" do
143
- # # files and directories to backup
144
- # files "/home/git/repositories"
145
- # # can have more then one 'files' lines or/and use an array
146
- # files ["/home/dev/work/foo", "/home/dev/work/bar"]
147
- # end
148
-
149
- # archive "etc-files" do
150
- # files "/etc"
151
- # # exlude those files/directories
152
- # exclude "/etc/puppet/other"
153
- # # can have multiple 'exclude' lines or/and use an array
154
- # exclude ["/etc/tmp/a", "/etc/tmp/b"]
155
- # end
156
-
157
- # archive "dot-configs" do
158
- # files "/home/*/.[^.]*"
159
- # end
160
-
161
- # archive "blog" do
162
- # files "/var/www/blog.astrails.com/"
163
- # # specify multiple files/directories as array
164
- # exclude ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"]
165
- # end
166
-
167
- # archive "site" do
168
- # files "/var/www/astrails.com/"
169
- # exclude ["/var/www/astrails.com/log", "/var/www/astrails.com/tmp"]
170
- # end
171
-
172
- # archive :misc do
173
- # files [ "/backup/*.rb" ]
174
- # end
175
- end
176
-
177
- # svndump do
178
- # repo :my_repo do
179
- # repo_path "/home/svn/my_repo"
180
- # end
181
- # end
182
-
183
- end
@@ -1,31 +0,0 @@
1
- lib = File.expand_path('lib', __dir__)
2
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
- require 'webtranslateit/safe/version'
4
-
5
- Gem::Specification.new do |spec|
6
- spec.name = 'webtranslateit-safe'
7
- spec.version = WebTranslateIt::Safe::VERSION
8
- spec.authors = ['Edouard Briere', 'Vitaly Kushner']
9
- spec.email = ['support@webtranslateit.com']
10
- spec.required_ruby_version = '>= 3.2'
11
- spec.description = <<-DESC
12
- WebTranslateIt-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
13
- Backups can be stored locally or remotely and can be enctypted.
14
- Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain FTP/SFTP.
15
- DESC
16
- spec.summary = 'Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote server/service (with encryption)'
17
- spec.homepage = 'http://github.com/webtranslateit/safe'
18
- spec.license = 'MIT'
19
-
20
- spec.default_executable = 'webtranslateit-safe'
21
-
22
- spec.files = `git ls-files`.split($/)
23
- spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
24
- spec.require_paths = ['lib']
25
-
26
- spec.add_dependency 'aws-s3'
27
- spec.add_dependency 'cloudfiles'
28
- spec.add_dependency 'net-sftp'
29
-
30
- spec.metadata['rubygems_mfa_required'] = 'true'
31
- end