backup 3.0.16 → 3.0.18

Sign up to get free protection for your applications and to get access to all the features.
Files changed (81) hide show
  1. data/.travis.yml +10 -0
  2. data/Gemfile.lock +50 -47
  3. data/Guardfile +3 -3
  4. data/README.md +136 -81
  5. data/backup.gemspec +3 -2
  6. data/bin/backup +36 -15
  7. data/lib/backup.rb +30 -20
  8. data/lib/backup/cli.rb +30 -2
  9. data/lib/backup/compressor/lzma.rb +63 -0
  10. data/lib/backup/configuration/compressor/lzma.rb +23 -0
  11. data/lib/backup/configuration/helpers.rb +10 -4
  12. data/lib/backup/configuration/notifier/mail.rb +5 -0
  13. data/lib/backup/configuration/storage/dropbox.rb +19 -4
  14. data/lib/backup/configuration/storage/ftp.rb +4 -0
  15. data/lib/backup/configuration/storage/local.rb +17 -0
  16. data/lib/backup/configuration/storage/ninefold.rb +20 -0
  17. data/lib/backup/configuration/storage/rsync.rb +4 -0
  18. data/lib/backup/database/postgresql.rb +12 -3
  19. data/lib/backup/database/redis.rb +5 -1
  20. data/lib/backup/dependency.rb +11 -12
  21. data/lib/backup/encryptor/gpg.rb +2 -0
  22. data/lib/backup/exception/command_failed.rb +8 -0
  23. data/lib/backup/finder.rb +49 -9
  24. data/lib/backup/notifier/mail.rb +7 -1
  25. data/lib/backup/notifier/twitter.rb +1 -1
  26. data/lib/backup/storage/dropbox.rb +93 -16
  27. data/lib/backup/storage/ftp.rb +10 -3
  28. data/lib/backup/storage/local.rb +78 -0
  29. data/lib/backup/storage/ninefold.rb +96 -0
  30. data/lib/backup/storage/rsync.rb +37 -20
  31. data/lib/backup/storage/s3.rb +1 -1
  32. data/lib/backup/storage/scp.rb +1 -1
  33. data/lib/backup/syncer/rsync.rb +1 -1
  34. data/lib/backup/version.rb +1 -1
  35. data/lib/templates/compressor/lzma +7 -0
  36. data/lib/templates/storage/dropbox +2 -2
  37. data/lib/templates/storage/ftp +8 -7
  38. data/lib/templates/storage/local +7 -0
  39. data/lib/templates/storage/ninefold +9 -0
  40. data/lib/templates/storage/rsync +1 -0
  41. data/spec/archive_spec.rb +0 -1
  42. data/spec/compressor/bzip2_spec.rb +0 -1
  43. data/spec/compressor/gzip_spec.rb +0 -1
  44. data/spec/compressor/lzma_spec.rb +58 -0
  45. data/spec/configuration/compressor/bzip2_spec.rb +28 -0
  46. data/spec/configuration/compressor/lzma_spec.rb +28 -0
  47. data/spec/configuration/database/mongodb_spec.rb +16 -0
  48. data/spec/configuration/database/mysql_spec.rb +17 -0
  49. data/spec/configuration/database/postgresql_spec.rb +17 -0
  50. data/spec/configuration/database/redis_spec.rb +16 -0
  51. data/spec/configuration/notifier/campfire_spec.rb +11 -0
  52. data/spec/configuration/notifier/mail_spec.rb +20 -0
  53. data/spec/configuration/notifier/presently_spec.rb +34 -0
  54. data/spec/configuration/notifier/twitter_spec.rb +12 -0
  55. data/spec/configuration/storage/dropbox_spec.rb +0 -6
  56. data/spec/configuration/storage/ftp_spec.rb +15 -12
  57. data/spec/configuration/storage/local_spec.rb +28 -0
  58. data/spec/configuration/storage/ninefold_spec.rb +31 -0
  59. data/spec/configuration/storage/rsync_spec.rb +2 -0
  60. data/spec/database/mongodb_spec.rb +0 -1
  61. data/spec/database/mysql_spec.rb +0 -1
  62. data/spec/database/postgresql_spec.rb +31 -11
  63. data/spec/database/redis_spec.rb +9 -4
  64. data/spec/encryptor/gpg_spec.rb +1 -1
  65. data/spec/encryptor/open_ssl_spec.rb +0 -1
  66. data/spec/logger_spec.rb +32 -24
  67. data/spec/model_spec.rb +15 -15
  68. data/spec/spec_helper.rb +8 -4
  69. data/spec/storage/base_spec.rb +0 -4
  70. data/spec/storage/cloudfiles_spec.rb +0 -1
  71. data/spec/storage/dropbox_spec.rb +44 -14
  72. data/spec/storage/ftp_spec.rb +26 -15
  73. data/spec/storage/local_spec.rb +83 -0
  74. data/spec/storage/ninefold_spec.rb +142 -0
  75. data/spec/storage/object_spec.rb +1 -1
  76. data/spec/storage/rsync_spec.rb +17 -7
  77. data/spec/storage/s3_spec.rb +4 -3
  78. data/spec/storage/scp_spec.rb +0 -1
  79. data/spec/storage/sftp_spec.rb +0 -1
  80. data/spec/syncer/rsync_spec.rb +8 -8
  81. metadata +62 -36
@@ -18,6 +18,10 @@ module Backup
18
18
  # Path to store backups to
19
19
  attr_accessor :path
20
20
 
21
+ ##
22
+ # use passive mode?
23
+ attr_accessor :passive_mode
24
+
21
25
  end
22
26
  end
23
27
  end
@@ -0,0 +1,17 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Configuration
5
+ module Storage
6
+ class Local < Base
7
+ class << self
8
+
9
+ ##
10
+ # Path to store backups to
11
+ attr_accessor :path
12
+
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,20 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Configuration
5
+ module Storage
6
+ class Ninefold < Base
7
+ class << self
8
+
9
+ ##
10
+ # Ninefold Credentials
11
+ attr_accessor :storage_token, :storage_secret
12
+
13
+ ##
14
+ # Ninefold path
15
+ attr_accessor :path
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -18,6 +18,10 @@ module Backup
18
18
  # Path to store backups to
19
19
  attr_accessor :path
20
20
 
21
+ ##
22
+ # Flag to use local backups
23
+ attr_accessor :local
24
+
21
25
  end
22
26
  end
23
27
  end
@@ -65,11 +65,19 @@ module Backup
65
65
  ##
66
66
  # Builds the credentials PostgreSQL syntax to authenticate the user
67
67
  # to perform the database dumping process
68
- def credential_options
68
+ def username_options
69
69
  return '' unless username.is_a?(String) and not username.empty?
70
70
  "--username='#{username}'"
71
71
  end
72
72
 
73
+ ##
74
+ # Builds the password syntax PostgreSQL uses to authenticate the user
75
+ # to perform database dumping
76
+ def password_options
77
+ return '' unless password.is_a?(String) and not username.empty?
78
+ "PGPASSWORD='#{password}'"
79
+ end
80
+
73
81
  ##
74
82
  # Builds the PostgreSQL connectivity options syntax to connect the user
75
83
  # to perform the database dumping process, socket gets gsub'd to host since
@@ -92,8 +100,9 @@ module Backup
92
100
  ##
93
101
  # Builds the full pgdump string based on all attributes
94
102
  def pgdump
95
- "#{ utility(:pg_dump) } #{ credential_options } #{ connectivity_options } " +
96
- "#{ options } #{ tables_to_dump } #{ tables_to_skip } #{ name }"
103
+ ("#{password_options} " +
104
+ "#{ utility(:pg_dump) } #{ username_options } #{ connectivity_options } " +
105
+ "#{ options } #{ tables_to_dump } #{ tables_to_skip } #{ name }").strip
97
106
  end
98
107
 
99
108
  ##
@@ -85,7 +85,7 @@ module Backup
85
85
  def invoke_save!
86
86
  response = run("#{ utility('redis-cli') } #{ credential_options } #{ connectivity_options } #{ additional_options } SAVE")
87
87
  unless response =~ /OK/
88
- Logger.error "Could not invoke the Redis SAVE command. The #{ database } file might not be contain the most recent data."
88
+ Logger.error "Could not invoke the Redis SAVE command. The #{ database } file might not contain the most recent data."
89
89
  Logger.error "Please check if the server is running, the credentials (if any) are correct, and the host/port/socket are correct."
90
90
  end
91
91
  end
@@ -98,7 +98,11 @@ module Backup
98
98
  exit
99
99
  end
100
100
 
101
+ # Temporarily remove a custom `utility_path` setting so that the system
102
+ # `cp` utility can be found, then restore the old value just in case.
103
+ old_path, self.utility_path = self.utility_path, nil
101
104
  run("#{ utility(:cp) } '#{ File.join(path, database) }' '#{ File.join(dump_path, database) }'")
105
+ self.utility_path = old_path
102
106
  end
103
107
  end
104
108
  end
@@ -18,13 +18,13 @@ module Backup
18
18
  {
19
19
  'fog' => {
20
20
  :require => 'fog',
21
- :version => '~> 0.7.0',
21
+ :version => '>= 0.11.0',
22
22
  :for => 'Amazon S3, Rackspace Cloud Files (S3, CloudFiles Storages)'
23
23
  },
24
24
 
25
25
  'dropbox' => {
26
26
  :require => 'dropbox',
27
- :version => '~> 1.2.3',
27
+ :version => '~> 1.3.0',
28
28
  :for => 'Dropbox Web Service (Dropbox Storage)'
29
29
  },
30
30
 
@@ -42,19 +42,19 @@ module Backup
42
42
 
43
43
  'net-ssh' => {
44
44
  :require => 'net/ssh',
45
- :version => '~> 2.1.3',
45
+ :version => '~> 2.1.4',
46
46
  :for => 'SSH Protocol (SSH Storage)'
47
47
  },
48
48
 
49
49
  'mail' => {
50
50
  :require => 'mail',
51
- :version => '~> 2.2.15',
51
+ :version => '~> 2.3.0',
52
52
  :for => 'Sending Emails (Mail Notifier)'
53
53
  },
54
54
 
55
55
  'twitter' => {
56
56
  :require => 'twitter',
57
- :version => '~> 1.1.2',
57
+ :version => '>= 1.7.1',
58
58
  :for => 'Sending Twitter Updates (Twitter Notifier)'
59
59
  },
60
60
 
@@ -68,7 +68,7 @@ module Backup
68
68
  :require => 'json',
69
69
  :version => '~> 1.5.1',
70
70
  :for => 'Parsing JSON for HTTParty'
71
- }
71
+ },
72
72
  }
73
73
  end
74
74
 
@@ -81,13 +81,12 @@ module Backup
81
81
  gem(name, all[name][:version])
82
82
  require(all[name][:require])
83
83
  rescue LoadError
84
- Backup::Logger.error("Dependency missing. Please install #{name} version #{all[name][:version]} and try again.")
85
- puts "\n\s\sgem install #{name} -v '#{all[name][:version]}'\n\n"
86
- puts "Dependency required for:"
84
+ Backup::Logger.error("Dependency missing.")
85
+ puts "\nDependency required for:"
87
86
  puts "\n\s\s#{all[name][:for]}"
88
- puts "\nTrying to install the #{name} gem for you.. please wait."
89
- puts "Once installed, retry the backup procedure.\n\n"
90
- puts run("#{ utility(:gem) } install #{name} -v '#{all[name][:version]}'")
87
+ puts "\nTo install the gem, issue the following command:"
88
+ puts "\n\s\sgem install #{name} -v '#{all[name][:version]}'"
89
+ puts "\nPlease try again after installing the missing dependency."
91
90
  exit
92
91
  end
93
92
  end
@@ -62,6 +62,8 @@ module Backup
62
62
  # Creates a new temp file and writes the provided public gpg key to it
63
63
  def write_tmp_file!
64
64
  @tmp_file = Tempfile.new('backup.pub')
65
+ FileUtils.chown(USER, nil, @tmp_file.path)
66
+ FileUtils.chmod(0600, @tmp_file.path)
65
67
  @tmp_file.write(key)
66
68
  @tmp_file.close
67
69
  end
@@ -0,0 +1,8 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Exception
5
+ class CommandFailed < StandardError
6
+ end
7
+ end
8
+ end
@@ -4,25 +4,24 @@ module Backup
4
4
  class Finder
5
5
  attr_accessor :trigger, :config
6
6
 
7
+ ##
8
+ # The wildcard character to match triggers
9
+ # Can be used alone or in mask (e.g. web_* )
10
+ WILDCARD = '*'
11
+
7
12
  ##
8
13
  # Initializes a new Backup::Finder object
9
14
  # and stores the path to the configuration file
10
- def initialize(trigger, config)
15
+ def initialize(trigger, config = CONFIG_FILE)
11
16
  @trigger = trigger.to_sym
12
17
  @config = config
13
18
  end
14
19
 
15
20
  ##
16
- # Tries to find and load the configuration file and return the proper
21
+ # Tries to find and return the proper
17
22
  # backup model configuration (specified by the 'trigger')
18
23
  def find
19
- unless File.exist?(config)
20
- puts "Could not find a configuration file in '#{config}'."; exit
21
- end
22
-
23
- ##
24
- # Loads the backup configuration file
25
- instance_eval(File.read(config))
24
+ load_config!
26
25
 
27
26
  ##
28
27
  # Iterates through all the instantiated backup models and returns
@@ -35,5 +34,46 @@ module Backup
35
34
 
36
35
  puts "Could not find trigger '#{trigger}' in '#{config}'."; exit
37
36
  end
37
+
38
+ ##
39
+ # Tries to find and return the all triggers
40
+ # matching wildcard (specified by the 'trigger')
41
+ def matching
42
+ ##
43
+ # Define the TIME constants unless defined
44
+ ::Backup.send(:const_set, :TIME, Time.now.strftime("%Y.%m.%d.%H.%M.%S")) unless defined? Backup::TIME
45
+
46
+ ##
47
+ # Parses the backup configuration file
48
+ load_config!
49
+
50
+ triggers = Backup::Model.all.map{|model| model.trigger.to_s }
51
+
52
+ ##
53
+ # Removes the TIME constant
54
+ ::Backup.send(:remove_const, :TIME) if defined? Backup::TIME
55
+
56
+ ##
57
+ # Make regexp replacing wildcard character by (.+)
58
+ wildcard = %r{^#{trigger.to_s.gsub(WILDCARD, '(.+)')}$}
59
+
60
+ ##
61
+ # Returns all trigger names matching wildcard
62
+ triggers.select { |trigger| trigger =~ wildcard }
63
+ end
64
+
65
+ private
66
+
67
+ ##
68
+ # Tries to find and load the configuration file
69
+ def load_config!
70
+ unless File.exist?(config)
71
+ puts "Could not find a configuration file in '#{config}'."; exit
72
+ end
73
+
74
+ ##
75
+ # Loads the backup configuration file
76
+ instance_eval(File.read(config))
77
+ end
38
78
  end
39
79
  end
@@ -56,6 +56,11 @@ module Backup
56
56
  # Example: true
57
57
  attr_accessor :enable_starttls_auto
58
58
 
59
+ ##
60
+ # OpenSSL Verify Mode
61
+ # Example: none - Only use this option for a self-signed and/or wildcard certificate
62
+ attr_accessor :openssl_verify_mode
63
+
59
64
  ##
60
65
  # Instantiates a new Backup::Notifier::Mail object
61
66
  def initialize(&block)
@@ -116,7 +121,8 @@ module Backup
116
121
  :user_name => @user_name,
117
122
  :password => @password,
118
123
  :authentication => @authentication,
119
- :enable_starttls_auto => @enable_starttls_auto
124
+ :enable_starttls_auto => @enable_starttls_auto,
125
+ :openssl_verify_mode => @openssl_verify_mode
120
126
  }
121
127
 
122
128
  ::Mail.defaults do
@@ -79,7 +79,7 @@ module Backup
79
79
  config.oauth_token = @oauth_token
80
80
  config.oauth_token_secret = @oauth_token_secret
81
81
  end
82
- @twitter_client = ::Twitter.client
82
+ @twitter_client = ::Twitter::Client.new
83
83
  end
84
84
 
85
85
  end
@@ -4,14 +4,14 @@
4
4
  # Only load the Dropbox gem when the Backup::Storage::Dropbox class is loaded
5
5
  Backup::Dependency.load('dropbox')
6
6
 
7
+ ##
8
+ # Only load the timeout library when the Backup::Storage::Dropbox class is loaded
9
+ require 'timeout'
10
+
7
11
  module Backup
8
12
  module Storage
9
13
  class Dropbox < Base
10
14
 
11
- ##
12
- # Dropbox user credentials
13
- attr_accessor :email, :password
14
-
15
15
  ##
16
16
  # Dropbox API credentials
17
17
  attr_accessor :api_key, :api_secret
@@ -29,7 +29,7 @@ module Backup
29
29
  # First it sets the defaults (if any exist) and then evaluates
30
30
  # the configuration block which may overwrite these defaults
31
31
  def initialize(&block)
32
- load_defaults!
32
+ load_defaults!(:except => ['password', 'email'])
33
33
 
34
34
  @path ||= 'backups'
35
35
 
@@ -55,18 +55,27 @@ module Backup
55
55
  private
56
56
 
57
57
  ##
58
- # Establishes a connection to Dropbox and returns the Dropbox::Session object.
59
- # Not doing any instance variable caching because this object gets persisted in YAML
60
- # format to a file and will issues. This, however has no impact on performance since it only
61
- # gets invoked once per object for a #transfer! and once for a remove! Backups run in the
62
- # background anyway so even if it were a bit slower it shouldn't matter.
58
+ # The initial connection to Dropbox will provide the user with an authorization url.
59
+ # The user must open this URL and confirm that the authorization successfully took place.
60
+ # If this is the case, then the user hits 'enter' and the session will be properly established.
61
+ # Immediately after establishing the session, the session will be serialized and written to a cache file
62
+ # in Backup::CACHE_PATH. The cached file will be used from that point on to re-establish a connection with
63
+ # Dropbox at a later time. This allows the user to avoid having to go to a new Dropbox URL to authorize over and over again.
63
64
  def connection
64
- session = ::Dropbox::Session.new(api_key, api_secret)
65
- session.mode = :dropbox
66
- session.authorizing_user = email
67
- session.authorizing_password = password
68
- session.authorize!
69
- session
65
+ if cache_exists?
66
+ begin
67
+ cached_session = ::Dropbox::Session.deserialize(File.read(cached_file))
68
+ if cached_session.authorized?
69
+ Logger.message "Session data loaded from cache!"
70
+ return cached_session
71
+ end
72
+ rescue ArgumentError => error
73
+ Logger.warn "Could not read session data from cache. Cache data might be corrupt."
74
+ end
75
+ end
76
+
77
+ Logger.message "Creating a new session!"
78
+ create_write_and_return_new_session!
70
79
  end
71
80
 
72
81
  ##
@@ -86,6 +95,74 @@ module Backup
86
95
  end
87
96
  end
88
97
 
98
+ ##
99
+ # Create a new session, write a serialized version of it to the
100
+ # .cache directory, and return the session object
101
+ def create_write_and_return_new_session!
102
+ session = ::Dropbox::Session.new(api_key, api_secret)
103
+ session.mode = :dropbox
104
+ Logger.message "Open the following URL in a browser to authorize a session for your Dropbox account:"
105
+ Logger.message ""
106
+ Logger.message "\s\s#{session.authorize_url}"
107
+ Logger.message ""
108
+ Logger.message "Once Dropbox says you're authorized, hit enter to proceed."
109
+ Timeout::timeout(180) { STDIN.gets }
110
+ begin
111
+ session.authorize
112
+ rescue OAuth::Unauthorized => error
113
+ Logger.error "Authorization failed!"
114
+ raise error
115
+ end
116
+ Logger.message "Authorized!"
117
+
118
+ Logger.message "Caching session data to file: #{cached_file}.."
119
+ write_cache!(session)
120
+ Logger.message "Cache data written! You will no longer need to manually authorize this Dropbox account via an URL on this machine."
121
+ Logger.message "Note: If you run Backup with this Dropbox account on other machines, you will need to either authorize them the same way,"
122
+ Logger.message "\s\sor simply copy over #{cached_file} to the cache directory"
123
+ Logger.message "\s\son your other machines to use this Dropbox account there as well."
124
+
125
+ session
126
+ end
127
+
128
+ ##
129
+ # Returns the path to the cached file
130
+ def cached_file
131
+ File.join(Backup::CACHE_PATH, "#{api_key + api_secret}")
132
+ end
133
+
134
+ ##
135
+ # Checks to see if the cache file exists
136
+ def cache_exists?
137
+ File.exist?(cached_file)
138
+ end
139
+
140
+ ##
141
+ # Serializes and writes the Dropbox session to a cache file
142
+ def write_cache!(session)
143
+ File.open(cached_file, "w") do |cache_file|
144
+ cache_file.write(session.serialize)
145
+ end
146
+ end
147
+
148
+ public # DEPRECATED METHODS #############################################
149
+
150
+ def email
151
+ Logger.warn "[DEPRECATED] Backup::Storage::Dropbox.email is deprecated and will be removed at some point."
152
+ end
153
+
154
+ def email=(value)
155
+ Logger.warn "[DEPRECATED] Backup::Storage::Dropbox.email= is deprecated and will be removed at some point."
156
+ end
157
+
158
+ def password
159
+ Logger.warn "[DEPRECATED] Backup::Storage::Dropbox.password is deprecated and will be removed at some point."
160
+ end
161
+
162
+ def password=(value)
163
+ Logger.warn "[DEPRECATED] Backup::Storage::Dropbox.password= is deprecated and will be removed at some point."
164
+ end
165
+
89
166
  end
90
167
  end
91
168
  end