cm-backup 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (133) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +20 -0
  3. data/bin/backup +5 -0
  4. data/lib/backup.rb +144 -0
  5. data/lib/backup/archive.rb +170 -0
  6. data/lib/backup/binder.rb +22 -0
  7. data/lib/backup/cleaner.rb +116 -0
  8. data/lib/backup/cli.rb +374 -0
  9. data/lib/backup/cloud_io/base.rb +41 -0
  10. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  11. data/lib/backup/cloud_io/s3.rb +260 -0
  12. data/lib/backup/compressor/base.rb +35 -0
  13. data/lib/backup/compressor/bzip2.rb +39 -0
  14. data/lib/backup/compressor/custom.rb +53 -0
  15. data/lib/backup/compressor/gzip.rb +74 -0
  16. data/lib/backup/config.rb +119 -0
  17. data/lib/backup/config/dsl.rb +103 -0
  18. data/lib/backup/config/helpers.rb +143 -0
  19. data/lib/backup/database/base.rb +85 -0
  20. data/lib/backup/database/mongodb.rb +187 -0
  21. data/lib/backup/database/mysql.rb +192 -0
  22. data/lib/backup/database/openldap.rb +95 -0
  23. data/lib/backup/database/postgresql.rb +133 -0
  24. data/lib/backup/database/redis.rb +179 -0
  25. data/lib/backup/database/riak.rb +82 -0
  26. data/lib/backup/database/sqlite.rb +57 -0
  27. data/lib/backup/encryptor/base.rb +29 -0
  28. data/lib/backup/encryptor/gpg.rb +747 -0
  29. data/lib/backup/encryptor/open_ssl.rb +77 -0
  30. data/lib/backup/errors.rb +58 -0
  31. data/lib/backup/logger.rb +199 -0
  32. data/lib/backup/logger/console.rb +51 -0
  33. data/lib/backup/logger/fog_adapter.rb +29 -0
  34. data/lib/backup/logger/logfile.rb +133 -0
  35. data/lib/backup/logger/syslog.rb +116 -0
  36. data/lib/backup/model.rb +479 -0
  37. data/lib/backup/notifier/base.rb +128 -0
  38. data/lib/backup/notifier/campfire.rb +63 -0
  39. data/lib/backup/notifier/command.rb +102 -0
  40. data/lib/backup/notifier/datadog.rb +107 -0
  41. data/lib/backup/notifier/flowdock.rb +103 -0
  42. data/lib/backup/notifier/hipchat.rb +118 -0
  43. data/lib/backup/notifier/http_post.rb +117 -0
  44. data/lib/backup/notifier/mail.rb +249 -0
  45. data/lib/backup/notifier/nagios.rb +69 -0
  46. data/lib/backup/notifier/pagerduty.rb +81 -0
  47. data/lib/backup/notifier/prowl.rb +68 -0
  48. data/lib/backup/notifier/pushover.rb +74 -0
  49. data/lib/backup/notifier/ses.rb +105 -0
  50. data/lib/backup/notifier/slack.rb +148 -0
  51. data/lib/backup/notifier/twitter.rb +58 -0
  52. data/lib/backup/notifier/zabbix.rb +63 -0
  53. data/lib/backup/package.rb +55 -0
  54. data/lib/backup/packager.rb +107 -0
  55. data/lib/backup/pipeline.rb +124 -0
  56. data/lib/backup/splitter.rb +76 -0
  57. data/lib/backup/storage/base.rb +69 -0
  58. data/lib/backup/storage/cloud_files.rb +158 -0
  59. data/lib/backup/storage/cycler.rb +75 -0
  60. data/lib/backup/storage/dropbox.rb +212 -0
  61. data/lib/backup/storage/ftp.rb +112 -0
  62. data/lib/backup/storage/local.rb +64 -0
  63. data/lib/backup/storage/qiniu.rb +65 -0
  64. data/lib/backup/storage/rsync.rb +248 -0
  65. data/lib/backup/storage/s3.rb +156 -0
  66. data/lib/backup/storage/scp.rb +67 -0
  67. data/lib/backup/storage/sftp.rb +82 -0
  68. data/lib/backup/syncer/base.rb +70 -0
  69. data/lib/backup/syncer/cloud/base.rb +179 -0
  70. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  71. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  72. data/lib/backup/syncer/cloud/s3.rb +110 -0
  73. data/lib/backup/syncer/rsync/base.rb +54 -0
  74. data/lib/backup/syncer/rsync/local.rb +31 -0
  75. data/lib/backup/syncer/rsync/pull.rb +51 -0
  76. data/lib/backup/syncer/rsync/push.rb +205 -0
  77. data/lib/backup/template.rb +46 -0
  78. data/lib/backup/utilities.rb +224 -0
  79. data/lib/backup/version.rb +5 -0
  80. data/templates/cli/archive +28 -0
  81. data/templates/cli/compressor/bzip2 +4 -0
  82. data/templates/cli/compressor/custom +7 -0
  83. data/templates/cli/compressor/gzip +4 -0
  84. data/templates/cli/config +123 -0
  85. data/templates/cli/databases/mongodb +15 -0
  86. data/templates/cli/databases/mysql +18 -0
  87. data/templates/cli/databases/openldap +24 -0
  88. data/templates/cli/databases/postgresql +16 -0
  89. data/templates/cli/databases/redis +16 -0
  90. data/templates/cli/databases/riak +17 -0
  91. data/templates/cli/databases/sqlite +11 -0
  92. data/templates/cli/encryptor/gpg +27 -0
  93. data/templates/cli/encryptor/openssl +9 -0
  94. data/templates/cli/model +26 -0
  95. data/templates/cli/notifier/zabbix +15 -0
  96. data/templates/cli/notifiers/campfire +12 -0
  97. data/templates/cli/notifiers/command +32 -0
  98. data/templates/cli/notifiers/datadog +57 -0
  99. data/templates/cli/notifiers/flowdock +16 -0
  100. data/templates/cli/notifiers/hipchat +16 -0
  101. data/templates/cli/notifiers/http_post +32 -0
  102. data/templates/cli/notifiers/mail +24 -0
  103. data/templates/cli/notifiers/nagios +13 -0
  104. data/templates/cli/notifiers/pagerduty +12 -0
  105. data/templates/cli/notifiers/prowl +11 -0
  106. data/templates/cli/notifiers/pushover +11 -0
  107. data/templates/cli/notifiers/ses +15 -0
  108. data/templates/cli/notifiers/slack +22 -0
  109. data/templates/cli/notifiers/twitter +13 -0
  110. data/templates/cli/splitter +7 -0
  111. data/templates/cli/storages/cloud_files +11 -0
  112. data/templates/cli/storages/dropbox +20 -0
  113. data/templates/cli/storages/ftp +13 -0
  114. data/templates/cli/storages/local +8 -0
  115. data/templates/cli/storages/qiniu +12 -0
  116. data/templates/cli/storages/rsync +17 -0
  117. data/templates/cli/storages/s3 +16 -0
  118. data/templates/cli/storages/scp +15 -0
  119. data/templates/cli/storages/sftp +15 -0
  120. data/templates/cli/syncers/cloud_files +22 -0
  121. data/templates/cli/syncers/rsync_local +20 -0
  122. data/templates/cli/syncers/rsync_pull +28 -0
  123. data/templates/cli/syncers/rsync_push +28 -0
  124. data/templates/cli/syncers/s3 +27 -0
  125. data/templates/general/links +3 -0
  126. data/templates/general/version.erb +2 -0
  127. data/templates/notifier/mail/failure.erb +16 -0
  128. data/templates/notifier/mail/success.erb +16 -0
  129. data/templates/notifier/mail/warning.erb +16 -0
  130. data/templates/storage/dropbox/authorization_url.erb +6 -0
  131. data/templates/storage/dropbox/authorized.erb +4 -0
  132. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  133. metadata +1077 -0
@@ -0,0 +1,15 @@
1
+ ##
2
+ # MongoDB [Database]
3
+ #
4
+ database MongoDB do |db|
5
+ db.name = "my_database_name"
6
+ db.username = "my_username"
7
+ db.password = "my_password"
8
+ db.host = "localhost"
9
+ db.port = 27017
10
+ db.ipv6 = false
11
+ db.only_collections = ["only", "these", "collections"]
12
+ db.additional_options = []
13
+ db.lock = false
14
+ db.oplog = false
15
+ end
@@ -0,0 +1,18 @@
1
+ ##
2
+ # MySQL [Database]
3
+ #
4
+ database MySQL do |db|
5
+ # To dump all databases, set `db.name = :all` (or leave blank)
6
+ db.name = "my_database_name"
7
+ db.username = "my_username"
8
+ db.password = "my_password"
9
+ db.host = "localhost"
10
+ db.port = 3306
11
+ db.socket = "/tmp/mysql.sock"
12
+ # Note: when using `skip_tables` with the `db.name = :all` option,
13
+ # table names should be prefixed with a database name.
14
+ # e.g. ["db_name.table_to_skip", ...]
15
+ db.skip_tables = ["skip", "these", "tables"]
16
+ db.only_tables = ["only", "these", "tables"]
17
+ db.additional_options = ["--quick", "--single-transaction"]
18
+ end
@@ -0,0 +1,24 @@
1
+ ##
2
+ # OpenLDAP [Database]
3
+ #
4
+ database OpenLDAP do |db|
5
+ # Name of the ldap backup
6
+ db.name = "my_database_name"
7
+
8
+ # Additional "slapcat" options
9
+ # defaults to an empty list of options
10
+ # db.slapcat_args = []
11
+
12
+ # run slapcat under sudo if needed
13
+ # defaults to "false"
14
+ # db.use_sudo = true
15
+
16
+ # Stores the location of the slapd.conf,
17
+ # defaults to "/etc/ldap/slapd.d" confdir
18
+ # you can use also conf file instead of directory
19
+ # db.slapcat_conf = /path/to/slapd.conf
20
+
21
+ # Optional: Use to set the location of slapcat utility
22
+ # if it cannot be found by name in your $PATH
23
+ # db.slapcat_utility = "/opt/local/bin/slapcat"
24
+ end
@@ -0,0 +1,16 @@
1
+ ##
2
+ # PostgreSQL [Database]
3
+ #
4
+ database PostgreSQL do |db|
5
+ # To dump all databases, set `db.name = :all` (or leave blank)
6
+ db.name = "my_database_name"
7
+ db.username = "my_username"
8
+ db.password = "my_password"
9
+ db.host = "localhost"
10
+ db.port = 5432
11
+ db.socket = "/tmp/pg.sock"
12
+ # When dumping all databases, `skip_tables` and `only_tables` are ignored.
13
+ db.skip_tables = ["skip", "these", "tables"]
14
+ db.only_tables = ["only", "these", "tables"]
15
+ db.additional_options = ["-xc", "-E=utf8"]
16
+ end
@@ -0,0 +1,16 @@
1
+ ##
2
+ # Redis [Database]
3
+ #
4
+ database Redis do |db|
5
+ db.mode = :copy # or :sync
6
+ # Full path to redis dump file for :copy mode.
7
+ db.rdb_path = '/var/lib/redis/dump.rdb'
8
+ # When :copy mode is used, perform a SAVE before
9
+ # copying the dump file specified by `rdb_path`.
10
+ db.invoke_save = false
11
+ db.host = 'localhost'
12
+ db.port = 6379
13
+ db.socket = '/tmp/redis.sock'
14
+ db.password = 'my_password'
15
+ db.additional_options = []
16
+ end
@@ -0,0 +1,17 @@
1
+ ##
2
+ # Riak [Database]
3
+ #
4
+ database Riak do |db|
5
+ ##
6
+ # The node from which to perform the backup.
7
+ # default: 'riak@127.0.0.1'
8
+ db.node = 'riak@hostname'
9
+ ##
10
+ # The Erlang cookie/shared secret used to connect to the node.
11
+ # default: 'riak'
12
+ db.cookie = 'cookie'
13
+ ##
14
+ # The user for the Riak instance.
15
+ # default: 'riak'
16
+ db.user = 'riak'
17
+ end
@@ -0,0 +1,11 @@
1
+ ##
2
+ # SQLite [Database]
3
+ #
4
+ database SQLite do |db|
5
+ # Path to database file
6
+ db.path = "/path/to/my/sqlite/db.sqlite"
7
+
8
+ # Optional: Use to set the location of this utility
9
+ # if it cannot be found by name in your $PATH
10
+ db.sqlitedump_utility = "/opt/local/bin/sqlite3"
11
+ end
@@ -0,0 +1,27 @@
1
+ ##
2
+ # GPG [Encryptor]
3
+ #
4
+ # Setting up #keys, as well as #gpg_homedir and #gpg_config,
5
+ # would be best set in config.rb using Encryptor::GPG.defaults
6
+ #
7
+ encrypt_with GPG do |encryption|
8
+ # Setup public keys for #recipients
9
+ encryption.keys = {}
10
+ encryption.keys['user@domain.com'] = <<-KEY
11
+ -----BEGIN PGP PUBLIC KEY BLOCK-----
12
+ Version: GnuPG v1.4.11 (Darwin)
13
+
14
+ <Your GPG Public Key Here>
15
+ -----END PGP PUBLIC KEY BLOCK-----
16
+ KEY
17
+
18
+ # Specify mode (:asymmetric, :symmetric or :both)
19
+ encryption.mode = :both # defaults to :asymmetric
20
+
21
+ # Specify recipients from #keys (for :asymmetric encryption)
22
+ encryption.recipients = ['user@domain.com']
23
+
24
+ # Specify passphrase or passphrase_file (for :symmetric encryption)
25
+ encryption.passphrase = 'a secret'
26
+ # encryption.passphrase_file = '~/backup_passphrase'
27
+ end
@@ -0,0 +1,9 @@
1
+ ##
2
+ # OpenSSL [Encryptor]
3
+ #
4
+ encrypt_with OpenSSL do |encryption|
5
+ encryption.password = "my_password" # From String
6
+ encryption.password_file = "/path/to/password/file" # Or from File
7
+ encryption.base64 = true
8
+ encryption.salt = true
9
+ end
@@ -0,0 +1,26 @@
1
+ # encoding: utf-8
2
+
3
+ ##
4
+ # Backup Generated: <%= @options[:trigger] %>
5
+ # Once configured, you can run the backup with the following command:
6
+ #
7
+ # $ backup perform -t <%= @options[:trigger] %> [-c <path_to_configuration_file>]
8
+ #
9
+ # For more information about Backup's components, see the documentation at:
10
+ # http://backup.github.io/backup
11
+ #
12
+ Model.new(:<%= @options[:trigger] %>, 'Description for <%= @options[:trigger] %>') do
13
+ <% if @options[:splitter] %>
14
+ <%= Backup::Template.new.result('cli/splitter') %>
15
+ <% end; if @options[:archives] %>
16
+ <%= Backup::Template.new.result('cli/archive') %>
17
+ <% end; %w{ databases storages syncers encryptor compressor notifiers }.each do |item|
18
+ if @options[item]
19
+ @options[item].split(',').map(&:strip).uniq.each do |entry|
20
+ if File.exist?(File.join(Backup::TEMPLATE_PATH, 'cli', item, entry)) %>
21
+ <%= Backup::Template.new.result("cli/#{ item }/#{ entry }") %>
22
+ <% end
23
+ end
24
+ end
25
+ end %>
26
+ end
@@ -0,0 +1,15 @@
1
+ ##
2
+ # Zabbix [Notifier]
3
+ #
4
+ notify_by Zabbix do |zabbix|
5
+ zabbix.on_success = true
6
+ zabbix.on_warning = true
7
+ zabbix.on_failure = true
8
+
9
+ zabbix.zabbix_host = "zabbix_server_hostname"
10
+ zabbix.zabbix_port = 10051
11
+ zabbix.service_name = "Backup trigger"
12
+ zabbix.service_host = "zabbix_host"
13
+ zabbix.item_key = "backup_status"
14
+ end
15
+
@@ -0,0 +1,12 @@
1
+ ##
2
+ # Campfire [Notifier]
3
+ #
4
+ notify_by Campfire do |campfire|
5
+ campfire.on_success = true
6
+ campfire.on_warning = true
7
+ campfire.on_failure = true
8
+
9
+ campfire.api_token = "my_api_authentication_token"
10
+ campfire.subdomain = "my_subdomain"
11
+ campfire.room_id = "my_room_id"
12
+ end
@@ -0,0 +1,32 @@
1
+ ##
2
+ # Command [Notifier]
3
+ #
4
+ notify_by Command do |cmd|
5
+ cmd.on_success = true
6
+ cmd.on_warning = true
7
+ cmd.on_failure = true
8
+
9
+ # Command to execute
10
+ cmd.command = 'notify-send'
11
+
12
+ # Arguments to pass to the command.
13
+ #
14
+ # Must be an array of strings or callable objects.
15
+ #
16
+ # Callables will be invoked with #call(model, status),
17
+ # and the return value used as the argument.
18
+ #
19
+ # In strings you can use the following placeholders:
20
+ #
21
+ # %l - Model label
22
+ # %t - Model trigger
23
+ # %s - Status (success/failure/warning)
24
+ # %v - Status verb (succeeded/failed/succeeded with warnings)
25
+ #
26
+ # All placeholders can be used with uppercase letters to capitalize
27
+ # the value.
28
+ #
29
+ # Defaults to ["%L %v"]
30
+ #
31
+ # cmd.args = ["Backup %L", "%V"]
32
+ end
@@ -0,0 +1,57 @@
1
+ ##
2
+ # DataDog [Notifier]
3
+ #
4
+ notify_by DataDog do |datadog|
5
+ datadog.on_success = true
6
+ datadog.on_warning = true
7
+ datadog.on_failure = true
8
+
9
+ datadog.api_key = 'my_api_key'
10
+
11
+ ##
12
+ # Optional
13
+ #
14
+ # Override Default Title
15
+ # Default is: "Backup #{:label}"
16
+ # datadog.title = "Backup #{:status}"
17
+ #
18
+ # Override Default Text
19
+ # Default is "Backup Notification for #{:label}"
20
+ # datadog.text = "Backup #{:status} - #{:message}"
21
+ #
22
+ # Provide a hostname to associate this backup to
23
+ # Default is nil
24
+ # datadog.host = 'db.example.com'
25
+ #
26
+ # Add Tags to the Event
27
+ # Default is nil
28
+ # valid option is an Array
29
+ # datadog.tags = ['backup', 'env:production']
30
+ #
31
+ # Override the Alert Type
32
+ # Default is based on the :status of the backup:
33
+ # :success => 'success'
34
+ # :warning => 'warning'
35
+ # :failure => 'error'
36
+ # valid options are: 'info', 'success', 'warning', 'error'
37
+ # datadog.alert_type = 'info'
38
+ #
39
+ # Add a Source Type
40
+ # Default is nil
41
+ # see api docs for valid source_type_names
42
+ # datadog.source_type_name = 'my apps'
43
+ #
44
+ # Override the Priority Level
45
+ # Default is 'normal'
46
+ # valid options are: 'normal' or 'low'
47
+ # datadog.priority = 'low'
48
+ #
49
+ # Override the Event Time (must be a unix Timestamp)
50
+ # Default is Time.now.to_i
51
+ # datadog.date_happened = Time.now.to_i
52
+ #
53
+ # Add an Aggregation Key
54
+ # Default is nil
55
+ # max length allowed is 100 characters
56
+ # datadog.aggregation_key = 'my_aggregation'
57
+ end
@@ -0,0 +1,16 @@
1
+ ##
2
+ # Flowdock [Notifier]
3
+ #
4
+ notify_by FlowDock do |flowdock|
5
+ flowdock.on_success = true
6
+ flowdock.on_warning = true
7
+ flowdock.on_failure = true
8
+
9
+ flowdock.token = "token"
10
+ flowdock.from_name = 'my_name'
11
+ flowdock.from_email = 'email@example.com'
12
+ flowdock.subject = 'My Daily Backup'
13
+ flowdock.source = 'Backup'
14
+ flowdock.tags = ['prod', 'backup']
15
+ flowdock.link = 'www.example.com'
16
+ end
@@ -0,0 +1,16 @@
1
+ ##
2
+ # Hipchat [Notifier]
3
+ #
4
+ notify_by Hipchat do |hipchat|
5
+ hipchat.on_success = true
6
+ hipchat.on_warning = true
7
+ hipchat.on_failure = true
8
+
9
+ hipchat.token = "token"
10
+ hipchat.from = "DB Backup"
11
+ hipchat.rooms_notified = ["activity"]
12
+ hipchat.success_color = "green"
13
+ hipchat.warning_color = "yellow"
14
+ hipchat.failure_color = "red"
15
+ hipchat.api_version = "v1"
16
+ end
@@ -0,0 +1,32 @@
1
+ ##
2
+ # HttpPost [Notifier]
3
+ #
4
+ notify_by HttpPost do |post|
5
+ post.on_success = true
6
+ post.on_warning = true
7
+ post.on_failure = true
8
+
9
+ # URI to post the notification to.
10
+ # Port may be specified if needed.
11
+ # If Basic Authentication is required, supply user:pass.
12
+ post.uri = 'https://user:pass@your.domain.com:8443/path'
13
+
14
+ ##
15
+ # Optional
16
+ #
17
+ # Additional headers to send.
18
+ # post.headers = { 'Authentication' => 'my_auth_info' }
19
+ #
20
+ # Additional form params to post.
21
+ # post.params = { 'auth_token' => 'my_token' }
22
+ #
23
+ # Successful response codes. Default: 200
24
+ # post.success_codes = [200, 201, 204]
25
+ #
26
+ # Defaults to true on most systems.
27
+ # Force with +true+, disable with +false+
28
+ # post.ssl_verify_peer = false
29
+ #
30
+ # Supplied by default. Override with a custom 'cacert.pem' file.
31
+ # post.ssl_ca_file = '/my/cacert.pem'
32
+ end
@@ -0,0 +1,24 @@
1
+ ##
2
+ # Mail [Notifier]
3
+ #
4
+ # The default delivery method for Mail Notifiers is 'SMTP'.
5
+ # See the documentation for other delivery options.
6
+ #
7
+ notify_by Mail do |mail|
8
+ mail.on_success = true
9
+ mail.on_warning = true
10
+ mail.on_failure = true
11
+
12
+ mail.from = "sender@email.com"
13
+ mail.to = "receiver@email.com"
14
+ mail.cc = "cc@email.com"
15
+ mail.bcc = "bcc@email.com"
16
+ mail.reply_to = "reply_to@email.com"
17
+ mail.address = "smtp.gmail.com"
18
+ mail.port = 587
19
+ mail.domain = "your.host.name"
20
+ mail.user_name = "sender@email.com"
21
+ mail.password = "my_password"
22
+ mail.authentication = "plain"
23
+ mail.encryption = :starttls
24
+ end
@@ -0,0 +1,13 @@
1
+ ##
2
+ # Nagios [Notifier]
3
+ #
4
+ notify_by Nagios do |nagios|
5
+ nagios.on_success = true
6
+ nagios.on_warning = true
7
+ nagios.on_failure = true
8
+
9
+ nagios.nagios_host = 'nagioshost'
10
+ nagios.nagios_port = 5667
11
+ nagios.service_name = 'My Backup'
12
+ nagios.service_host = 'backuphost'
13
+ end
@@ -0,0 +1,12 @@
1
+ ##
2
+ # PagerDuty [Notifier]
3
+ #
4
+
5
+ notify_by PagerDuty do |pagerduty|
6
+ pagerduty.on_success = true
7
+ pagerduty.on_warning = true
8
+ pagerduty.on_failure = true
9
+
10
+ pagerduty.service_key = '0123456789abcdef01234567890abcde'
11
+ pagerduty.resolve_on_warning = true
12
+ end
@@ -0,0 +1,11 @@
1
+ ##
2
+ # Prowl [Notifier]
3
+ #
4
+ notify_by Prowl do |prowl|
5
+ prowl.on_success = true
6
+ prowl.on_warning = true
7
+ prowl.on_failure = true
8
+
9
+ prowl.application = "my_application"
10
+ prowl.api_key = "my_api_key"
11
+ end