mongolly 0.2.10 → 0.2.11

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0e5dfd7e55a93db2f0ab03b50b7676d97b25a212
4
- data.tar.gz: 26304958d35572763588aeb6521f6f86037caa9f
3
+ metadata.gz: f0eff9f2f6fa9f6d84af21df7c18e5e5d5d245d6
4
+ data.tar.gz: 34f7bc7b32932fba31b23b5a59b860137f712492
5
5
  SHA512:
6
- metadata.gz: 19cfd8db4e216c2789aeb4e4099ab7c1e6e7812631ee2c0f64b032cf0537cd9b871f9188b286d89f98258dab9e362c5fdfdb87f43c16172c1e5893a6ee8fba48
7
- data.tar.gz: d9538c0a1fcd46158a1bee43a662174191347f8f0661316a896b3916326382095a411508fdb327682a87e1317009b794fab0b4d8f1dd92959cf93f6c25f98869
6
+ metadata.gz: 3b6a9a24534f3a0a02e82c89646b397a47b2ef7435677948b9765c5bf8ae2497cca6022b45df00c719f44006922269fad91ab051c1768b982894bbc28bf7186e
7
+ data.tar.gz: 1c76a9c0696fafe6321bf0d6e97efcb0a688860d0cefda2f51d34168ec4cea84af89f4cc38d833afc26622739b7acedae72787d1504b5aedeb41b763f707cc94
@@ -0,0 +1,74 @@
1
+ AllCops:
2
+ DisplayCopNames: true
3
+ DisplayStyleGuide: true
4
+ TargetRubyVersion: 2.2
5
+
6
+ Style/AlignParameters:
7
+ EnforcedStyle: with_fixed_indentation
8
+
9
+ Style/DotPosition:
10
+ EnforcedStyle: leading
11
+
12
+ Style/Encoding:
13
+ Enabled: false
14
+
15
+ Style/EmptyLinesAroundClassBody:
16
+ Enabled: false
17
+
18
+ Style/ExtraSpacing:
19
+ Enabled: false
20
+
21
+ Style/GlobalVars:
22
+ AllowedVariables: [$statsd, $rollout, $rails_rake_task]
23
+
24
+ Style/FileName:
25
+ Enabled: false
26
+
27
+ Style/StringLiterals:
28
+ EnforcedStyle: double_quotes
29
+
30
+ Metrics/AbcSize:
31
+ Enabled: false
32
+
33
+ Metrics/CyclomaticComplexity:
34
+ Enabled: false
35
+
36
+ Metrics/MethodLength:
37
+ Enabled: false
38
+
39
+ Metrics/LineLength:
40
+ Enabled: false
41
+
42
+ Metrics/BlockLength:
43
+ Enabled: true
44
+ ExcludedMethods: ['describe', 'context']
45
+ Exclude:
46
+ - 'Rakefile'
47
+ - '**/*.rake'
48
+ - 'spec/**/*.rb'
49
+
50
+ Lint/AssignmentInCondition:
51
+ Enabled: true
52
+
53
+ Style/ClassAndModuleChildren:
54
+ Enabled: false
55
+
56
+ Rails:
57
+ Enabled: true
58
+
59
+ Rails/ActionFilter:
60
+ Enabled: false
61
+
62
+ Rails/HttpPositionalArguments:
63
+ Enabled: false
64
+
65
+ Rails/SkipsModelValidations:
66
+ Enabled: true
67
+ Exclude:
68
+ - 'spec/**/*.rb'
69
+
70
+ Style/FrozenStringLiteralComment:
71
+ Enabled: true
72
+
73
+ Performance/CaseWhenSplat:
74
+ Enabled: false
@@ -1,40 +1,41 @@
1
1
  #!/usr/bin/env ruby
2
- require 'thor'
3
- require 'yaml'
4
- require 'time'
5
- require 'mongolly'
2
+ require "thor"
3
+ require "yaml"
4
+ require "time"
5
+ require "mongolly"
6
6
 
7
7
  module Mongolly
8
8
  class Runner < Thor
9
- class_option :config, :type => :string, aliases: '-c', default: '~/.mongolly', desc: 'Path to config file'
9
+ class_option :config, type: :string, aliases: "-c", default: "~/.mongolly", desc: "Path to config file"
10
10
 
11
11
  desc "backup", "Snapshots the Database EBS Volumes"
12
- method_option :dry_run, type: :boolean, desc: 'Step through command without changes'
12
+ method_option :dry_run, type: :boolean, desc: "Step through command without changes"
13
13
  def backup
14
- Shepherd.new({dry_run: options[:dry_run]}.merge(config)).backup
14
+ Shepherd.new({ dry_run: options[:dry_run] }.merge(config)).backup
15
15
  end
16
16
 
17
17
  desc "clean", "Removes old Database EBS Snapshots"
18
- method_option :age, aliases: '-a', required: true
19
- method_option :dry_run, type: :boolean, desc: 'Step through command without changes'
18
+ method_option :age, aliases: "-a", required: true
19
+ method_option :dry_run, type: :boolean, desc: "Step through command without changes"
20
20
  def clean
21
- age = Time.parse(options[:age])
22
- Shepherd.new({dry_run: options[:dry_run]}.merge(config)).cleanup(age)
21
+ age = Time.parse.utc(options[:age])
22
+ Shepherd.new({ dry_run: options[:dry_run] }.merge(config)).cleanup(age)
23
23
  end
24
24
 
25
- private
25
+ private
26
+
26
27
  def config
27
28
  @config ||= read_config
28
29
  end
29
30
 
30
31
  def read_config
31
- unless File.exists? config_path
32
+ unless File.exist? config_path
32
33
  seed_config
33
34
  exit 1
34
35
  end
35
36
 
36
37
  begin
37
- cfg = YAML::load( File.read( config_path ) )
38
+ cfg = YAML.load(File.read(config_path))
38
39
  rescue e
39
40
  puts " ** Unable to read config at #{config_path}"
40
41
  raise e
@@ -50,16 +51,16 @@ module Mongolly
50
51
  db_password: nil,
51
52
  access_key_id: nil,
52
53
  secret_access_key: nil,
53
- region: 'us-east-1',
54
- log_level: 'info',
54
+ region: "us-east-1",
55
+ log_level: "info",
55
56
  mongo_start_command: nil,
56
57
  mongo_stop_command: nil,
57
58
  config_server_ssh_user: nil,
58
59
  config_server_ssh_keypath: nil
59
60
  }
60
61
 
61
- File.open( config_path, "w" ) do |f|
62
- f.write( empty_config.to_yaml )
62
+ File.open(config_path, "w") do |f|
63
+ f.write(empty_config.to_yaml)
63
64
  end
64
65
 
65
66
  puts " ** An empty configuration file has been written to #{config_path}."
@@ -70,13 +71,13 @@ module Mongolly
70
71
 
71
72
  def validated_config(cfg)
72
73
  %w(database access_key_id secret_access_key).each do |arg|
73
- raise ArgumentError.new("#{arg} cannot be empty") if cfg[arg.to_sym].to_s.strip.empty?
74
+ raise ArgumentError, "#{arg} cannot be empty" if cfg[arg.to_sym].to_s.strip.empty?
74
75
  end
75
76
  cfg
76
77
  end
77
78
 
78
79
  def config_path
79
- config_path ||= File.expand_path(options[:config])
80
+ @config_path ||= File.expand_path(options[:config])
80
81
  end
81
82
  end
82
83
  end
@@ -1,11 +1,11 @@
1
- require 'mongolly/extensions/aws/ec2/instance'
2
- require 'mongolly/extensions/aws/ec2/instance_collection'
3
- require 'mongolly/extensions/bson/timestamp'
4
- require 'mongolly/extensions/mongo/mongo_client'
5
- require 'mongolly/extensions/mongo/mongo_replica_set_client'
1
+ require "mongolly/extensions/aws/ec2/instance"
2
+ require "mongolly/extensions/aws/ec2/instance_collection"
3
+ require "mongolly/extensions/bson/timestamp"
4
+ require "mongolly/extensions/mongo/mongo_client"
5
+ require "mongolly/extensions/mongo/mongo_replica_set_client"
6
6
 
7
- require 'mongolly/version'
8
- require 'mongolly/shepherd'
7
+ require "mongolly/version"
8
+ require "mongolly/shepherd"
9
9
 
10
10
  module Mongolly
11
11
  end
@@ -1,6 +1,5 @@
1
- require 'extensions/aws/ec2/instance'
2
- require 'extensions/aws/ec2/instance_collection'
3
- require 'extensions/bson/timestamp'
4
- require 'extensions/mongo/mongo_client'
5
- require 'extensions/mongo/mongo_replica_set_client'
6
-
1
+ require "extensions/aws/ec2/instance"
2
+ require "extensions/aws/ec2/instance_collection"
3
+ require "extensions/bson/timestamp"
4
+ require "extensions/mongo/mongo_client"
5
+ require "extensions/mongo/mongo_replica_set_client"
@@ -1,13 +1,13 @@
1
- require 'aws-sdk'
1
+ require "aws-sdk"
2
2
 
3
3
  class AWS::EC2::Instance
4
4
 
5
5
  def volumes_with_tag(key)
6
6
  volumes = []
7
- attachments.each do |name,attachment|
7
+ attachments.each do |_, attachment|
8
8
  next unless attachment.status == :attached
9
9
  volume = attachment.volume
10
- volumes << volume if volume.status == :in_use && volume.tags.has_key?(key)
10
+ volumes << volume if volume.status == :in_use && volume.tags.key?(key)
11
11
  end
12
12
  volumes
13
13
  end
@@ -1,30 +1,31 @@
1
- require 'aws-sdk'
2
- require 'socket'
3
- require 'ipaddress'
1
+ require "aws-sdk"
2
+ require "socket"
3
+ require "ipaddress"
4
4
 
5
5
  class AWS::EC2::InstanceCollection
6
6
 
7
- def find_from_address(address, port = 27107)
8
- ip_address = convert_address_to_ip(address, port)
7
+ def find_from_address(address, port = 27107) # rubocop:disable Style/NumericLiterals
8
+ ip_address = convert_address_to_ip(address, port)
9
9
 
10
- instances = select do |instance|
11
- instance.public_ip_address == ip_address || instance.private_ip_address == ip_address
12
- end
13
- raise error_class "InstanceNotFound" if instances.length != 1
14
-
15
- return instances.first
16
- rescue SocketError
17
- raise RuntimeError.new("Unable to determine IP address from #{address}:#{port}")
10
+ instances = select do |instance|
11
+ instance.public_ip_address == ip_address || instance.private_ip_address == ip_address
18
12
  end
13
+ raise error_class "InstanceNotFound" if instances.length != 1
14
+
15
+ return instances.first
16
+ rescue SocketError
17
+ raise RuntimeError.new("Unable to determine IP address from #{address}:#{port}")
18
+ end
19
+
20
+ private
19
21
 
20
- private
21
22
  def convert_address_to_ip(address, port)
22
23
  return address if ::IPAddress.valid? address
23
24
 
24
25
  ip_addresses = ::Addrinfo.getaddrinfo(address, port, nil, :STREAM)
25
- raise error_class "MultipleIpAddressFound" if ip_addresses.length > 1
26
+ raise error_class "MultipleIpAddressFound" if ip_addresses.length > 1
26
27
 
27
- return ip_addresses[0].ip_address
28
+ ip_addresses[0].ip_address
28
29
  end
29
30
 
30
31
  end
@@ -1,4 +1,4 @@
1
- require 'mongo'
1
+ require "mongo"
2
2
 
3
3
  class BSON::Timestamp
4
4
  include Comparable
@@ -1,18 +1,19 @@
1
- require 'mongo'
2
- require 'logger'
3
- require 'net/ssh'
4
- require 'retries'
1
+ require "mongo"
2
+ require "logger"
3
+ require "net/ssh"
4
+ require "retries"
5
5
 
6
6
  class Mongo::MongoClient
7
- MAX_DISABLE_BALANCER_WAIT = 60*8 # 8 Minutes
8
- REPLICA_SNAPSHOT_THRESHOLD = 60*5 # 5 Minutes
7
+ MAX_DISABLE_BALANCER_WAIT = 60 * 8 # 8 Minutes
8
+ REPLICA_SNAPSHOT_THRESHOLD = 60 * 5 # 5 Minutes
9
9
  REPLICA_SNAPSHOT_PREFER_HIDDEN = true
10
+ DEFAULT_MONGO_PORT = 27017 # rubocop: disable Style/NumericLiterals
10
11
 
11
- def snapshot_ebs(options={})
12
+ def snapshot_ebs(options = {})
12
13
  @mongolly_dry_run = options[:dry_run] || false
13
14
  @mongolly_logger = options[:logger] || Logger.new(STDOUT)
14
- options[:volume_tag] ||= 'mongolly'
15
- options[:backup_key] ||= (0...8).map{65.+(rand(25)).chr}.join
15
+ options[:volume_tag] ||= "mongolly"
16
+ options[:backup_key] ||= (0...8).map { 65.+(rand(25)).chr }.join
16
17
 
17
18
  @ec2 = AWS::EC2.new(access_key_id: options[:access_key_id], secret_access_key: options[:secret_access_key], region: options[:region])
18
19
 
@@ -20,131 +21,93 @@ class Mongo::MongoClient
20
21
  @mongolly_logger.info("Detected sharded cluster")
21
22
  with_disabled_balancing do
22
23
  with_config_server_stopped(options) do
23
- backup_instance(config_server, options, false)
24
+ backup_instance(config_server, options)
24
25
 
25
- shards.each do |name,hosts|
26
+ shards.each do |name, hosts|
26
27
  @mongolly_logger.debug("Found Shard #{name} with hosts #{hosts}.")
27
28
  replica_set_connection(hosts, options).snapshot_ebs(options)
28
29
  end
29
30
  end
30
31
  end
31
32
  else
32
- backup_instance(snapshot_ebs_target(REPLICA_SNAPSHOT_THRESHOLD, REPLICA_SNAPSHOT_PREFER_HIDDEN), options, false )
33
+ backup_instance(snapshot_ebs_target(REPLICA_SNAPSHOT_THRESHOLD, REPLICA_SNAPSHOT_PREFER_HIDDEN), options.merge(strict_connection: true))
33
34
  end
34
35
  end
35
36
 
36
- protected
37
- def snapshot_ebs_target(threshold=nil, prefer_hidden=nil)
38
- host_port.join(':')
39
- end
40
-
41
- def backup_instance(address, options, lock = true)
42
- host, port = address.split(':')
43
- instance = @ec2.instances.find_from_address(host, port)
44
-
45
- @mongolly_logger.info("Backing up instance #{instance.id} from #{host}:#{port}")
46
-
47
- volumes = instance.volumes_with_tag(options[:volume_tag])
37
+ protected
48
38
 
49
- @mongolly_logger.debug("Found target volumes #{volumes.map(&:id).join(', ')} ")
50
-
51
- raise RuntimeError.new "no suitable volumes found" unless volumes.length > 0
52
-
53
- # Force lock with multiple volumes
54
- lock = true if volumes.length > 1
55
-
56
- backup_block = proc do
57
- volumes.each do |volume|
58
- @mongolly_logger.debug("Snapshotting #{volume.id} with tag #{options[:backup_key]}")
59
- unless @mongolly_dry_run
60
- snapshot = volume.create_snapshot("#{options[:backup_key]} #{Time.now} mongolly #{host}")
61
- snapshot.add_tag('created_at', value: Time.now)
62
- snapshot.add_tag('backup_key', value: options[:backup_key])
63
- end
64
- end
65
- end
66
-
67
- if lock
68
- with_database_locked &backup_block
69
- else
70
- backup_block.call
71
- end
39
+ def snapshot_ebs_target(_threshold = nil, _prefer_hidden = nil)
40
+ host_port.join(":")
72
41
  end
73
42
 
74
43
  def disable_balancing
75
44
  @mongolly_logger.debug "Disabling Shard Balancing"
76
- self['config'].collection('settings').update({_id: 'balancer'}, {'$set' => {stopped: true}}, upsert: true) unless @mongolly_dry_run
45
+ self["config"].collection("settings").update({ _id: "balancer" }, { "$set" => { stopped: true } }, upsert: true) unless @mongolly_dry_run
77
46
  end
78
47
 
79
48
  def enable_balancing
80
49
  @mongolly_logger.debug "Enabling Shard Balancing"
81
- retry_logger = Proc.new do |exception, attempt_number, total_delay|
50
+ retry_logger = proc do |_, attempt_number, total_delay|
82
51
  @mongolly_logger.debug "Error enabling balancing (config server not up?); retry attempt #{attempt_number}; #{total_delay} seconds have passed."
83
52
  end
84
53
  with_retries(max_tries: 5, handler: retry_logger, rescue: Mongo::OperationFailure, base_sleep_seconds: 5, max_sleep_seconds: 120) do
85
- self['config'].collection('settings').update({_id: 'balancer'}, {'$set' => {stopped: false}}, upsert: true) unless @mongolly_dry_run
54
+ self["config"].collection("settings").update({ _id: "balancer" }, { "$set" => { stopped: false } }, upsert: true) unless @mongolly_dry_run
86
55
  end
87
56
  end
88
57
 
89
58
  def balancer_active?
90
- self['config'].collection('locks').find({_id: 'balancer', state: {'$ne' => 0}}).count > 0
59
+ self["config"].collection("locks").find(_id: "balancer", state: { "$ne" => 0 }).count > 0
91
60
  end
92
61
 
93
62
  def config_server
94
63
  unless @config_server
95
- @config_server = self['admin'].command( { getCmdLineOpts: 1 } )["parsed"]["sharding"]["configDB"].split(",").sort.first.split(":").first
64
+ @config_server = self["admin"].command(getCmdLineOpts: 1)["parsed"]["sharding"]["configDB"].split(",").sort.first.split(":").first
96
65
  @mongolly_logger.debug "Found config server #{@config_server}"
97
66
  end
98
- return @config_server
67
+ @config_server
99
68
  end
100
69
 
101
- def with_config_server_stopped(options={})
102
- begin
103
- # Stop Config Server
104
- ssh_command(options[:config_server_ssh_user], config_server, options[:mongo_stop_command], options[:config_server_ssh_keypath])
105
- yield
106
- rescue => ex
107
- @mongolly_logger.error "Error with config server stopped: #{ex.to_s}"
108
- ensure
109
- # Start Config Server
110
- ssh_command(options[:config_server_ssh_user], config_server, options[:mongo_start_command], options[:config_server_ssh_keypath])
111
- end
70
+ def with_config_server_stopped(options = {})
71
+ # Stop Config Server
72
+ ssh_command(options[:config_server_ssh_user], config_server, options[:mongo_stop_command], options[:config_server_ssh_keypath])
73
+ yield
74
+ rescue => ex
75
+ @mongolly_logger.error "Error with config server stopped: #{ex}"
76
+ ensure
77
+ # Start Config Server
78
+ ssh_command(options[:config_server_ssh_user], config_server, options[:mongo_start_command], options[:config_server_ssh_keypath])
112
79
  end
113
80
 
114
81
  def with_disabled_balancing
115
- begin
116
- disable_balancing
117
- term_time = Time.now + MAX_DISABLE_BALANCER_WAIT
118
- while !@mongolly_dry_run && (Time.now < term_time) && balancer_active?
119
- @mongolly_logger.info "Balancer active, sleeping for 10s (#{(term_time - Time.now).round}s remaining)"
120
- sleep 10
121
- end
122
- if !@mongolly_dry_run && balancer_active?
123
- raise RuntimeError.new "Unable to disable balancer within #{MAX_DISABLE_BALANCER_WAIT}s"
124
- end
125
- @mongolly_logger.debug "With shard balancing disabled..."
126
- yield
127
- rescue => ex
128
- @mongolly_logger.error "Error with disabled balancer: #{ex.to_s}"
129
- ensure
130
- enable_balancing
131
- end
82
+ disable_balancing
83
+ term_time = Time.now.utc + MAX_DISABLE_BALANCER_WAIT
84
+ while !@mongolly_dry_run && (Time.now.utc < term_time) && balancer_active?
85
+ @mongolly_logger.info "Balancer active, sleeping for 10s (#{(term_time - Time.now.utc).round}s remaining)"
86
+ sleep 10
87
+ end
88
+ if !@mongolly_dry_run && balancer_active?
89
+ raise "Unable to disable balancer within #{MAX_DISABLE_BALANCER_WAIT}s"
90
+ end
91
+ @mongolly_logger.debug "With shard balancing disabled..."
92
+ yield
93
+ rescue => ex
94
+ @mongolly_logger.error "Error with disabled balancer: #{ex}"
95
+ ensure
96
+ enable_balancing
132
97
  end
133
98
 
134
99
  def with_database_locked
135
- begin
136
- @mongolly_logger.debug "Locking database..."
137
- disable_profiling
138
- lock! unless @mongolly_dry_run || locked?
139
- @mongolly_logger.debug "With database locked..."
140
- yield
141
- rescue => ex
142
- @mongolly_logger.error "Error with database locked: #{ex.to_s}"
143
- ensure
144
- @mongolly_logger.debug "Unlocking database..."
145
- unlock! if !@mongolly_dry_run && locked?
146
- enable_profiling
147
- end
100
+ @mongolly_logger.debug "Locking database..."
101
+ disable_profiling
102
+ lock! unless @mongolly_dry_run || locked?
103
+ @mongolly_logger.debug "With database locked..."
104
+ yield
105
+ rescue => ex
106
+ @mongolly_logger.error "Error with database locked: #{ex}"
107
+ ensure
108
+ @mongolly_logger.debug "Unlocking database..."
109
+ unlock! if !@mongolly_dry_run && locked?
110
+ enable_profiling
148
111
  end
149
112
 
150
113
  def disable_profiling
@@ -156,7 +119,7 @@ protected
156
119
  unless self[db].profiling_level == :off
157
120
  @mongolly_logger.debug("Disabling profiling for #{db}, level #{self[db].profiling_level}")
158
121
  @profiled_dbs[db] = self[db].profiling_level
159
- self[db].profiling_level = :off unless @mongolly_dry_run
122
+ self[db].profiling_level = :off unless @mongolly_dry_run
160
123
  end
161
124
  rescue Mongo::InvalidNSName
162
125
  @mongolly_logger.debug("Skipping database #{db} due to invalid name")
@@ -169,61 +132,101 @@ protected
169
132
  @mongolly_logger.debug("Database locked, can't turn on profiling")
170
133
  return false
171
134
  end
135
+
172
136
  unless @profiled_dbs
173
137
  @monglly_logger.debug("No dbs in @profiled_dbs")
174
138
  return true
175
139
  end
176
140
 
177
- @profiled_dbs.each do |db,level|
141
+ @profiled_dbs.each do |db, level|
178
142
  begin
179
143
  @mongolly_logger.debug("Enabling profiling for #{db}, level #{level}")
180
- self[db].profiling_level = level unless @mongolly_dry_run
144
+ self[db].profiling_level = level unless @mongolly_dry_run
181
145
  rescue Mongo::InvalidNSName
182
146
  @mongolly_logger.debug("Skipping database #{db} due to invalid name")
183
147
  end
184
148
  end
185
- return true
149
+ true
186
150
  end
187
151
 
188
152
  def shards
189
153
  shards = {}
190
- self['config']['shards'].find().each do |shard|
191
- shards[shard['_id']] = shard['host'].split("/")[1].split(",")
154
+ self["config"]["shards"].find.each do |shard|
155
+ shards[shard["_id"]] = shard["host"].split("/")[1].split(",")
192
156
  end
193
157
  shards
194
158
  end
195
159
 
196
160
  def replica_set_connection(hosts, options)
197
161
  db = Mongo::MongoReplicaSetClient.new(hosts)
198
- db['admin'].authenticate(options[:db_username], options[:db_password]) if options[:db_username]
199
- return db
162
+ db["admin"].authenticate(options[:db_username], options[:db_password]) if options[:db_username]
163
+ db
200
164
  end
201
165
 
202
166
  def ssh_command(user, host, command, keypath = nil)
203
167
  @mongolly_logger.debug("Running #{command} on #{host} as #{user}")
204
168
  return if @mongolly_dry_run
205
169
  exit_code = nil
206
- output = ''
170
+ output = ""
207
171
  Net::SSH.start(host, user.strip, keys: keypath) do |ssh|
208
172
  channel = ssh.open_channel do |ch|
209
173
  ch.request_pty
210
- ch.exec(command.strip) do |ch, success|
211
- raise "Unable to exec #{command.strip} on #{host}" unless success
174
+ ch.exec(command.strip) do |_, success|
175
+ raise "Unable to exec #{command.strip} on #{host}" unless success
212
176
  end
213
177
  end
214
- channel.on_request("exit-status") do |ch,data|
178
+ channel.on_request("exit-status") do |_, data|
215
179
  exit_code = data.read_long
216
180
  end
217
- channel.on_extended_data do |ch,type,data|
181
+ channel.on_extended_data do |_, _, data|
218
182
  output += data
219
183
  end
220
184
  ssh.loop
221
185
  end
222
186
 
223
187
  if exit_code != 0
224
- raise RuntimeError.new "Unable to exec #{command} on #{host}, #{output}"
188
+ raise "Unable to exec #{command} on #{host}, #{output}"
225
189
  end
226
190
  end
227
191
 
192
+ private
193
+
194
+ def backup_instance(address, options)
195
+ host, port = address.split(":")
196
+ port ||= DEFAULT_MONGO_PORT
197
+
198
+ # Ensure we're directly connected to the target node for backup
199
+ # This prevents a subclassed replica set from still acting against the
200
+ # primary
201
+ if options[:strict_connection] && (self.host != host || self.port.to_i != port.to_i)
202
+ return Mongo::MongoClient.new(host, port.to_i, slave_ok: true).snapshot_ebs(options)
203
+ end
204
+
205
+ instance = @ec2.instances.find_from_address(host, port)
206
+
207
+ @mongolly_logger.info("Backing up instance #{instance.id} from #{host}:#{port}")
208
+
209
+ volumes = instance.volumes_with_tag(options[:volume_tag])
210
+
211
+ @mongolly_logger.debug("Found target volumes #{volumes.map(&:id).join(', ')} ")
212
+
213
+ raise "no suitable volumes found" if volumes.empty?
214
+
215
+ backup_block = proc do
216
+ volumes.each do |volume|
217
+ @mongolly_logger.debug("Snapshotting #{volume.id} with tag #{options[:backup_key]}")
218
+ next if @mongolly_dry_run
219
+ snapshot = volume.create_snapshot("#{options[:backup_key]} #{Time.now.utc} mongolly #{host}")
220
+ snapshot.add_tag("created_at", value: Time.now.utc)
221
+ snapshot.add_tag("backup_key", value: options[:backup_key])
222
+ end
223
+ end
224
+
225
+ if volumes.length > 1
226
+ with_database_locked(&backup_block)
227
+ else
228
+ backup_block.call
229
+ end
230
+ end
228
231
 
229
232
  end
@@ -1,17 +1,17 @@
1
- require 'mongo'
1
+ require "mongo"
2
2
 
3
3
  class Mongo::MongoReplicaSetClient
4
4
 
5
5
  def most_current_secondary(threshold = 0, prefer_hidden = true)
6
- replica = self['admin'].command( replSetGetStatus: 1 )
7
- secondaries = replica['members'].select { |m| m['state'] == 2 }.sort_by { |m| [m['optime'], m['name']] }
6
+ replica = self["admin"].command(replSetGetStatus: 1)
7
+ secondaries = replica["members"].select { |m| m["state"] == 2 }.sort_by { |m| [m["optime"], m["name"]] }
8
8
  most_current = secondaries.first
9
9
 
10
- hidden = self['local']['system']['replset'].find_one['members'].select { |mem| mem['hidden'] }.map { |mem| mem['host'] }
10
+ hidden = self["local"]["system"]["replset"].find_one["members"].select { |mem| mem["hidden"] }.map { |mem| mem["host"] }
11
11
 
12
- if prefer_hidden && !hidden.include?(most_current['name'])
12
+ if prefer_hidden && !hidden.include?(most_current["name"])
13
13
  secondaries[1..-1].each do |secondary|
14
- if hidden.include?(secondary['name']) && (most_current['optime'] - secondary['optime']) < threshold
14
+ if hidden.include?(secondary["name"]) && (most_current["optime"] - secondary["optime"]) < threshold
15
15
  most_current = secondary
16
16
  break
17
17
  end
@@ -19,10 +19,11 @@ class Mongo::MongoReplicaSetClient
19
19
  end
20
20
 
21
21
  @mongolly_logger.debug("Found most current secondary #{most_current['name']}, hidden: #{hidden.include? most_current['name']}")
22
- most_current['name']
22
+ most_current["name"]
23
23
  end
24
24
 
25
- protected
25
+ protected
26
+
26
27
  def snapshot_ebs_target(threshold = 0, prefer_hidden = true)
27
28
  most_current_secondary(threshold, prefer_hidden)
28
29
  end
@@ -1,34 +1,33 @@
1
1
  module Mongolly
2
2
  class Shepherd
3
-
4
- def initialize(options={})
3
+ def initialize(options = {})
5
4
  @options = options
6
5
  @access_key_id = options[:access_key_id]
7
6
  @secret_access_key = options[:secret_access_key]
8
- @region = options[:region] || 'us-east-1'
7
+ @region = options[:region] || "us-east-1"
9
8
  @database = options[:database]
10
9
  @db_username = options[:db_username]
11
10
  @db_password = options[:db_password]
12
11
  @dry_run = options[:dry_run]
13
12
  @logger = options[:logger] || Logger.new(STDOUT)
14
13
  @logger.level = case options[:log_level].strip
15
- when 'fatal'; then Logger::FATAL
16
- when 'error'; then Logger::ERROR
17
- when 'warn' ; then Logger::WARN
18
- when 'debug'; then Logger::DEBUG
19
- else Logger::INFO
14
+ when "fatal"; then Logger::FATAL
15
+ when "error"; then Logger::ERROR
16
+ when "warn"; then Logger::WARN
17
+ when "debug"; then Logger::DEBUG
18
+ else Logger::INFO
20
19
  end
21
20
  end
22
21
 
23
22
  def backup
24
23
  @logger.info "Starting backup..."
25
- connection.snapshot_ebs({logger: @logger}.merge(@options))
24
+ connection.snapshot_ebs({ logger: @logger }.merge(@options))
26
25
  @logger.info "Backup complete."
27
26
  end
28
27
 
29
28
  def cleanup(age)
30
29
  @logger.info "Starting cleanup..."
31
- raise ArgumentError.new("Must provide a Time object to cleanup") unless age.class <= Time
30
+ raise ArgumentError, "Must provide a Time object to cleanup" unless age.class <= Time
32
31
 
33
32
  ec2 = AWS::EC2.new(access_key_id: @access_key_id,
34
33
  secret_access_key: @secret_access_key,
@@ -36,11 +35,10 @@ module Mongolly
36
35
 
37
36
  @logger.debug "deleting snapshots older than #{age}}"
38
37
  ec2.snapshots.with_owner(:self).each do |snapshot|
39
- unless snapshot.tags[:created_at].nil? || snapshot.tags[:backup_key].nil?
40
- if Time.parse(snapshot.tags[:created_at]) < age
41
- @logger.debug "deleting snapshot #{snapshot.id} tagged #{snapshot.tags[:backup_key]} created at #{snapshot.tags[:created_at]}, earlier than #{age}"
42
- snapshot.delete unless @dry_run
43
- end
38
+ next if snapshot.tags[:created_at].nil? || snapshot.tags[:backup_key].nil?
39
+ if Time.parse.utc(snapshot.tags[:created_at]) < age
40
+ @logger.debug "deleting snapshot #{snapshot.id} tagged #{snapshot.tags[:backup_key]} created at #{snapshot.tags[:created_at]}, earlier than #{age}"
41
+ snapshot.delete unless @dry_run
44
42
  end
45
43
  end
46
44
  @logger.info "Cleanup complete."
@@ -48,14 +46,16 @@ module Mongolly
48
46
 
49
47
  def connection
50
48
  db = if @database.is_a? Array
51
- @logger.debug "connecting to a replica set #{@database}"
52
- Mongo::MongoReplicaSetClient.new(@database)
53
- else
54
- @logger.debug "connecting to a single instance #{@database}"
55
- Mongo::MongoClient.new(*@database.split(':'))
49
+ @logger.debug "connecting to a replica set #{@database}"
50
+ Mongo::MongoReplicaSetClient.new(@database)
51
+ else
52
+ @logger.debug "connecting to a single instance #{@database}"
53
+ Mongo::MongoClient.new(*@database.split(":"))
54
+ end
55
+ if @db_username && @db_password
56
+ db["admin"].authenticate(@db_username, @db_password)
56
57
  end
57
- db['admin'].authenticate(@db_username, @db_password) if @db_username && @db_password
58
- return db
58
+ db
59
59
  end
60
60
  end
61
61
  end
@@ -1,3 +1,3 @@
1
1
  module Mongolly
2
- VERSION = "0.2.10"
2
+ VERSION = "0.2.11".freeze
3
3
  end
@@ -1,28 +1,27 @@
1
1
  # -*- encoding: utf-8 -*-
2
- lib = File.expand_path('../lib', __FILE__)
2
+ lib = File.expand_path("../lib", __FILE__)
3
3
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
- require 'mongolly/version'
4
+ require "mongolly/version"
5
5
 
6
6
  Gem::Specification.new do |gem|
7
7
  gem.name = "mongolly"
8
8
  gem.version = Mongolly::VERSION
9
9
  gem.authors = ["Michael Saffitz"]
10
10
  gem.email = ["m@saffitz.com"]
11
- gem.description = %q{Easy backups for EBS-based MongoDB Databases}
12
- gem.summary = %q{Easy backups for EBS-based MongoDB Databases}
11
+ gem.description = "Easy backups for EBS-based MongoDB Databases"
12
+ gem.summary = "Easy backups for EBS-based MongoDB Databases"
13
13
  gem.homepage = "http://www.github.com/msaffitz/mongolly"
14
14
 
15
- gem.files = `git ls-files`.split($/)
16
- gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
15
+ gem.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
16
+ gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
17
17
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
18
18
  gem.require_paths = ["lib"]
19
19
 
20
20
  gem.add_dependency("thor")
21
- gem.add_dependency("mongo")
21
+ gem.add_dependency("mongo", "~>1")
22
22
  gem.add_dependency("bson_ext")
23
23
  gem.add_dependency("aws-sdk", "~>1")
24
24
  gem.add_dependency("ipaddress")
25
25
  gem.add_dependency("net-ssh")
26
26
  gem.add_dependency("retries")
27
-
28
27
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mongolly
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.10
4
+ version: 0.2.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Michael Saffitz
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-01-28 00:00:00.000000000 Z
11
+ date: 2017-07-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor
@@ -28,16 +28,16 @@ dependencies:
28
28
  name: mongo
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ">="
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '0'
33
+ version: '1'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ">="
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '0'
40
+ version: '1'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: bson_ext
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -117,6 +117,7 @@ extensions: []
117
117
  extra_rdoc_files: []
118
118
  files:
119
119
  - ".gitignore"
120
+ - ".rubocop.yml"
120
121
  - Gemfile
121
122
  - LICENSE.txt
122
123
  - README.md
@@ -151,9 +152,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
151
152
  version: '0'
152
153
  requirements: []
153
154
  rubyforge_project:
154
- rubygems_version: 2.2.3
155
+ rubygems_version: 2.4.5.1
155
156
  signing_key:
156
157
  specification_version: 4
157
158
  summary: Easy backups for EBS-based MongoDB Databases
158
159
  test_files: []
159
- has_rdoc: