jetpants 0.7.8 → 0.7.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,106 @@
1
+ # JetCollins monkeypatches to add Collins integration
2
+
3
+ module Jetpants
4
+ class Shard < Pool
5
+
6
+ ##### JETCOLLINS MIX-IN ####################################################
7
+
8
+ include Plugin::JetCollins
9
+
10
+ collins_attr_accessor :shard_min_id, :shard_max_id, :shard_state, :shard_parent
11
+
12
+ # Returns a Collins::Asset for this pool
13
+ def collins_asset(create_if_missing=false)
14
+ selector = {
15
+ operation: 'and',
16
+ details: true,
17
+ type: 'CONFIGURATION',
18
+ primary_role: 'MYSQL_SHARD',
19
+ shard_min_id: "^#{@min_id}$",
20
+ shard_max_id: "^#{@max_id}$",
21
+ }
22
+ selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup']
23
+
24
+ results = Plugin::JetCollins.find selector
25
+
26
+ # If we got back multiple results, try ignoring the remote datacenter ones
27
+ if results.count > 1
28
+ filtered_results = results.select {|a| a.location.nil? || a.location.upcase == Plugin::JetCollins.datacenter}
29
+ results = filtered_results if filtered_results.count > 0
30
+ end
31
+
32
+ if results.count > 1
33
+ raise "Multiple configuration assets found for pool #{name}"
34
+ elsif results.count == 0 && create_if_missing
35
+ output "Could not find configuration asset for pool; creating now"
36
+ new_tag = 'mysql-' + @name
37
+ asset = Collins::Asset.new type: 'CONFIGURATION', tag: new_tag, status: 'Allocated'
38
+ begin
39
+ Plugin::JetCollins.create!(asset)
40
+ rescue
41
+ collins_set asset: asset,
42
+ status: 'Allocated'
43
+ end
44
+ collins_set asset: asset,
45
+ primary_role: 'MYSQL_SHARD',
46
+ pool: @name.upcase,
47
+ shard_min_id: @min_id,
48
+ shard_max_id: @max_id
49
+ Plugin::JetCollins.get new_tag
50
+ elsif results.count == 0 && !create_if_missing
51
+ raise "Could not find configuration asset for pool #{name}"
52
+ else
53
+ results.first
54
+ end
55
+ end
56
+
57
+
58
+ ##### METHOD OVERRIDES #####################################################
59
+
60
+ # Examines the current state of the pool (as known to Jetpants) and updates
61
+ # Collins to reflect this, in terms of the pool's configuration asset as
62
+ # well as the individual hosts.
63
+ def sync_configuration
64
+ status = case
65
+ when in_config? then 'Allocated'
66
+ when @state == :deprecated then 'Cancelled'
67
+ when @state == :recycle then 'Decommissioned'
68
+ else 'Provisioning'
69
+ end
70
+ collins_set asset: collins_asset(true),
71
+ status: status,
72
+ shard_state: @state.to_s.upcase,
73
+ shard_parent: @parent ? @parent.name : ''
74
+ if @state == :recycle
75
+ [@master, @master.slaves].flatten.each do |db|
76
+ db.collins_pool = ''
77
+ db.collins_secondary_role = ''
78
+ db.collins_status = 'Unallocated'
79
+ end
80
+ else
81
+ # Note that we don't call Pool#slaves here to get all 3 types in one shot,
82
+ # because that potentially includes child shards, and we don't want to overwrite
83
+ # their pool setting...
84
+ [@master, active_slaves, standby_slaves, backup_slaves].flatten.each do |db|
85
+ current_status = (db.collins_status || '').downcase
86
+ db.collins_status = 'Allocated' unless current_status == 'maintenance'
87
+ db.collins_pool = @name
88
+ end
89
+ @master.collins_secondary_role = 'MASTER'
90
+
91
+ standby_slaves.each {|db| db.collins_secondary_role = 'STANDBY_SLAVE'}
92
+ backup_slaves.each {|db| db.collins_secondary_role = 'BACKUP_SLAVE'}
93
+ end
94
+ true
95
+ end
96
+
97
+
98
+ ##### CALLBACKS ############################################################
99
+
100
+ # After altering the state of a shard, sync the change to Collins immediately.
101
+ def after_state=(value)
102
+ sync_configuration
103
+ end
104
+
105
+ end
106
+ end
@@ -0,0 +1,239 @@
1
+ # JetCollins monkeypatches to add Collins integration
2
+
3
+ require 'yaml'
4
+ require 'json'
5
+
6
+ module Jetpants
7
+ class Topology
8
+
9
+ ##### METHOD OVERRIDES #####################################################
10
+
11
+ # IMPORTANT NOTE
12
+ # This plugin does NOT implement write_config, since this format of
13
+ # your app configuration file entirely depends on your web framework!
14
+ #
15
+ # You will have to implement this yourself in a separate plugin; recommended
16
+ # approach is to add serialization methods to Pool and Shard, and call it
17
+ # on each @pool, writing out to a file or pinging a config service, depending
18
+ # on whatever your application uses.
19
+
20
+
21
+ # Initializes list of pools + shards from Collins
22
+ def load_pools
23
+ # We keep a cache of Collins::Asset objects, organized as pool_name => role => [asset, asset, ...]
24
+ @pool_role_assets = {}
25
+
26
+ # Populate the cache for all master and active_slave nodes. (We restrict to these types
27
+ # because this is sufficient for creating Pool objects and generating app config files.)
28
+ Jetpants.topology.server_node_assets(false, :master, :active_slave)
29
+
30
+ functional_partition_assets = []
31
+ shard_assets = []
32
+ configuration_assets('MYSQL_POOL', 'MYSQL_SHARD').each do |asset|
33
+ if asset.primary_role.upcase == 'MYSQL_SHARD'
34
+ shard_assets << asset
35
+ else
36
+ functional_partition_assets << asset
37
+ end
38
+ end
39
+
40
+ functional_partition_assets.sort_by! {|a| (a.config_sort_order || 0).to_i}
41
+ shard_assets.sort_by! {|a| a.shard_min_id.to_i}
42
+ @pools = functional_partition_assets.map(&:to_pool) + shard_assets.map(&:to_pool)
43
+ @pools.compact! # remove nils from pools that had no master
44
+
45
+ # Set up parent/child relationships between shards currently being split.
46
+ # We do this in a separate step afterwards so that Topology#pool can find the parent
47
+ # by name, regardless of ordering in Collins
48
+ @pools.select {|p| p.has_parent}.each do |p|
49
+ parent = pool(p.has_parent) or raise "Cannot find parent shard named #{p.has_parent}"
50
+ parent.add_child(p)
51
+ end
52
+ true
53
+ end
54
+
55
+
56
+ # Returns (count) DB objects. Pulls from machines in the Provisioned status
57
+ # and converts them to the Allocated status.
58
+ # You can pass in :role to request spares with a particular secondary_role
59
+ def claim_spares(count, options={})
60
+ assets = query_spare_assets(count, options)
61
+ raise "Not enough spare machines available! Found #{assets.count}, needed #{count}" if assets.count < count
62
+ assets.map do |asset|
63
+ db = asset.to_db
64
+ db.collins_pool = ''
65
+ db.collins_secondary_role = ''
66
+ db.collins_slave_weight = ''
67
+ db.collins_status = 'Allocated'
68
+ db
69
+ end
70
+ end
71
+
72
+
73
+ # This method won't ever return a number higher than 100, but that's
74
+ # not a problem, since no single operation requires that many spares
75
+ def count_spares(options={})
76
+ query_spare_assets(100, options).count
77
+ end
78
+
79
+
80
+ ##### NEW METHODS ##########################################################
81
+
82
+ # Returns an array of Collins::Asset objects meeting the given criteria.
83
+ # Caches the result for subsequent use.
84
+ # Optionally supply a pool name to restrict the result to that pool.
85
+ # Optionally supply one or more role symbols (:master, :active_slave,
86
+ # :standby_slave, :backup_slave) to filter the result to just those
87
+ # SECONDARY_ROLE values in Collins.
88
+ def server_node_assets(pool_name=false, *roles)
89
+ roles = normalize_roles(roles) if roles.count > 0
90
+
91
+ # Check for previously-cached result. (Only usable if a pool_name supplied.)
92
+ if pool_name && @pool_role_assets[pool_name]
93
+ if roles.count > 0 && roles.all? {|r| @pool_role_assets[pool_name].has_key? r}
94
+ return roles.map {|r| @pool_role_assets[pool_name][r]}.flatten
95
+ elsif roles.count == 0 && valid_roles.all? {|r| @pool_role_assets[pool_name].has_key? r}
96
+ return @pool_role_assets[pool_name].values.flatten
97
+ end
98
+ end
99
+
100
+ per_page = 200
101
+ selector = {
102
+ operation: 'and',
103
+ details: true,
104
+ size: per_page,
105
+ query: 'primary_role = ^DATABASE$ AND type = ^SERVER_NODE$'
106
+ }
107
+ selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup']
108
+ selector[:query] += " AND pool = ^#{pool_name}$" if pool_name
109
+ if roles.count == 1
110
+ selector[:query] += " AND secondary_role = ^#{roles.first}$"
111
+ elsif roles.count > 1
112
+ values = roles.map {|r| "secondary_role = ^#{r}$"}
113
+ selector[:query] += ' AND (' + values.join(' OR ') + ')'
114
+ end
115
+
116
+ assets = []
117
+ done = false
118
+ page = 0
119
+
120
+ # Query Collins one or more times, until we've seen all the results
121
+ until done do
122
+ selector[:page] = page
123
+ results = Plugin::JetCollins.find selector.dup # find apparently alters the selector object now, so we dup it
124
+ done = results.count < per_page
125
+ page += 1
126
+ assets.concat results
127
+ end
128
+
129
+ # Next we need to update our @pool_role_assets cache. But first let's set it to [] for each pool/role
130
+ # that we queried. This intentionally nukes any previous cached data, and also allows us to differentiate
131
+ # between an empty result and a cache miss.
132
+ roles = valid_roles if roles.count == 0
133
+ seen_pools = assets.map {|a| a.pool.downcase}
134
+ seen_pools << pool_name if pool_name
135
+ seen_pools.uniq.each do |p|
136
+ @pool_role_assets[p] ||= {}
137
+ roles.each {|r| @pool_role_assets[p][r] = []}
138
+ end
139
+
140
+ # Filter
141
+ assets.select! {|a| a.pool && a.secondary_role && %w(allocated maintenance).include?(a.status.downcase)}
142
+
143
+ # Cache
144
+ assets.each {|a| @pool_role_assets[a.pool.downcase][a.secondary_role.downcase.to_sym] << a}
145
+
146
+ # Return
147
+ assets
148
+ end
149
+
150
+
151
+ # Returns an array of configuration assets with the supplied primary role(s)
152
+ def configuration_assets(*primary_roles)
153
+ raise "Must supply at least one primary_role" if primary_roles.count < 1
154
+ per_page = 200
155
+
156
+ selector = {
157
+ operation: 'and',
158
+ details: true,
159
+ size: per_page,
160
+ }
161
+
162
+ if primary_roles.count == 1
163
+ selector[:type] = '^CONFIGURATION$'
164
+ else
165
+ values = primary_roles.map {|r| "primary_role = ^#{r}$"}
166
+ selector[:query] = 'type = ^CONFIGURATION$ AND (' + values.join(' OR ') + ')'
167
+ end
168
+
169
+ selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup']
170
+
171
+ done = false
172
+ page = 0
173
+ assets = []
174
+ until done do
175
+ selector[:page] = page
176
+ page_of_results = Plugin::JetCollins.find selector.dup # find apparently alters the selector object now, so we dup it
177
+ assets += page_of_results
178
+ done = page_of_results.count < per_page
179
+ page += 1
180
+ end
181
+
182
+ # If remote lookup is enabled, remove the remote copy of any pool that exists
183
+ # in both local and remote datacenters.
184
+ if Jetpants.plugins['jetpants_collins']['remote_lookup']
185
+ dc_pool_map = {Plugin::JetCollins.datacenter => {}}
186
+
187
+ assets.each do |a|
188
+ location = a.location || Plugin::JetCollins.datacenter
189
+ pool = a.pool ? a.pool.downcase : a.tag[6..-1].downcase # if no pool, strip 'mysql-' off front and use that
190
+ dc_pool_map[location] ||= {}
191
+ dc_pool_map[location][pool] = a
192
+ end
193
+
194
+ # Grab everything from current DC first (higher priority over other
195
+ # datacenters), then grab everything from remote DCs.
196
+ final_assets = dc_pool_map[Plugin::JetCollins.datacenter].values
197
+ dc_pool_map.each do |dc, pool_to_assets|
198
+ next if dc == Plugin::JetCollins.datacenter
199
+ pool_to_assets.each do |pool, a|
200
+ final_assets << a unless dc_pool_map[Plugin::JetCollins.datacenter][pool]
201
+ end
202
+ end
203
+ assets = final_assets
204
+ end
205
+
206
+ # Remove decommissioned nodes
207
+ assets.reject {|a| a.status == 'Decommissioned'}
208
+ end
209
+
210
+
211
+ def clear_asset_cache(pool_name=false)
212
+ if pool_name
213
+ @pool_role_assets.delete pool_name
214
+ else
215
+ @pool_role_assets = {}
216
+ end
217
+ end
218
+
219
+
220
+ private
221
+
222
+ # Helper method to query Collins for spare provisioned DBs.
223
+ def query_spare_assets(count, options={})
224
+ # Intentionally no remoteLookup=true here. We only want to grab spare nodes
225
+ # from the datacenter that Jetpants is running in.
226
+ selector = {
227
+ operation: 'and',
228
+ details: true,
229
+ type: 'SERVER_NODE',
230
+ status: 'Provisioned',
231
+ primary_role: 'DATABASE',
232
+ size: count,
233
+ }
234
+ selector[:secondary_role] = options[:role].to_s.downcase if options[:role]
235
+ Plugin::JetCollins.find selector
236
+ end
237
+
238
+ end
239
+ end
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: jetpants
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease:
5
- version: 0.7.8
5
+ version: 0.7.10
6
6
  platform: ruby
7
7
  authors:
8
8
  - Evan Elias
@@ -11,7 +11,7 @@ autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
13
 
14
- date: 2012-09-25 00:00:00 Z
14
+ date: 2012-10-31 00:00:00 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: mysql2
@@ -90,6 +90,17 @@ dependencies:
90
90
  version: "1.2"
91
91
  type: :runtime
92
92
  version_requirements: *id007
93
+ - !ruby/object:Gem::Dependency
94
+ name: collins_client
95
+ prerelease: false
96
+ requirement: &id008 !ruby/object:Gem::Requirement
97
+ none: false
98
+ requirements:
99
+ - - ~>
100
+ - !ruby/object:Gem::Version
101
+ version: 0.2.7
102
+ type: :runtime
103
+ version_requirements: *id008
93
104
  description: Jetpants is an automation toolkit for handling monstrously large MySQL database topologies. It is geared towards common operational tasks like cloning slaves, rebalancing shards, and performing master promotions. It features a command suite for easy use by operations staff, though it's also a full Ruby library for use in developing custom migration scripts and database automation.
94
105
  email:
95
106
  - me@evanelias.com
@@ -100,41 +111,50 @@ extensions: []
100
111
 
101
112
  extra_rdoc_files:
102
113
  - README.rdoc
114
+ - doc/faq.rdoc
103
115
  - doc/plugins.rdoc
116
+ - doc/jetpants_collins.rdoc
117
+ - doc/requirements.rdoc
104
118
  - doc/configuration.rdoc
105
- - doc/faq.rdoc
106
119
  - doc/commands.rdoc
107
- - doc/requirements.rdoc
108
120
  files:
109
121
  - Gemfile
110
122
  - README.rdoc
123
+ - doc/faq.rdoc
111
124
  - doc/plugins.rdoc
125
+ - doc/jetpants_collins.rdoc
126
+ - doc/requirements.rdoc
112
127
  - doc/configuration.rdoc
113
- - doc/faq.rdoc
114
128
  - doc/commands.rdoc
115
- - doc/requirements.rdoc
116
- - lib/jetpants/callback.rb
129
+ - lib/jetpants.rb
117
130
  - lib/jetpants/topology.rb
118
- - lib/jetpants/db/server.rb
119
- - lib/jetpants/db/state.rb
131
+ - lib/jetpants/db.rb
132
+ - lib/jetpants/host.rb
133
+ - lib/jetpants/table.rb
134
+ - lib/jetpants/callback.rb
120
135
  - lib/jetpants/db/import_export.rb
121
- - lib/jetpants/db/privileges.rb
136
+ - lib/jetpants/db/state.rb
137
+ - lib/jetpants/db/server.rb
122
138
  - lib/jetpants/db/client.rb
139
+ - lib/jetpants/db/privileges.rb
123
140
  - lib/jetpants/db/replication.rb
124
141
  - lib/jetpants/shard.rb
125
- - lib/jetpants/db.rb
126
- - lib/jetpants/host.rb
127
142
  - lib/jetpants/pool.rb
128
143
  - lib/jetpants/monkeypatch.rb
129
- - lib/jetpants/table.rb
130
- - lib/jetpants.rb
131
144
  - bin/jetpants
145
+ - plugins/simple_tracker/commandsuite.rb
132
146
  - plugins/simple_tracker/topology.rb
133
- - plugins/simple_tracker/shard.rb
134
- - plugins/simple_tracker/simple_tracker.rb
135
147
  - plugins/simple_tracker/db.rb
148
+ - plugins/simple_tracker/shard.rb
136
149
  - plugins/simple_tracker/pool.rb
137
- - plugins/simple_tracker/commandsuite.rb
150
+ - plugins/simple_tracker/simple_tracker.rb
151
+ - plugins/jetpants_collins/jetpants_collins.rb
152
+ - plugins/jetpants_collins/topology.rb
153
+ - plugins/jetpants_collins/db.rb
154
+ - plugins/jetpants_collins/host.rb
155
+ - plugins/jetpants_collins/asset.rb
156
+ - plugins/jetpants_collins/shard.rb
157
+ - plugins/jetpants_collins/pool.rb
138
158
  - etc/jetpants.yaml.sample
139
159
  homepage: https://github.com/tumblr/jetpants/
140
160
  licenses: []