cf_light_api 2.0.0.pre1 → 2.0.0.pre2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: bdbf36a622784f98ea2133973966ae40eb500572
4
- data.tar.gz: 8b12c0cec0408a0f16d31392836eea9286f62fd7
3
+ metadata.gz: d66b56d2709d066bd7bdf1a87ebbf8b1e240a219
4
+ data.tar.gz: 6aebb9a1303ba0e48927e361a38ac1796b87a983
5
5
  SHA512:
6
- metadata.gz: 6d65857061da6b5b87dd19928b5386ef2d339649bae6989cec2a981607cc09d28b3f8be72a2d2c97039517a5151e53ec9e50a55e290618e4c9e918bb27eaeb12
7
- data.tar.gz: c5f6a83db24dbe514f9ce1272824274ac052f71b4f1296462793b4925ad290053273be7e60b21a6715e3555cca98b58823607b9679a2e1b880bc2451dd77f6e3
6
+ metadata.gz: 2030c3344aea542a4faee1e364f5c2242fea81f764baa41dfc3236db8635fda2f88ef5827b8d0f17640dfa2bcbeae8af77e37649f17bf03373bfa7560ed16372
7
+ data.tar.gz: f2730633fc1827d02937eda58b7b591fea38e0604dc8357d17caf34747639caff977f232a646feb0a97acb04373ddad93f165efd9799c4b7168a882ecbf219c2
@@ -0,0 +1,60 @@
1
+ def formatted_instance_stats_for_app app
2
+ instances = cf_rest("/v2/apps/#{app['metadata']['guid']}/stats")[0]
3
+ raise "Unable to retrieve app instance stats: '#{instances['error_code']}'" if instances['error_code']
4
+ instances.map{|key,value|value}
5
+ end
6
+
7
+ def cf_rest(path, method='GET')
8
+ @logger.info "Making #{method} request for #{path}..."
9
+
10
+ resources = []
11
+ response = JSON.parse(@cf_client.base.rest_client.request(method, path)[1][:body])
12
+
13
+ # Some endpoints return a 'resources' array, others are flat, depending on the path.
14
+ if response['resources']
15
+ resources << response['resources']
16
+ else
17
+ resources << response
18
+ end
19
+
20
+ # Handle the pagination by recursing over myself until we get a response which doesn't contain a 'next_url'
21
+ # at which point all the resources are returned up the stack and flattened.
22
+ resources << cf_rest(response['next_url'], method) unless response['next_url'] == nil
23
+ resources.flatten
24
+ end
25
+
26
+ def get_client(cf_api=ENV['CF_API'], cf_user=ENV['CF_USER'], cf_password=ENV['CF_PASSWORD'])
27
+ client = CFoundry::Client.get(cf_api)
28
+ client.login({:username => cf_user, :password => cf_password})
29
+ client
30
+ end
31
+
32
+ def send_instance_usage_data_to_graphite(instance_stats, org, space, app_name)
33
+ app_name.gsub! ".", "_" # Some apps have dots in the app name
34
+
35
+ instance_stats.each_with_index do |instance_data, index|
36
+ graphite_base_key = "cf_apps.#{org}.#{space}.#{app_name}.#{index}"
37
+ @logger.info " Exporting app instance \##{index} usage statistics to Graphite, path '#{graphite_base_key}'"
38
+
39
+ # Quota data
40
+ ['mem_quota', 'disk_quota'].each do |key|
41
+ @graphite.metrics "#{graphite_base_key}.#{key}" => instance_data['stats'][key]
42
+ end
43
+
44
+ # Usage data
45
+ ['mem', 'disk', 'cpu'].each do |key|
46
+ @graphite.metrics "#{graphite_base_key}.#{key}" => instance_data['stats']['usage'][key]
47
+ end
48
+ end
49
+ end
50
+
51
+ def put_in_redis(key, data)
52
+ REDIS.set key, data.to_json
53
+ end
54
+
55
+ def format_duration(elapsed_seconds)
56
+ seconds = elapsed_seconds % 60
57
+ minutes = (elapsed_seconds / 60) % 60
58
+ hours = elapsed_seconds / (60 * 60)
59
+ format("%02d hrs, %02d mins, %02d secs", hours, minutes, seconds)
60
+ end
@@ -1,12 +1,13 @@
1
1
  require 'cfoundry'
2
2
  require 'json'
3
3
  require 'rufus-scheduler'
4
- require 'parallel'
5
4
  require 'redlock'
6
5
  require 'logger'
7
6
  require 'graphite-api'
8
7
  require 'date'
9
8
 
9
+ require_relative 'lib.rb'
10
+
10
11
  @logger = Logger.new(STDOUT)
11
12
  @logger.formatter = proc do |severity, datetime, progname, msg|
12
13
  "#{datetime} [cf_light_api:worker]: #{msg}\n"
@@ -19,62 +20,34 @@ end
19
20
  end
20
21
  end
21
22
 
22
- PARALLEL_MAPS = (ENV['PARALLEL_MAPS'] || 4 ).to_i
23
+ # If either of the Graphite settings are set, verify that they are both set, or exit with an error.
24
+ if ENV['GRAPHITE_HOST'] or ENV['GRAPHITE_PORT']
25
+ ['GRAPHITE_HOST', 'GRAPHITE_PORT'].each do |env|
26
+ unless ENV[env]
27
+ @logger.info "Error: please set the '#{env}' environment variable to enable exporting to Graphite."
28
+ exit 1
29
+ end
30
+ end
31
+ end
32
+
23
33
  UPDATE_INTERVAL = (ENV['UPDATE_INTERVAL'] || '5m').to_s # If you change the default '5m' here, also remember to change the default age validity in sinatra/cf_light_api.rb:31
24
34
  UPDATE_TIMEOUT = (ENV['UPDATE_TIMEOUT'] || '5m').to_s
25
35
 
26
36
  lock_manager = Redlock::Client.new([ENV['REDIS_URI']])
27
37
  scheduler = Rufus::Scheduler.new
28
38
 
29
- @logger.info "Parallel maps: '#{PARALLEL_MAPS}'"
30
39
  @logger.info "Update interval: '#{UPDATE_INTERVAL}'"
31
40
  @logger.info "Update timeout: '#{UPDATE_TIMEOUT}'"
32
- @logger.info "Graphite server: #{ENV['GRAPHITE']}" if ENV['GRAPHITE']
33
-
34
- #@domains = {}
35
- # @domains = cf_rest('/v2/domains?results-per-page=100') # We're retrieving this from the app instance 'uris' key instead.
36
- # @routes = cf_rest('/v2/routes?results-per-page=100')
37
-
38
- # def formatted_routes_for_app app
39
- # routes = cf_rest(app['entity']['routes_url'])
40
-
41
- # routes.collect do |route|
42
- # domain = @domains.find{|a_domain| a_domain['metadata']['guid'] == route['entity']['domain_guid']}['entity']['name']
43
- # # Suffix the hostname with a period for concatenation, unless it's blank (which can happen for apex routes).
44
- # host = route['entity']['host'] != '' ? "#{route['entity']['host']}." : ''
45
- # path = route['entity']['path']
46
- # "#{host}#{domain}#{path}"
47
- # end
48
- # end
49
-
50
- def formatted_instance_stats_for_app app
51
- instances = cf_rest("/v2/apps/#{app['metadata']['guid']}/stats")[0]
52
- raise "Unable to retrieve app instance stats: '#{instances['error_code']}'" if instances['error_code']
53
- instances.map{|key,value|value}
54
- end
55
-
56
- def cf_rest(path, method='GET')
57
- @logger.info "Making #{method} request for #{path}..."
58
41
 
59
- resources = []
60
- response = JSON.parse(@cf_client.base.rest_client.request(method, path)[1][:body])
61
-
62
- # Some endpoints return a 'resources' array, others are flat, depending on the path.
63
- if response['resources']
64
- resources << response['resources']
65
- else
66
- resources << response
67
- end
68
-
69
- # Handle the pagination by recursing over myself until we get a response which doesn't contain a 'next_url'
70
- # at which point all the resources are returned up the stack and flattened.
71
- resources << cf_rest(response['next_url'], method) unless response['next_url'] == nil
72
- resources.flatten
42
+ if ENV['GRAPHITE_HOST'] and ENV['GRAPHITE_PORT']
43
+ @logger.info "Graphite server: #{ENV['GRAPHITE_HOST']}:#{ENV['GRAPHITE_PORT']}"
44
+ else
45
+ @logger.info 'Graphite server: Disabled'
73
46
  end
74
47
 
75
48
  scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout => UPDATE_TIMEOUT do
76
49
  @cf_client = nil
77
- # graphite = GraphiteAPI.new(graphite: ENV['GRAPHITE']) if ENV['GRAPHITE']
50
+ @graphite = GraphiteAPI.new(graphite: "#{ENV['GRAPHITE_HOST']}:#{ENV['GRAPHITE_PORT']}") if ENV['GRAPHITE_HOST'] and ENV['GRAPHITE_PORT']
78
51
 
79
52
  begin
80
53
  lock_manager.lock("#{ENV['REDIS_KEY_PREFIX']}:lock", 5*60*1000) do |lock|
@@ -91,9 +64,6 @@ scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout
91
64
  @spaces = cf_rest('/v2/spaces?results-per-page=100')
92
65
  @stacks = cf_rest('/v2/stacks?results-per-page=100')
93
66
 
94
- # org_data = get_org_data(@cf_client)
95
- # app_data = get_app_data(@cf_client, graphite)
96
-
97
67
  formatted_orgs = @orgs.map do |org|
98
68
  quota = @quotas.find{|a_quota| a_quota['metadata']['guid'] == org['entity']['quota_definition_guid']}
99
69
 
@@ -110,7 +80,7 @@ scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout
110
80
  end
111
81
 
112
82
  formatted_apps = @apps.map do |app|
113
- # TODO: This is a bit repetative, could improve.
83
+ # TODO: This is a bit repetative, could maybe improve?
114
84
  space = @spaces.find{|a_space| a_space['metadata']['guid'] == app['entity']['space_guid']}
115
85
  org = @orgs.find{|an_org| an_org['metadata']['guid'] == space['entity']['organization_guid']}
116
86
  stack = @stacks.find{|a_stack| a_stack['metadata']['guid'] == app['entity']['stack_guid']}
@@ -124,23 +94,28 @@ scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout
124
94
  :space => space['entity']['name'],
125
95
  :stack => stack['entity']['name'],
126
96
  :buildpack => app['entity']['buildpack'],
127
- # This requires a call to /v2/apps/[guid]/routes for each app, or we can just use the 'uris' key from /v2/apps/[guid]/stats
128
- # which we have to call anyway, to get app instance usage stats..
129
- # :routes => running ? formatted_routes_for_app(app) : [],
130
97
  :data_from => Time.now.to_i,
131
98
  :last_uploaded => app['metadata']['updated_at'] ? DateTime.parse(app['metadata']['updated_at']).strftime('%Y-%m-%d %T %z') : nil
132
99
  }
133
100
 
101
+ # Add additional data, such as instance usage statistics, and routes - but this is only possible
102
+ # if the instance is running.
134
103
  additional_data = {}
104
+
135
105
  begin
136
106
  instance_stats = []
137
107
  routes = []
138
108
  if running
109
+ # Finds the first running app instance that has a set of routes, in case there are stopped/crashed app instances that don't have any routes.
139
110
  instance_stats = formatted_instance_stats_for_app(app)
140
- # Finds the first running app instance that has a set of routes, in case there are stopped/crashed app instances that don't have any.
141
111
  running_instances = instance_stats.select{|instance| instance['stats']['uris'] if instance['state'] == 'RUNNING'}
142
112
  raise "Unable to retrieve app routes - no app instances are running." if running_instances.empty?
113
+
143
114
  routes = running_instances.first['stats']['uris']
115
+
116
+ if @graphite
117
+ send_instance_usage_data_to_graphite(instance_stats, org['entity']['name'], space['entity']['name'], app['entity']['name'])
118
+ end
144
119
  end
145
120
 
146
121
  additional_data = {
@@ -149,7 +124,11 @@ scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout
149
124
  :routes => routes,
150
125
  :error => nil
151
126
  }
127
+
152
128
  rescue => e
129
+ # Most exceptions here will be caused by the app or one of the instances being in a non-standard state,
130
+ # for example, trying to query an app which was present when the worker began updating, but was stopped
131
+ # before we reached this section, so we just catch all exceptions, log the reason and move on.
153
132
  @logger.info " #{org['entity']['name']} #{space['entity']['name']}: '#{app['entity']['name']}' error: #{e.message}"
154
133
  additional_data = {
155
134
  :running => 'error',
@@ -178,118 +157,3 @@ scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout
178
157
  @cf_client.logout
179
158
  end
180
159
  end
181
-
182
- def get_client(cf_api=ENV['CF_API'], cf_user=ENV['CF_USER'], cf_password=ENV['CF_PASSWORD'])
183
- client = CFoundry::Client.get(cf_api)
184
- client.login({:username => cf_user, :password => cf_password})
185
- client
186
- end
187
-
188
- # def send_data_to_graphite(data, graphite)
189
- # org = data[:org]
190
- # space = data[:space]
191
- # name = data[:name].sub ".", "_" # Some apps have dots in the app name
192
-
193
- # data[:instances].each_with_index do |instance_data, index|
194
- # graphite_base_key = "cf_apps.#{org}.#{space}.#{name}.#{index}"
195
-
196
- # # Quota data
197
- # [:mem_quota, :disk_quota].each do |key|
198
- # graphite.metrics "#{graphite_base_key}.#{key}" => instance_data[:stats][key]
199
- # end
200
-
201
- # # Usage data
202
- # [:mem, :disk, :cpu].each do |key|
203
- # graphite.metrics "#{graphite_base_key}.#{key}" => instance_data[:stats][:usage][key]
204
- # end
205
- # end
206
- # end
207
-
208
- # def get_app_data(cf_client, graphite)
209
- # Parallel.map(cf_client.organizations, :in_threads=> PARALLEL_MAPS) do |org|
210
- # org_name = org.name
211
- # Parallel.map(org.spaces, :in_threads => PARALLEL_MAPS) do |space|
212
- # space_name = space.name
213
- # @logger.info "Getting app data for apps in #{org_name}:#{space_name}..."
214
- # Parallel.map(space.apps, :in_threads=> PARALLEL_MAPS) do |app|
215
- # begin
216
- # # It's possible for an app to have been terminated before this stage is reached.
217
- # formatted_app_data = format_app_data(app, org_name, space_name)
218
- # if graphite
219
- # send_data_to_graphite(formatted_app_data, graphite)
220
- # end
221
- # formatted_app_data
222
- # rescue CFoundry::AppNotFound
223
- # next
224
- # end
225
- # end
226
- # end
227
- # end.flatten.compact
228
- # end
229
-
230
- # def get_org_data(cf_client)
231
- # Parallel.map( cf_client.organizations, :in_threads=> PARALLEL_MAPS) do |org|
232
- # @logger.info "Getting org data for #{org.name}..."
233
- # # The CFoundry client returns memory_limit in MB, so we need to normalise to bytes to match the Apps.
234
- # {
235
- # :guid => org.guid,
236
- # :name => org.name,
237
- # :quota => {
238
- # :total_services => org.quota_definition.total_services,
239
- # :memory_limit => org.quota_definition.memory_limit * 1024 * 1024
240
- # }
241
- # }
242
- # end.flatten.compact
243
- # end
244
-
245
- # def format_app_data(app, org_name, space_name)
246
-
247
- # last_uploaded = (app.manifest[:entity][:package_updated_at] ||= nil)
248
-
249
- # base_data = {
250
- # :guid => app.guid,
251
- # :name => app.name,
252
- # :org => org_name,
253
- # :space => space_name,
254
- # :stack => app.stack.name,
255
- # :buildpack => app.buildpack,
256
- # :routes => app.running? ? routes_for_app(app) : [],
257
- # :data_from => Time.now.to_i,
258
- # :last_uploaded => last_uploaded ? DateTime.parse(last_uploaded).strftime('%Y-%m-%d %T %z') : nil
259
- # }
260
-
261
- # additional_data = {}
262
- # begin
263
- # additional_data = {
264
- # :running => app.running?,
265
- # :instances => app.running? ? app.stats.map{|key, value| value} : [],
266
- # :error => nil
267
- # }
268
- # rescue => e
269
- # @logger.info " #{org_name} #{space_name}: '#{app.name}'' error: #{e.message}"
270
- # additional_data = {
271
- # :running => 'error',
272
- # :instances => [],
273
- # :error => e.message
274
- # }
275
- # end
276
-
277
- # base_data.merge additional_data
278
- # end
279
-
280
- # def routes_for_app app
281
- # guids = space.apps.first.routes.collect{|route| route.guid}
282
- # guids.collect{|guid| @domains.select{|domain| domain[:guid] == guid}}
283
- # @domains.collect
284
- # end
285
-
286
- def put_in_redis(key, data)
287
- REDIS.set key, data.to_json
288
- end
289
-
290
- def format_duration(elapsed_seconds)
291
- seconds = elapsed_seconds % 60
292
- minutes = (elapsed_seconds / 60) % 60
293
- hours = elapsed_seconds / (60 * 60)
294
- format("%02d hrs, %02d mins, %02d secs", hours, minutes, seconds)
295
- end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cf_light_api
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0.pre1
4
+ version: 2.0.0.pre2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Springer Platform Engineering
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-05-19 00:00:00.000000000 Z
11
+ date: 2016-06-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: cfoundry
@@ -38,20 +38,6 @@ dependencies:
38
38
  - - ~>
39
39
  - !ruby/object:Gem::Version
40
40
  version: 3.2.1
41
- - !ruby/object:Gem::Dependency
42
- name: parallel
43
- requirement: !ruby/object:Gem::Requirement
44
- requirements:
45
- - - ~>
46
- - !ruby/object:Gem::Version
47
- version: 1.4.1
48
- type: :runtime
49
- prerelease: false
50
- version_requirements: !ruby/object:Gem::Requirement
51
- requirements:
52
- - - ~>
53
- - !ruby/object:Gem::Version
54
- version: 1.4.1
55
41
  - !ruby/object:Gem::Dependency
56
42
  name: rufus-scheduler
57
43
  requirement: !ruby/object:Gem::Requirement
@@ -117,6 +103,7 @@ extensions: []
117
103
  extra_rdoc_files: []
118
104
  files:
119
105
  - ./lib/sinatra/cf_light_api.rb
106
+ - ./lib/cf_light_api/lib.rb
120
107
  - ./lib/cf_light_api/redis.rb
121
108
  - ./lib/cf_light_api/worker.rb
122
109
  - bin/cf_light_api