cf_light_api 2.0.0.pre1 → 2.0.0.pre2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bdbf36a622784f98ea2133973966ae40eb500572
-  data.tar.gz: 8b12c0cec0408a0f16d31392836eea9286f62fd7
+  metadata.gz: d66b56d2709d066bd7bdf1a87ebbf8b1e240a219
+  data.tar.gz: 6aebb9a1303ba0e48927e361a38ac1796b87a983
 SHA512:
-  metadata.gz: 6d65857061da6b5b87dd19928b5386ef2d339649bae6989cec2a981607cc09d28b3f8be72a2d2c97039517a5151e53ec9e50a55e290618e4c9e918bb27eaeb12
-  data.tar.gz: c5f6a83db24dbe514f9ce1272824274ac052f71b4f1296462793b4925ad290053273be7e60b21a6715e3555cca98b58823607b9679a2e1b880bc2451dd77f6e3
+  metadata.gz: 2030c3344aea542a4faee1e364f5c2242fea81f764baa41dfc3236db8635fda2f88ef5827b8d0f17640dfa2bcbeae8af77e37649f17bf03373bfa7560ed16372
+  data.tar.gz: f2730633fc1827d02937eda58b7b591fea38e0604dc8357d17caf34747639caff977f232a646feb0a97acb04373ddad93f165efd9799c4b7168a882ecbf219c2

data/lib/cf_light_api/lib.rb

@@ -0,0 +1,60 @@
+def formatted_instance_stats_for_app app
+  instances = cf_rest("/v2/apps/#{app['metadata']['guid']}/stats")[0]
+  raise "Unable to retrieve app instance stats: '#{instances['error_code']}'" if instances['error_code']
+  instances.map{|key,value|value}
+end
+
+def cf_rest(path, method='GET')
+  @logger.info "Making #{method} request for #{path}..."
+
+  resources = []
+  response = JSON.parse(@cf_client.base.rest_client.request(method, path)[1][:body])
+  
+  # Some endpoints return a 'resources' array, others are flat, depending on the path.
+  if response['resources'] 
+    resources << response['resources']
+  else
+    resources << response
+  end
+
+  # Handle the pagination by recursing over myself until we get a response which doesn't contain a 'next_url'
+  # at which point all the resources are returned up the stack and flattened.
+  resources << cf_rest(response['next_url'], method) unless response['next_url'] == nil
+  resources.flatten
+end
+
+def get_client(cf_api=ENV['CF_API'], cf_user=ENV['CF_USER'], cf_password=ENV['CF_PASSWORD'])
+  client = CFoundry::Client.get(cf_api)
+  client.login({:username => cf_user, :password => cf_password})
+  client
+end
+
+def send_instance_usage_data_to_graphite(instance_stats, org, space, app_name)
+  app_name.gsub! ".", "_" # Some apps have dots in the app name
+
+  instance_stats.each_with_index do |instance_data, index|
+    graphite_base_key = "cf_apps.#{org}.#{space}.#{app_name}.#{index}"
+    @logger.info "  Exporting app instance \##{index} usage statistics to Graphite, path '#{graphite_base_key}'"
+
+    # Quota data
+    ['mem_quota', 'disk_quota'].each do |key|
+      @graphite.metrics "#{graphite_base_key}.#{key}" => instance_data['stats'][key]
+    end
+
+    # Usage data
+    ['mem', 'disk', 'cpu'].each do |key|
+      @graphite.metrics "#{graphite_base_key}.#{key}" => instance_data['stats']['usage'][key]
+    end
+  end
+end
+
+def put_in_redis(key, data)
+  REDIS.set key, data.to_json
+end
+
+def format_duration(elapsed_seconds)
+  seconds = elapsed_seconds % 60
+  minutes = (elapsed_seconds / 60) % 60
+  hours   = elapsed_seconds / (60 * 60)
+  format("%02d hrs, %02d mins, %02d secs", hours, minutes, seconds)
+end

data/lib/cf_light_api/worker.rb

@@ -1,12 +1,13 @@
 require 'cfoundry'
 require 'json'
 require 'rufus-scheduler'
-require 'parallel'
 require 'redlock'
 require 'logger'
 require 'graphite-api'
 require 'date'
 
+require_relative 'lib.rb'
+
 @logger = Logger.new(STDOUT)
 @logger.formatter = proc do |severity, datetime, progname, msg|
    "#{datetime} [cf_light_api:worker]: #{msg}\n"
@@ -19,62 +20,34 @@ end
   end
 end
 
-PARALLEL_MAPS   = (ENV['PARALLEL_MAPS']   || 4   ).to_i
+# If either of the Graphite settings are set, verify that they are both set, or exit with an error.
+if ENV['GRAPHITE_HOST'] or ENV['GRAPHITE_PORT']
+  ['GRAPHITE_HOST', 'GRAPHITE_PORT'].each do |env|
+    unless ENV[env]
+      @logger.info "Error: please set the '#{env}' environment variable to enable exporting to Graphite."
+      exit 1
+    end
+  end
+end
+
 UPDATE_INTERVAL = (ENV['UPDATE_INTERVAL'] || '5m').to_s # If you change the default '5m' here, also remember to change the default age validity in sinatra/cf_light_api.rb:31
 UPDATE_TIMEOUT  = (ENV['UPDATE_TIMEOUT']  || '5m').to_s
 
 lock_manager = Redlock::Client.new([ENV['REDIS_URI']])
 scheduler    = Rufus::Scheduler.new
 
-@logger.info "Parallel maps:   '#{PARALLEL_MAPS}'"
 @logger.info "Update interval: '#{UPDATE_INTERVAL}'"
 @logger.info "Update timeout:  '#{UPDATE_TIMEOUT}'"
-@logger.info "Graphite server: #{ENV['GRAPHITE']}" if ENV['GRAPHITE']
-
-#@domains = {}
-# @domains   = cf_rest('/v2/domains?results-per-page=100') # We're retrieving this from the app instance 'uris' key instead.
-# @routes    = cf_rest('/v2/routes?results-per-page=100')
-
-# def formatted_routes_for_app app
-#   routes = cf_rest(app['entity']['routes_url'])
-
-#   routes.collect do |route|
-#     domain = @domains.find{|a_domain| a_domain['metadata']['guid'] ==  route['entity']['domain_guid']}['entity']['name']
-#     # Suffix the hostname with a period for concatenation, unless it's blank (which can happen for apex routes).
-#     host = route['entity']['host'] != '' ? "#{route['entity']['host']}." : ''
-#     path = route['entity']['path']
-#     "#{host}#{domain}#{path}"
-#   end
-# end
-
-def formatted_instance_stats_for_app app
-  instances = cf_rest("/v2/apps/#{app['metadata']['guid']}/stats")[0]
-  raise "Unable to retrieve app instance stats: '#{instances['error_code']}'" if instances['error_code']
-  instances.map{|key,value|value}
-end
-
-def cf_rest(path, method='GET')
-  @logger.info "Making #{method} request for #{path}..."
 
-  resources = []
-  response = JSON.parse(@cf_client.base.rest_client.request(method, path)[1][:body])
-  
-  # Some endpoints return a 'resources' array, others are flat, depending on the path.
-  if response['resources'] 
-    resources << response['resources']
-  else
-    resources << response
-  end
-
-  # Handle the pagination by recursing over myself until we get a response which doesn't contain a 'next_url'
-  # at which point all the resources are returned up the stack and flattened.
-  resources << cf_rest(response['next_url'], method) unless response['next_url'] == nil
-  resources.flatten
+if ENV['GRAPHITE_HOST'] and ENV['GRAPHITE_PORT']
+  @logger.info "Graphite server: #{ENV['GRAPHITE_HOST']}:#{ENV['GRAPHITE_PORT']}" 
+else
+  @logger.info 'Graphite server: Disabled'
 end
 
 scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout => UPDATE_TIMEOUT do
   @cf_client = nil
-  # graphite  = GraphiteAPI.new(graphite: ENV['GRAPHITE']) if ENV['GRAPHITE']
+  @graphite  = GraphiteAPI.new(graphite: "#{ENV['GRAPHITE_HOST']}:#{ENV['GRAPHITE_PORT']}") if ENV['GRAPHITE_HOST'] and ENV['GRAPHITE_PORT']
 
   begin
     lock_manager.lock("#{ENV['REDIS_KEY_PREFIX']}:lock", 5*60*1000) do |lock|
@@ -91,9 +64,6 @@ scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout
         @spaces    = cf_rest('/v2/spaces?results-per-page=100')
         @stacks    = cf_rest('/v2/stacks?results-per-page=100')
 
-        # org_data = get_org_data(@cf_client)
-        # app_data = get_app_data(@cf_client, graphite)
-
         formatted_orgs = @orgs.map do |org|
           quota = @quotas.find{|a_quota| a_quota['metadata']['guid'] == org['entity']['quota_definition_guid']}
 
@@ -110,7 +80,7 @@ scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout
         end
 
         formatted_apps = @apps.map do |app|
-          # TODO: This is a bit repetative, could improve.
+          # TODO: This is a bit repetative, could maybe improve?
           space = @spaces.find{|a_space| a_space['metadata']['guid'] == app['entity']['space_guid']}
           org   = @orgs.find{|an_org|     an_org['metadata']['guid'] == space['entity']['organization_guid']}
           stack = @stacks.find{|a_stack| a_stack['metadata']['guid'] == app['entity']['stack_guid']}
@@ -124,23 +94,28 @@ scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout
             :space         => space['entity']['name'],
             :stack         => stack['entity']['name'],
             :buildpack     => app['entity']['buildpack'],
-            # This requires a call to /v2/apps/[guid]/routes for each app, or we can just use the 'uris' key from /v2/apps/[guid]/stats
-            # which we have to call anyway, to get app instance usage stats..
-            # :routes        => running ? formatted_routes_for_app(app) : [],
             :data_from     => Time.now.to_i,
             :last_uploaded => app['metadata']['updated_at'] ? DateTime.parse(app['metadata']['updated_at']).strftime('%Y-%m-%d %T %z') : nil
           }
 
+          # Add additional data, such as instance usage statistics, and routes - but this is only possible
+          # if the instance is running.
           additional_data = {}
+
           begin
             instance_stats = []
             routes         = []
             if running
+              # Finds the first running app instance that has a set of routes, in case there are stopped/crashed app instances that don't have any routes.
               instance_stats = formatted_instance_stats_for_app(app)
-              # Finds the first running app instance that has a set of routes, in case there are stopped/crashed app instances that don't have any.
               running_instances = instance_stats.select{|instance| instance['stats']['uris'] if instance['state'] == 'RUNNING'}
               raise "Unable to retrieve app routes - no app instances are running." if running_instances.empty?
+              
               routes = running_instances.first['stats']['uris']
+
+              if @graphite
+                send_instance_usage_data_to_graphite(instance_stats, org['entity']['name'], space['entity']['name'], app['entity']['name'])
+              end
             end
 
             additional_data = {
@@ -149,7 +124,11 @@ scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout
              :routes    => routes,
              :error     => nil
             }
+
           rescue => e
+            # Most exceptions here will be caused by the app or one of the instances being in a non-standard state,
+            # for example, trying to query an app which was present when the worker began updating, but was stopped
+            # before we reached this section, so we just catch all exceptions, log the reason and move on.
             @logger.info "  #{org['entity']['name']} #{space['entity']['name']}: '#{app['entity']['name']}' error: #{e.message}"
             additional_data = {
               :running   => 'error',
@@ -178,118 +157,3 @@ scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout
     @cf_client.logout
   end
 end
-
-def get_client(cf_api=ENV['CF_API'], cf_user=ENV['CF_USER'], cf_password=ENV['CF_PASSWORD'])
-  client = CFoundry::Client.get(cf_api)
-  client.login({:username => cf_user, :password => cf_password})
-  client
-end
-
-# def send_data_to_graphite(data, graphite)
-#   org   = data[:org]
-#   space = data[:space]
-#   name  = data[:name].sub ".", "_" # Some apps have dots in the app name
-
-#   data[:instances].each_with_index do |instance_data, index|
-#     graphite_base_key = "cf_apps.#{org}.#{space}.#{name}.#{index}"
-
-#     # Quota data
-#     [:mem_quota, :disk_quota].each do |key|
-#       graphite.metrics "#{graphite_base_key}.#{key}" => instance_data[:stats][key]
-#     end
-
-#     # Usage data
-#     [:mem, :disk, :cpu].each do |key|
-#       graphite.metrics "#{graphite_base_key}.#{key}" => instance_data[:stats][:usage][key]
-#     end
-#   end
-# end
-
-# def get_app_data(cf_client, graphite)
-#   Parallel.map(cf_client.organizations, :in_threads=> PARALLEL_MAPS) do |org|
-#     org_name = org.name
-#     Parallel.map(org.spaces, :in_threads => PARALLEL_MAPS) do |space|
-#       space_name = space.name
-#       @logger.info "Getting app data for apps in #{org_name}:#{space_name}..."
-#       Parallel.map(space.apps, :in_threads=> PARALLEL_MAPS) do |app|
-#         begin
-#           # It's possible for an app to have been terminated before this stage is reached.
-#           formatted_app_data = format_app_data(app, org_name, space_name)
-#           if graphite
-#             send_data_to_graphite(formatted_app_data, graphite)
-#           end
-#           formatted_app_data
-#         rescue CFoundry::AppNotFound
-#           next
-#         end
-#       end
-#     end
-#   end.flatten.compact
-# end
-
-# def get_org_data(cf_client)
-#   Parallel.map( cf_client.organizations, :in_threads=> PARALLEL_MAPS) do |org|
-#     @logger.info "Getting org data for #{org.name}..."
-#     # The CFoundry client returns memory_limit in MB, so we need to normalise to bytes to match the Apps.
-#     {
-#       :guid => org.guid,
-#       :name => org.name,
-#       :quota => {
-#         :total_services => org.quota_definition.total_services,
-#         :memory_limit   => org.quota_definition.memory_limit * 1024 * 1024
-#       }
-#     }
-#   end.flatten.compact
-# end
-
-# def format_app_data(app, org_name, space_name)
-
-#   last_uploaded = (app.manifest[:entity][:package_updated_at] ||= nil)
-
-#   base_data = {
-#     :guid          => app.guid,
-#     :name          => app.name,
-#     :org           => org_name,
-#     :space         => space_name,
-#     :stack         => app.stack.name,
-#     :buildpack     => app.buildpack,
-#     :routes        => app.running? ? routes_for_app(app) : [],
-#     :data_from     => Time.now.to_i,
-#     :last_uploaded => last_uploaded ? DateTime.parse(last_uploaded).strftime('%Y-%m-%d %T %z') : nil
-#   }
-
-#   additional_data = {}
-#   begin
-#     additional_data = {
-#      :running   => app.running?,
-#      :instances => app.running? ? app.stats.map{|key, value| value} : [],
-#      :error     => nil
-#     }
-#   rescue => e
-#     @logger.info "  #{org_name} #{space_name}: '#{app.name}'' error: #{e.message}"
-#     additional_data = {
-#       :running   => 'error',
-#       :instances => [],
-#       :error     => e.message
-#     }
-#   end
-
-#   base_data.merge additional_data
-# end
-
-# def routes_for_app app
-#   guids   = space.apps.first.routes.collect{|route| route.guid}
-#   guids.collect{|guid| @domains.select{|domain| domain[:guid] == guid}}
-#   @domains.collect
-# end
-
-def put_in_redis(key, data)
-  REDIS.set key, data.to_json
-end
-
-def format_duration(elapsed_seconds)
-  seconds = elapsed_seconds % 60
-  minutes = (elapsed_seconds / 60) % 60
-  hours   = elapsed_seconds / (60 * 60)
-  format("%02d hrs, %02d mins, %02d secs", hours, minutes, seconds)
-end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cf_light_api
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre1
+  version: 2.0.0.pre2
 platform: ruby
 authors:
 - Springer Platform Engineering
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-19 00:00:00.000000000 Z
+date: 2016-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cfoundry
@@ -38,20 +38,6 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 3.2.1
-- !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.4.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.4.1
 - !ruby/object:Gem::Dependency
   name: rufus-scheduler
   requirement: !ruby/object:Gem::Requirement
@@ -117,6 +103,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ./lib/sinatra/cf_light_api.rb
+- ./lib/cf_light_api/lib.rb
 - ./lib/cf_light_api/redis.rb
 - ./lib/cf_light_api/worker.rb
 - bin/cf_light_api