cf_light_api 2.1.0 → 2.2.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c695feed1d26f00127af6fac5fb24de9d64e0b56
-  data.tar.gz: 42228f7a1efecd0f66f4f3ac0a1f31ad8de6d734
+  metadata.gz: 2285895d68ee80832326dad1a0e06ceb60630145
+  data.tar.gz: 6dafdfd47ebf9583f97eb8f8cfa0e2a1a5c8a891
 SHA512:
-  metadata.gz: 6a412171a1799c8797bcd53e064cdad186154c42ab01d2bf46145b15f2d56ac9b3fbb6303f9140546478bbfc48e9bcba0d832b777b26ae280850acf15222772e
-  data.tar.gz: 5de6fde920e3ad043ed7673a8fe0b70500665e377f076b1844a1f053b6c004180a95e87943a9c8f3fe028fedc525325e2ce51567db6f6463b5036ed4f57da00a
+  metadata.gz: 0d45598db25a9a7b54729a3b5caa0364b5c8cb21819549c1031f5ad042180fe0ae3e2211cda599f8cde9d9bfcc4fa22babfa56291d7d5f3c8cd1c04fe1e9a598
+  data.tar.gz: f1ba0ad880904e5c2bc36a2927468376ca6cd1c68169dd8ff09bc6e2aa10585ebd54b152fb807464353903de1335e9f647faa822bb0db307a90f9cb61479bcbe

data/bin/cf_light_api

@@ -4,6 +4,16 @@ $stdout.sync = true
 
 puts '[cf_light_api] Starting CF Light API and worker...'
 
+# If either of the minimum required New Relic settings are present, verify that they are both set, or exit with an error.
+if ENV['NEW_RELIC_APP_NAME'] or ENV['NEW_RELIC_LICENSE_KEY']
+  ['NEW_RELIC_APP_NAME', 'NEW_RELIC_LICENSE_KEY'].each do |env|
+    unless ENV[env]
+      puts "[cf_light_api] Error: please set the '#{env}' environment variable to enable New Relic integration."
+      exit 1
+    end
+  end
+end
+
 require_relative '../lib/cf_light_api/redis.rb'
 require_relative '../lib/cf_light_api/worker.rb'
 

data/lib/cf_light_api/worker.rb

@@ -6,154 +6,249 @@ require 'logger'
 require 'graphite-api'
 require 'date'
 
-require_relative 'lib.rb'
+class CFLightAPIWorker
+  if ENV['NEW_RELIC_LICENSE_KEY']
+    require 'newrelic_rpm'
+    include NewRelic::Agent::Instrumentation::ControllerInstrumentation
+    include NewRelic::Agent::MethodTracer
+  end
 
-@logger = Logger.new(STDOUT)
-@logger.formatter = proc do |severity, datetime, progname, msg|
-   "#{datetime} [cf_light_api:worker]: #{msg}\n"
-end
+  def initialize
+    @logger = Logger.new(STDOUT)
+    @logger.formatter = proc do |severity, datetime, progname, msg|
+       "#{datetime} [cf_light_api:worker]: #{msg}\n"
+    end
+
+    ['CF_API', 'CF_USER', 'CF_PASSWORD'].each do |env|
+      unless ENV[env]
+        @logger.info "Error: please set the '#{env}' environment variable."
+        exit 1
+      end
+    end
+
+    # If either of the Graphite settings are set, verify that they are both set, or exit with an error.
+    if ENV['GRAPHITE_HOST'] or ENV['GRAPHITE_PORT']
+      ['GRAPHITE_HOST', 'GRAPHITE_PORT'].each do |env|
+        unless ENV[env]
+          @logger.info "Error: please set the '#{env}' environment variable to enable exporting to Graphite."
+          exit 1
+        end
+      end
+    end
+
+    update_interval = (ENV['UPDATE_INTERVAL'] || '5m').to_s # If you change the default '5m' here, also remember to change the default age validity in sinatra/cf_light_api.rb:31
+    update_timeout  = (ENV['UPDATE_TIMEOUT']  || '5m').to_s
+
+    @lock_manager = Redlock::Client.new([ENV['REDIS_URI']])
+    @scheduler    = Rufus::Scheduler.new
+
+    @logger.info "Update interval: '#{@update_interval}'"
+    @logger.info "Update timeout:  '#{@update_timeout}'"
+
+    if ENV['GRAPHITE_HOST'] and ENV['GRAPHITE_PORT']
+      @logger.info "Graphite server: #{ENV['GRAPHITE_HOST']}:#{ENV['GRAPHITE_PORT']}" 
+    else
+      @logger.info 'Graphite server: Disabled'
+    end
+
+    @scheduler.every update_interval, :first_in => '5s', :overlap => false, :timeout => update_timeout do
+      update_cf_data
+    end
 
-['CF_API', 'CF_USER', 'CF_PASSWORD'].each do |env|
-  unless ENV[env]
-    @logger.info "Error: please set the '#{env}' environment variable."
-    exit 1
   end
-end
 
-# If either of the Graphite settings are set, verify that they are both set, or exit with an error.
-if ENV['GRAPHITE_HOST'] or ENV['GRAPHITE_PORT']
-  ['GRAPHITE_HOST', 'GRAPHITE_PORT'].each do |env|
-    unless ENV[env]
-      @logger.info "Error: please set the '#{env}' environment variable to enable exporting to Graphite."
-      exit 1
+  def formatted_instance_stats_for_app app
+    instances = cf_rest("/v2/apps/#{app['metadata']['guid']}/stats")[0]
+    raise "Unable to retrieve app instance stats: '#{instances['error_code']}'" if instances['error_code']
+    instances.map{|key,value|value}
+  end
+
+  def cf_rest(path, method='GET')
+    @logger.info "Making #{method} request for #{path}..."
+
+    resources = []
+    response = JSON.parse(@cf_client.base.rest_client.request(method, path)[1][:body])
+    
+    # Some endpoints return a 'resources' array, others are flat, depending on the path.
+    if response['resources'] 
+      resources << response['resources']
+    else
+      resources << response
     end
+
+    # Handle the pagination by recursing over myself until we get a response which doesn't contain a 'next_url'
+    # at which point all the resources are returned up the stack and flattened.
+    resources << cf_rest(response['next_url'], method) unless response['next_url'] == nil
+    resources.flatten
   end
-end
 
-UPDATE_INTERVAL = (ENV['UPDATE_INTERVAL'] || '5m').to_s # If you change the default '5m' here, also remember to change the default age validity in sinatra/cf_light_api.rb:31
-UPDATE_TIMEOUT  = (ENV['UPDATE_TIMEOUT']  || '5m').to_s
+  def get_client(cf_api=ENV['CF_API'], cf_user=ENV['CF_USER'], cf_password=ENV['CF_PASSWORD'])
+    client = CFoundry::Client.get(cf_api)
+    client.login({:username => cf_user, :password => cf_password})
+    client
+  end
 
-lock_manager = Redlock::Client.new([ENV['REDIS_URI']])
-scheduler    = Rufus::Scheduler.new
+  def send_instance_usage_data_to_graphite(instance_stats, org, space, app_name)
+    app_name.gsub! ".", "_" # Some apps have dots in the app name
 
-@logger.info "Update interval: '#{UPDATE_INTERVAL}'"
-@logger.info "Update timeout:  '#{UPDATE_TIMEOUT}'"
+    instance_stats.each_with_index do |instance_data, index|
+      graphite_base_key = "cf_apps.#{org}.#{space}.#{app_name}.#{index}"
+      @logger.info "  Exporting app instance \##{index} usage statistics to Graphite, path '#{graphite_base_key}'"
 
-if ENV['GRAPHITE_HOST'] and ENV['GRAPHITE_PORT']
-  @logger.info "Graphite server: #{ENV['GRAPHITE_HOST']}:#{ENV['GRAPHITE_PORT']}" 
-else
-  @logger.info 'Graphite server: Disabled'
-end
+      # Quota data
+      ['mem_quota', 'disk_quota'].each do |key|
+        @graphite.metrics "#{graphite_base_key}.#{key}" => instance_data['stats'][key]
+      end
 
-scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout => UPDATE_TIMEOUT do
-  @cf_client = nil
-  @graphite  = GraphiteAPI.new(graphite: "#{ENV['GRAPHITE_HOST']}:#{ENV['GRAPHITE_PORT']}") if ENV['GRAPHITE_HOST'] and ENV['GRAPHITE_PORT']
-
-  begin
-    lock_manager.lock("#{ENV['REDIS_KEY_PREFIX']}:lock", 5*60*1000) do |lock|
-      if lock
-        start_time = Time.now
-
-        @logger.info "Updating data..."
-
-        @cf_client = get_client() # Ensure we have a fresh auth token...
-
-        @apps      = cf_rest('/v2/apps?results-per-page=100')
-        @orgs      = cf_rest('/v2/organizations?results-per-page=100')
-        @quotas    = cf_rest('/v2/quota_definitions?results-per-page=100')
-        @spaces    = cf_rest('/v2/spaces?results-per-page=100')
-        @stacks    = cf_rest('/v2/stacks?results-per-page=100')
-        @domains   = cf_rest('/v2/domains?results-per-page=100')
-
-        formatted_orgs = @orgs.map do |org|
-          quota = @quotas.find{|a_quota| a_quota['metadata']['guid'] == org['entity']['quota_definition_guid']}
-
-          {
-            :guid => org['metadata']['guid'],
-            :name  => org['entity']['name'],
-            :quota => {
-              :name           => quota['entity']['name'],
-              :total_services => quota['entity']['total_services'],
-              :total_routes   => quota['entity']['total_routes'],
-              :memory_limit   => quota['entity']['memory_limit'] * 1024 * 1024
-            }
-          }
-        end
+      # Usage data
+      ['mem', 'disk', 'cpu'].each do |key|
+        @graphite.metrics "#{graphite_base_key}.#{key}" => instance_data['stats']['usage'][key]
+      end
+    end
+  end
 
-        formatted_apps = @apps.map do |app|
-          # TODO: This is a bit repetative, could maybe improve?
-          space  = @spaces.find{|a_space| a_space['metadata']['guid'] == app['entity']['space_guid']}
-          org    = @orgs.find{|an_org|     an_org['metadata']['guid'] == space['entity']['organization_guid']}
-          stack  = @stacks.find{|a_stack| a_stack['metadata']['guid'] == app['entity']['stack_guid']}
-          routes = format_routes_for_app(app)
-
-          running = (app['entity']['state'] == "STARTED")
-
-          base_data = {
-            :buildpack     => app['entity']['buildpack'],
-            :data_from     => Time.now.to_i,
-            :diego         => app['entity']['diego'],
-            :docker        => app['entity']['docker_image'] ? true : false,
-            :docker_image  => app['entity']['docker_image'],
-            :guid          => app['metadata']['guid'],
-            :last_uploaded => app['metadata']['updated_at'] ? DateTime.parse(app['metadata']['updated_at']).strftime('%Y-%m-%d %T %z') : nil,
-            :name          => app['entity']['name'],
-            :org           => org['entity']['name'],
-            :routes        => routes,
-            :space         => space['entity']['name'],
-            :stack         => stack['entity']['name'],
-            :state         => app['entity']['state']
-          }
-
-          # Add additional data, such as instance usage statistics - but this is only possible if the instances are running.
-          additional_data = {}
-
-          begin
-            instance_stats = []
-            if running
-              instance_stats = formatted_instance_stats_for_app(app)
-              running_instances = instance_stats.select{|instance| instance['stats']['uris'] if instance['state'] == 'RUNNING'}
-              raise "There are no running instances of this app." if running_instances.empty?
-              
-              if @graphite
-                send_instance_usage_data_to_graphite(instance_stats, org['entity']['name'], space['entity']['name'], app['entity']['name'])
-              end
-            end
+  def put_in_redis(key, data)
+    REDIS.set key, data.to_json
+  end
 
-            additional_data = {
-             :running   => running,
-             :instances => instance_stats,
-             :error     => nil
-            }
+  def format_duration(elapsed_seconds)
+    seconds = elapsed_seconds % 60
+    minutes = (elapsed_seconds / 60) % 60
+    hours   = elapsed_seconds / (60 * 60)
+    format("%02d hrs, %02d mins, %02d secs", hours, minutes, seconds)
+  end
+
+  def format_routes_for_app app
+    routes = cf_rest app['entity']['routes_url']
+    routes.collect do |route|
+      host   = route['entity']['host']
+      path   = route['entity']['path']
+
+      domain = @domains.find{|a_domain| a_domain['metadata']['guid'] == route['entity']['domain_guid']}
+      domain = domain['entity']['name']
+
+      "#{host}.#{domain}#{path}"
+    end
+  end
 
-          rescue => e
-            # Most exceptions here will be caused by the app or one of the instances being in a non-standard state,
-            # for example, trying to query an app which was present when the worker began updating, but was stopped
-            # before we reached this section, so we just catch all exceptions, log the reason and move on.
-            @logger.info "  #{org['entity']['name']} #{space['entity']['name']}: '#{app['entity']['name']}' error: #{e.message}"
-            additional_data = {
-              :running   => 'error',
-              :instances => [],
-              :error     => e.message
+  def update_cf_data
+    @cf_client = nil
+    @graphite  = GraphiteAPI.new(graphite: "#{ENV['GRAPHITE_HOST']}:#{ENV['GRAPHITE_PORT']}") if ENV['GRAPHITE_HOST'] and ENV['GRAPHITE_PORT']
+
+    begin
+      @lock_manager.lock("#{ENV['REDIS_KEY_PREFIX']}:lock", 5*60*1000) do |lock|
+        if lock
+          start_time = Time.now
+
+          @logger.info "Updating data..."
+
+          @cf_client = get_client() # Ensure we have a fresh auth token...
+
+          @apps      = cf_rest('/v2/apps?results-per-page=100')
+          @orgs      = cf_rest('/v2/organizations?results-per-page=100')
+          @quotas    = cf_rest('/v2/quota_definitions?results-per-page=100')
+          @spaces    = cf_rest('/v2/spaces?results-per-page=100')
+          @stacks    = cf_rest('/v2/stacks?results-per-page=100')
+          @domains   = cf_rest('/v2/domains?results-per-page=100')
+
+          formatted_orgs = @orgs.map do |org|
+            quota = @quotas.find{|a_quota| a_quota['metadata']['guid'] == org['entity']['quota_definition_guid']}
+
+            {
+              :guid => org['metadata']['guid'],
+              :name  => org['entity']['name'],
+              :quota => {
+                :name           => quota['entity']['name'],
+                :total_services => quota['entity']['total_services'],
+                :total_routes   => quota['entity']['total_routes'],
+                :memory_limit   => quota['entity']['memory_limit'] * 1024 * 1024
+              }
             }
           end
 
-          base_data.merge additional_data
-        end
+          formatted_apps = @apps.map do |app|
+            # TODO: This is a bit repetative, could maybe improve?
+            space  = @spaces.find{|a_space| a_space['metadata']['guid'] == app['entity']['space_guid']}
+            org    = @orgs.find{|an_org|     an_org['metadata']['guid'] == space['entity']['organization_guid']}
+            stack  = @stacks.find{|a_stack| a_stack['metadata']['guid'] == app['entity']['stack_guid']}
+            routes = format_routes_for_app(app)
+
+            running = (app['entity']['state'] == "STARTED")
+
+            base_data = {
+              :buildpack     => app['entity']['buildpack'],
+              :data_from     => Time.now.to_i,
+              :diego         => app['entity']['diego'],
+              :docker        => app['entity']['docker_image'] ? true : false,
+              :docker_image  => app['entity']['docker_image'],
+              :guid          => app['metadata']['guid'],
+              :last_uploaded => app['metadata']['updated_at'] ? DateTime.parse(app['metadata']['updated_at']).strftime('%Y-%m-%d %T %z') : nil,
+              :name          => app['entity']['name'],
+              :org           => org['entity']['name'],
+              :routes        => routes,
+              :space         => space['entity']['name'],
+              :stack         => stack['entity']['name'],
+              :state         => app['entity']['state']
+            }
+
+            # Add additional data, such as instance usage statistics - but this is only possible if the instances are running.
+            additional_data = {}
+
+            begin
+              instance_stats = []
+              if running
+                instance_stats = formatted_instance_stats_for_app(app)
+                running_instances = instance_stats.select{|instance| instance['stats']['uris'] if instance['state'] == 'RUNNING'}
+                raise "There are no running instances of this app." if running_instances.empty?
+                
+                if @graphite
+                  send_instance_usage_data_to_graphite(instance_stats, org['entity']['name'], space['entity']['name'], app['entity']['name'])
+                end
+              end
 
-        put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:orgs", formatted_orgs
-        put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:apps", formatted_apps
-        put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:last_updated", {:last_updated => Time.now}
+              additional_data = {
+               :running   => running,
+               :instances => instance_stats,
+               :error     => nil
+              }
+
+            rescue => e
+              # Most exceptions here will be caused by the app or one of the instances being in a non-standard state,
+              # for example, trying to query an app which was present when the worker began updating, but was stopped
+              # before we reached this section, so we just catch all exceptions, log the reason and move on.
+              @logger.info "  #{org['entity']['name']} #{space['entity']['name']}: '#{app['entity']['name']}' error: #{e.message}"
+              additional_data = {
+                :running   => 'error',
+                :instances => [],
+                :error     => e.message
+              }
+            end
+
+            base_data.merge additional_data
+          end
 
-        @logger.info "Update completed in #{format_duration(Time.now.to_f - start_time.to_f)}..."
-        lock_manager.unlock(lock)
-        @cf_client.logout
-      else
-        @logger.info "Update already running in another instance!"
+          put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:orgs", formatted_orgs
+          put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:apps", formatted_apps
+          put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:last_updated", {:last_updated => Time.now}
+
+          @logger.info "Update completed in #{format_duration(Time.now.to_f - start_time.to_f)}..."
+          @lock_manager.unlock(lock)
+          @cf_client.logout
+        else
+          @logger.info "Update already running in another instance!"
+        end
       end
+    rescue Rufus::Scheduler::TimeoutError
+      @logger.info 'Data update took too long and was aborted, waiting for the lock to expire before trying again...'
+      @cf_client.logout
     end
-  rescue Rufus::Scheduler::TimeoutError
-    @logger.info 'Data update took too long and was aborted, waiting for the lock to expire before trying again...'
-    @cf_client.logout
   end
+
+  if ENV['NEW_RELIC_LICENSE_KEY']
+    add_transaction_tracer :update_cf_data, category: :task
+    add_method_tracer :update_cf_data
+  end
+
 end
+
+CFLightAPIWorker.new

data/lib/sinatra/cf_light_api.rb

@@ -1,9 +1,15 @@
 require 'json'
 
+if ENV['NEW_RELIC_LICENSE_KEY']
+  require 'newrelic_rpm'
+  NewRelic::Agent.manual_start
+end
+
 module Sinatra
   module CfLightAPI
 
     def self.registered(app)
+
       app.get '/v1/apps/?:org?' do
         content_type :json
         all_apps = JSON.parse(REDIS.get("#{ENV['REDIS_KEY_PREFIX']}:apps"))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cf_light_api
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0.pre1
 platform: ruby
 authors:
 - Springer Platform Engineering
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-09 00:00:00.000000000 Z
+date: 2016-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cfoundry
@@ -94,6 +94,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.6
+- !ruby/object:Gem::Dependency
+  name: newrelic_rpm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.17.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.17.1
 description: A super lightweight API for reading App and Org data from CloudFoundry,
   cached in Redis.
 email: ''
@@ -103,7 +117,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ./lib/sinatra/cf_light_api.rb
-- ./lib/cf_light_api/lib.rb
 - ./lib/cf_light_api/redis.rb
 - ./lib/cf_light_api/worker.rb
 - bin/cf_light_api
@@ -122,9 +135,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.0.14

data/lib/cf_light_api/lib.rb

@@ -1,73 +0,0 @@
-def formatted_instance_stats_for_app app
-  instances = cf_rest("/v2/apps/#{app['metadata']['guid']}/stats")[0]
-  raise "Unable to retrieve app instance stats: '#{instances['error_code']}'" if instances['error_code']
-  instances.map{|key,value|value}
-end
-
-def cf_rest(path, method='GET')
-  @logger.info "Making #{method} request for #{path}..."
-
-  resources = []
-  response = JSON.parse(@cf_client.base.rest_client.request(method, path)[1][:body])
-  
-  # Some endpoints return a 'resources' array, others are flat, depending on the path.
-  if response['resources'] 
-    resources << response['resources']
-  else
-    resources << response
-  end
-
-  # Handle the pagination by recursing over myself until we get a response which doesn't contain a 'next_url'
-  # at which point all the resources are returned up the stack and flattened.
-  resources << cf_rest(response['next_url'], method) unless response['next_url'] == nil
-  resources.flatten
-end
-
-def get_client(cf_api=ENV['CF_API'], cf_user=ENV['CF_USER'], cf_password=ENV['CF_PASSWORD'])
-  client = CFoundry::Client.get(cf_api)
-  client.login({:username => cf_user, :password => cf_password})
-  client
-end
-
-def send_instance_usage_data_to_graphite(instance_stats, org, space, app_name)
-  app_name.gsub! ".", "_" # Some apps have dots in the app name
-
-  instance_stats.each_with_index do |instance_data, index|
-    graphite_base_key = "cf_apps.#{org}.#{space}.#{app_name}.#{index}"
-    @logger.info "  Exporting app instance \##{index} usage statistics to Graphite, path '#{graphite_base_key}'"
-
-    # Quota data
-    ['mem_quota', 'disk_quota'].each do |key|
-      @graphite.metrics "#{graphite_base_key}.#{key}" => instance_data['stats'][key]
-    end
-
-    # Usage data
-    ['mem', 'disk', 'cpu'].each do |key|
-      @graphite.metrics "#{graphite_base_key}.#{key}" => instance_data['stats']['usage'][key]
-    end
-  end
-end
-
-def put_in_redis(key, data)
-  REDIS.set key, data.to_json
-end
-
-def format_duration(elapsed_seconds)
-  seconds = elapsed_seconds % 60
-  minutes = (elapsed_seconds / 60) % 60
-  hours   = elapsed_seconds / (60 * 60)
-  format("%02d hrs, %02d mins, %02d secs", hours, minutes, seconds)
-end
-
-def format_routes_for_app app
-  routes = cf_rest app['entity']['routes_url']
-  routes.collect do |route|
-    host   = route['entity']['host']
-    path   = route['entity']['path']
-
-    domain = @domains.find{|a_domain| a_domain['metadata']['guid'] == route['entity']['domain_guid']}
-    domain = domain['entity']['name']
-
-    "#{host}.#{domain}#{path}"
-  end
-end