cf_light_api 1.7.0 → 2.0.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3067cac56cd133c9fc4fd246f29e0874335f2998
-  data.tar.gz: 4fb42dd6f9e61ee969b6d966c15ac5bfc636cc6d
+  metadata.gz: bdbf36a622784f98ea2133973966ae40eb500572
+  data.tar.gz: 8b12c0cec0408a0f16d31392836eea9286f62fd7
 SHA512:
-  metadata.gz: 348c46fd0ae464fb196b78598a713ed1b620a989757bdb1d8dcdf61d542df19b8d7edf5bd985e921b4cd5b61ad3c05ded7831c5f49ec6c4b458c85043ea4fbf9
-  data.tar.gz: 64723974aad403fb3cff89385976ce7f7a8c2bb4214a3a5fd325f9ff65c8734a6fba658dde1496ca2f6c1e5e123c1ec3bdafa795ad208f5c54a9c2e1bc37ca97
+  metadata.gz: 6d65857061da6b5b87dd19928b5386ef2d339649bae6989cec2a981607cc09d28b3f8be72a2d2c97039517a5151e53ec9e50a55e290618e4c9e918bb27eaeb12
+  data.tar.gz: c5f6a83db24dbe514f9ce1272824274ac052f71b4f1296462793b4925ad290053273be7e60b21a6715e3555cca98b58823607b9679a2e1b880bc2451dd77f6e3

data/lib/cf_light_api/worker.rb

@@ -31,36 +31,151 @@ scheduler    = Rufus::Scheduler.new
 @logger.info "Update timeout:  '#{UPDATE_TIMEOUT}'"
 @logger.info "Graphite server: #{ENV['GRAPHITE']}" if ENV['GRAPHITE']
 
+#@domains = {}
+# @domains   = cf_rest('/v2/domains?results-per-page=100') # We're retrieving this from the app instance 'uris' key instead.
+# @routes    = cf_rest('/v2/routes?results-per-page=100')
+
+# def formatted_routes_for_app app
+#   routes = cf_rest(app['entity']['routes_url'])
+
+#   routes.collect do |route|
+#     domain = @domains.find{|a_domain| a_domain['metadata']['guid'] ==  route['entity']['domain_guid']}['entity']['name']
+#     # Suffix the hostname with a period for concatenation, unless it's blank (which can happen for apex routes).
+#     host = route['entity']['host'] != '' ? "#{route['entity']['host']}." : ''
+#     path = route['entity']['path']
+#     "#{host}#{domain}#{path}"
+#   end
+# end
+
+def formatted_instance_stats_for_app app
+  instances = cf_rest("/v2/apps/#{app['metadata']['guid']}/stats")[0]
+  raise "Unable to retrieve app instance stats: '#{instances['error_code']}'" if instances['error_code']
+  instances.map{|key,value|value}
+end
+
+def cf_rest(path, method='GET')
+  @logger.info "Making #{method} request for #{path}..."
+
+  resources = []
+  response = JSON.parse(@cf_client.base.rest_client.request(method, path)[1][:body])
+  
+  # Some endpoints return a 'resources' array, others are flat, depending on the path.
+  if response['resources'] 
+    resources << response['resources']
+  else
+    resources << response
+  end
+
+  # Handle the pagination by recursing over myself until we get a response which doesn't contain a 'next_url'
+  # at which point all the resources are returned up the stack and flattened.
+  resources << cf_rest(response['next_url'], method) unless response['next_url'] == nil
+  resources.flatten
+end
+
 scheduler.every UPDATE_INTERVAL, :first_in => '5s', :overlap => false, :timeout => UPDATE_TIMEOUT do
-  cf_client = nil
-  graphite  = GraphiteAPI.new(graphite: ENV['GRAPHITE']) if ENV['GRAPHITE']
+  @cf_client = nil
+  # graphite  = GraphiteAPI.new(graphite: ENV['GRAPHITE']) if ENV['GRAPHITE']
 
   begin
     lock_manager.lock("#{ENV['REDIS_KEY_PREFIX']}:lock", 5*60*1000) do |lock|
       if lock
         start_time = Time.now
 
-        @logger.info "Updating data in parallel..."
+        @logger.info "Updating data..."
+
+        @cf_client = get_client() # Ensure we have a fresh auth token...
+
+        @apps      = cf_rest('/v2/apps?results-per-page=100')
+        @orgs      = cf_rest('/v2/organizations?results-per-page=100')
+        @quotas    = cf_rest('/v2/quota_definitions?results-per-page=100')
+        @spaces    = cf_rest('/v2/spaces?results-per-page=100')
+        @stacks    = cf_rest('/v2/stacks?results-per-page=100')
+
+        # org_data = get_org_data(@cf_client)
+        # app_data = get_app_data(@cf_client, graphite)
+
+        formatted_orgs = @orgs.map do |org|
+          quota = @quotas.find{|a_quota| a_quota['metadata']['guid'] == org['entity']['quota_definition_guid']}
+
+          {
+            :guid => org['metadata']['guid'],
+            :name  => org['entity']['name'],
+            :quota => {
+              :name           => quota['entity']['name'],
+              :total_services => quota['entity']['total_services'],
+              :total_routes   => quota['entity']['total_routes'],
+              :memory_limit   => quota['entity']['memory_limit'] * 1024 * 1024
+            }
+          }
+        end
+
+        formatted_apps = @apps.map do |app|
+          # TODO: This is a bit repetative, could improve.
+          space = @spaces.find{|a_space| a_space['metadata']['guid'] == app['entity']['space_guid']}
+          org   = @orgs.find{|an_org|     an_org['metadata']['guid'] == space['entity']['organization_guid']}
+          stack = @stacks.find{|a_stack| a_stack['metadata']['guid'] == app['entity']['stack_guid']}
+
+          running = (app['entity']['state'] == "STARTED")
 
-        cf_client = get_client()
+          base_data = {
+            :guid          => app['metadata']['guid'],
+            :name          => app['entity']['name'],
+            :org           => org['entity']['name'],
+            :space         => space['entity']['name'],
+            :stack         => stack['entity']['name'],
+            :buildpack     => app['entity']['buildpack'],
+            # This requires a call to /v2/apps/[guid]/routes for each app, or we can just use the 'uris' key from /v2/apps/[guid]/stats
+            # which we have to call anyway, to get app instance usage stats..
+            # :routes        => running ? formatted_routes_for_app(app) : [],
+            :data_from     => Time.now.to_i,
+            :last_uploaded => app['metadata']['updated_at'] ? DateTime.parse(app['metadata']['updated_at']).strftime('%Y-%m-%d %T %z') : nil
+          }
 
-        org_data = get_org_data(cf_client)
-        app_data = get_app_data(cf_client, graphite)
+          additional_data = {}
+          begin
+            instance_stats = []
+            routes         = []
+            if running
+              instance_stats = formatted_instance_stats_for_app(app)
+              # Finds the first running app instance that has a set of routes, in case there are stopped/crashed app instances that don't have any.
+              running_instances = instance_stats.select{|instance| instance['stats']['uris'] if instance['state'] == 'RUNNING'}
+              raise "Unable to retrieve app routes - no app instances are running." if running_instances.empty?
+              routes = running_instances.first['stats']['uris']
+            end
 
-        put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:orgs", org_data
-        put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:apps", app_data
+            additional_data = {
+             :running   => running,
+             :instances => instance_stats,
+             :routes    => routes,
+             :error     => nil
+            }
+          rescue => e
+            @logger.info "  #{org['entity']['name']} #{space['entity']['name']}: '#{app['entity']['name']}' error: #{e.message}"
+            additional_data = {
+              :running   => 'error',
+              :instances => [],
+              :routes    => [],
+              :error     => e.message
+            }
+          end
+
+          base_data.merge additional_data
+        end
+
+        put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:orgs", formatted_orgs
+        put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:apps", formatted_apps
         put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:last_updated", {:last_updated => Time.now}
 
         @logger.info "Update completed in #{format_duration(Time.now.to_f - start_time.to_f)}..."
         lock_manager.unlock(lock)
-        cf_client.logout
+        @cf_client.logout
       else
         @logger.info "Update already running in another instance!"
       end
     end
   rescue Rufus::Scheduler::TimeoutError
     @logger.info 'Data update took too long and was aborted, waiting for the lock to expire before trying again...'
-    cf_client.logout
+    @cf_client.logout
   end
 end
 
@@ -70,97 +185,103 @@ def get_client(cf_api=ENV['CF_API'], cf_user=ENV['CF_USER'], cf_password=ENV['CF
   client
 end
 
-def send_data_to_graphite(data, graphite)
-  org   = data[:org]
-  space = data[:space]
-  name  = data[:name].sub ".", "_" # Some apps have dots in the app name
+# def send_data_to_graphite(data, graphite)
+#   org   = data[:org]
+#   space = data[:space]
+#   name  = data[:name].sub ".", "_" # Some apps have dots in the app name
 
-  data[:instances].each_with_index do |instance_data, index|
-    graphite_base_key = "cf_apps.#{org}.#{space}.#{name}.#{index}"
+#   data[:instances].each_with_index do |instance_data, index|
+#     graphite_base_key = "cf_apps.#{org}.#{space}.#{name}.#{index}"
 
-    # Quota data
-    [:mem_quota, :disk_quota].each do |key|
-      graphite.metrics "#{graphite_base_key}.#{key}" => instance_data[:stats][key]
-    end
+#     # Quota data
+#     [:mem_quota, :disk_quota].each do |key|
+#       graphite.metrics "#{graphite_base_key}.#{key}" => instance_data[:stats][key]
+#     end
 
-    # Usage data
-    [:mem, :disk, :cpu].each do |key|
-      graphite.metrics "#{graphite_base_key}.#{key}" => instance_data[:stats][:usage][key]
-    end
-  end
-end
+#     # Usage data
+#     [:mem, :disk, :cpu].each do |key|
+#       graphite.metrics "#{graphite_base_key}.#{key}" => instance_data[:stats][:usage][key]
+#     end
+#   end
+# end
 
-def get_app_data(cf_client, graphite)
-  Parallel.map(cf_client.organizations, :in_threads=> PARALLEL_MAPS) do |org|
-    org_name = org.name
-    Parallel.map(org.spaces, :in_threads => PARALLEL_MAPS) do |space|
-      space_name = space.name
-      @logger.info "Getting app data for apps in #{org_name}:#{space_name}..."
-      Parallel.map(space.apps, :in_threads=> PARALLEL_MAPS) do |app|
-        begin
-          # It's possible for an app to have been terminated before this stage is reached.
-          formatted_app_data = format_app_data(app, org_name, space_name)
-          if graphite
-            send_data_to_graphite(formatted_app_data, graphite)
-          end
-          formatted_app_data
-        rescue CFoundry::AppNotFound
-          next
-        end
-      end
-    end
-  end.flatten.compact
-end
+# def get_app_data(cf_client, graphite)
+#   Parallel.map(cf_client.organizations, :in_threads=> PARALLEL_MAPS) do |org|
+#     org_name = org.name
+#     Parallel.map(org.spaces, :in_threads => PARALLEL_MAPS) do |space|
+#       space_name = space.name
+#       @logger.info "Getting app data for apps in #{org_name}:#{space_name}..."
+#       Parallel.map(space.apps, :in_threads=> PARALLEL_MAPS) do |app|
+#         begin
+#           # It's possible for an app to have been terminated before this stage is reached.
+#           formatted_app_data = format_app_data(app, org_name, space_name)
+#           if graphite
+#             send_data_to_graphite(formatted_app_data, graphite)
+#           end
+#           formatted_app_data
+#         rescue CFoundry::AppNotFound
+#           next
+#         end
+#       end
+#     end
+#   end.flatten.compact
+# end
 
-def get_org_data(cf_client)
-  Parallel.map( cf_client.organizations, :in_threads=> PARALLEL_MAPS) do |org|
-    @logger.info "Getting org data for #{org.name}..."
-    # The CFoundry client returns memory_limit in MB, so we need to normalise to bytes to match the Apps.
-    {
-      :guid => org.guid,
-      :name => org.name,
-      :quota => {
-        :total_services => org.quota_definition.total_services,
-        :memory_limit   => org.quota_definition.memory_limit * 1024 * 1024
-      }
-    }
-  end.flatten.compact
-end
+# def get_org_data(cf_client)
+#   Parallel.map( cf_client.organizations, :in_threads=> PARALLEL_MAPS) do |org|
+#     @logger.info "Getting org data for #{org.name}..."
+#     # The CFoundry client returns memory_limit in MB, so we need to normalise to bytes to match the Apps.
+#     {
+#       :guid => org.guid,
+#       :name => org.name,
+#       :quota => {
+#         :total_services => org.quota_definition.total_services,
+#         :memory_limit   => org.quota_definition.memory_limit * 1024 * 1024
+#       }
+#     }
+#   end.flatten.compact
+# end
 
-def format_app_data(app, org_name, space_name)
+# def format_app_data(app, org_name, space_name)
 
-  last_uploaded = (app.manifest[:entity][:package_updated_at] ||= nil)
+#   last_uploaded = (app.manifest[:entity][:package_updated_at] ||= nil)
 
-  base_data = {
-    :guid          => app.guid,
-    :name          => app.name,
-    :org           => org_name,
-    :space         => space_name,
-    :stack         => app.stack.name,
-    :buildpack     => app.buildpack,
-    :routes        => app.routes.map {|route| route.name},
-    :data_from     => Time.now.to_i,
-    :last_uploaded => last_uploaded ? DateTime.parse(last_uploaded).strftime('%Y-%m-%d %T %z') : nil
-  }
+#   base_data = {
+#     :guid          => app.guid,
+#     :name          => app.name,
+#     :org           => org_name,
+#     :space         => space_name,
+#     :stack         => app.stack.name,
+#     :buildpack     => app.buildpack,
+#     :routes        => app.running? ? routes_for_app(app) : [],
+#     :data_from     => Time.now.to_i,
+#     :last_uploaded => last_uploaded ? DateTime.parse(last_uploaded).strftime('%Y-%m-%d %T %z') : nil
+#   }
 
-  additional_data = {}
-  begin
-    additional_data = {
-     :running   => app.running?,
-     :instances => app.running? ? app.stats.map{|key, value| value} : [],
-     :error     => nil
-    }
-  rescue => e
-    @logger.info "  #{org_name} #{space_name}: '#{app.name}'' error: #{e.message}"
-    additional_data = {
-      :running   => 'error',
-      :instances => [],
-      :error     => e.message
-    }
-  end
+#   additional_data = {}
+#   begin
+#     additional_data = {
+#      :running   => app.running?,
+#      :instances => app.running? ? app.stats.map{|key, value| value} : [],
+#      :error     => nil
+#     }
+#   rescue => e
+#     @logger.info "  #{org_name} #{space_name}: '#{app.name}'' error: #{e.message}"
+#     additional_data = {
+#       :running   => 'error',
+#       :instances => [],
+#       :error     => e.message
+#     }
+#   end
 
-  base_data.merge additional_data
-end
+#   base_data.merge additional_data
+# end
+
+# def routes_for_app app
+#   guids   = space.apps.first.routes.collect{|route| route.guid}
+#   guids.collect{|guid| @domains.select{|domain| domain[:guid] == guid}}
+#   @domains.collect
+# end
 
 def put_in_redis(key, data)
   REDIS.set key, data.to_json

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cf_light_api
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 2.0.0.pre1
 platform: ruby
 authors:
 - Springer Platform Engineering
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-22 00:00:00.000000000 Z
+date: 2016-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cfoundry
@@ -135,9 +135,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.0.14