cf_light_api 1.3.1 → 1.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e022501ee1125531178e1d883f0ca616f0877c2
-  data.tar.gz: 36785e5ffd9c69455b6333895ae8811c2d22261f
+  metadata.gz: c72fd921339d1874c9b1086404e38afee9f88d5f
+  data.tar.gz: e3f5e8773925d7515b6ecbb917054159fde095d4
 SHA512:
-  metadata.gz: 7a7d27982b5759dba905703c308cc03aa4d754d3945e537143cd7b28df82d96383f58b6d8a35ee587b6fc0bcb2bee6a22b6baa6c25738efb45155f1adb71f85d
-  data.tar.gz: 153f4bca6ed48fe0772b1e4f4fee2e723fa3be7c8d58ce9d38c70cc32d5d53995bd4418888dddf2d29fbb4f1fdc0363aacc9f62dcc4801d90ee55048e690088c
+  metadata.gz: 55bb2150f1c2d22ba2d483ddaea27b26e7649433c097531e9997478147ca06b938e0aa2fbeff2d8a45c20ef755f3e45e8cbbfd4f33e1b18623ca471c079b210b
+  data.tar.gz: 5b30c97ec7f672fc1f50577dbdaab07cdfac22bfdbb5b8dbf2497af72803ab83df76762e2c36c31714a08a74fe05461b7d280908591e32cd0e8b361c55fbd2e5

data/lib/cf_light_api/worker.rb

@@ -2,84 +2,89 @@ require 'cfoundry'
 require 'json'
 require 'rufus-scheduler'
 require 'parallel'
+require 'redlock'
+require 'logger'
+
+@logger = Logger.new(STDOUT)
+@logger.formatter = proc do |severity, datetime, progname, msg|
+   "#{datetime} [cf_light_api:worker]: #{msg}\n"
+end
 
 ['CF_API', 'CF_USER', 'CF_PASSWORD'].each do |env|
-  puts "[cf_light_api:worker] Error: please set the '#{env}' environment variable." unless ENV[env]
+  @logger.info "Error: please set the '#{env}' environment variable." unless ENV[env]
   next
 end
 
+lock_manager = Redlock::Client.new([ENV['REDIS_URI']])
 scheduler = Rufus::Scheduler.new
 scheduler.every '5m', :first_in => '5s', :overlap => false, :timeout => '5m' do
   begin
-    if locked?
-      puts "[cf_light_api:worker] Data update is already running in another worker, skipping..."
-      next
-    end
-
-    lock
-    start_time = Time.now
+    lock_manager.lock("#{ENV['REDIS_KEY_PREFIX']}:lock", 5*60*1000) do |lock|
+      if lock
+        start_time = Time.now
 
-    puts "[cf_light_api:worker] Updating data..."
+        @logger.info "Updating data..."
 
-    cf_client = get_client()
+        cf_client = get_client()
 
-    org_data = []
-    app_data = []
-    org_data = Parallel.map( cf_client.organizations, :in_processes => 4) do |org|
-      # The CFoundry client returns memory_limit in MB, so we need to normalise to Bytes to match the Apps.
-      { 
-        :name => org.name,
-        :quota => {
-          :total_services => org.quota_definition.total_services,
-          :memory_limit   => org.quota_definition.memory_limit * 1024 * 1024
-        }
-      }
-    end.flatten
-
-    app_data = Parallel.map(cf_client.organizations, :in_processes => 4) do |org|
-      Parallel.map(org.spaces, :in_processes => 4) do |space|
-        Parallel.map(space.apps, :in_processes => 4) do |app|
-          begin
-            # It's possible for an app to have been terminated before this stage is reached.
-            format_app_data(app, org.name, space.name)
-          rescue CFoundry::AppNotFound
-            next
-          end
-        end
-      end
-    end.flatten
+        org_data = get_org_data(cf_client)
+        app_data = get_app_data(cf_client)
 
-    unlock
-
-    put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:orgs", org_data
-    put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:apps", app_data
-
-    puts "[cf_light_api:worker] Update completed in #{format_duration(Time.now.to_f - start_time.to_f)}..."
+        put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:orgs", org_data
+        put_in_redis "#{ENV['REDIS_KEY_PREFIX']}:apps", app_data
 
+        @logger.info "Update completed in #{format_duration(Time.now.to_f - start_time.to_f)}..."
+        lock_manager.unlock(lock)
+        cf_client.logout
+      else
+        @logger.info "Update already running in another thread!"
+      end
+    end
   rescue Rufus::Scheduler::TimeoutError
-    puts '[cf_light_api:worker] Data update took too long and was aborted...'
+    @logger.info 'Data update took too long and was aborted...'
+    cf_client.logout
   end
 end
 
-def locked?
-  REDIS.get("#{ENV['REDIS_KEY_PREFIX']}:lock") ? true : false
-end
-
-def lock
-  REDIS.set "#{ENV['REDIS_KEY_PREFIX']}:lock", true
-  REDIS.expire "#{ENV['REDIS_KEY_PREFIX']}:lock", 900
-end
-
-def unlock
-  REDIS.del "#{ENV['REDIS_KEY_PREFIX']}:lock"
-end
-
 def get_client(cf_api=ENV['CF_API'], cf_user=ENV['CF_USER'], cf_password=ENV['CF_PASSWORD'])
   client = CFoundry::Client.get(cf_api)
   client.login({:username => cf_user, :password => cf_password})
   client
 end
 
+def get_app_data(cf_client)
+  Parallel.map(cf_client.organizations, :in_processes => 4) do |org|
+    org_name = org.name
+    Parallel.map(org.spaces, :in_processes => 4) do |space|
+      space_name = space.name
+      @logger.info "Getting app data for apps in #{org_name}:#{space_name}..."
+      Parallel.map(space.apps, :in_processes => 4) do |app|
+        begin
+          # It's possible for an app to have been terminated before this stage is reached.
+          format_app_data(app, org_name, space_name)
+        rescue CFoundry::AppNotFound
+          next
+        end
+      end
+    end
+  end.flatten.compact
+end
+
+def get_org_data(cf_client)
+  Parallel.map( cf_client.organizations, :in_processes => 4) do |org|
+    org_name = org.name
+    @logger.info "Getting org data for #{org_name}..."
+    # The CFoundry client returns memory_limit in MB, so we need to normalise to Bytes to match the Apps.
+    {
+      :name => org_name,
+      :quota => {
+        :total_services => org.quota_definition.total_services,
+        :memory_limit   => org.quota_definition.memory_limit * 1024 * 1024
+      }
+    }
+  end.flatten.compact
+end
+
 def format_app_data(app, org_name, space_name)
   base_data = {
     :guid      => app.guid,
@@ -99,7 +104,7 @@ def format_app_data(app, org_name, space_name)
      :error     => nil
     }
   rescue => e
-    puts "[cf_light_api:worker] #{org_name} #{space_name}: '#{app.name}'' error: #{e.message}"
+    @logger.info "  #{org_name} #{space_name}: '#{app.name}'' error: #{e.message}"
     additional_data = {
       :running   => 'error',
       :instances => [],
@@ -111,7 +116,6 @@ def format_app_data(app, org_name, space_name)
 end
 
 def put_in_redis(key, data)
-  puts "[cf_light_api:worker] Putting data #{data} into redis key #{key}"
   REDIS.set key, data.to_json
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cf_light_api
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.3.2
 platform: ruby
 authors:
 - Springer Platform Engineering
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-28 00:00:00.000000000 Z
+date: 2015-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cfoundry
@@ -80,6 +80,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.4.5
+- !ruby/object:Gem::Dependency
+  name: redlock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.1
 description: A super lightweight API for reading App and Org data from CloudFoundry,
   cached in Redis.
 email: ''
@@ -88,9 +102,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ./lib/sinatra/cf_light_api.rb
 - ./lib/cf_light_api/redis.rb
 - ./lib/cf_light_api/worker.rb
+- ./lib/sinatra/cf_light_api.rb
 - bin/cf_light_api
 homepage: https://github.com/springerpe/cf-light-api
 licenses:
@@ -112,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: A super lightweight API for reading App and Org data from CloudFoundry, cached