boxxspring-workers 1.1.4 → 1.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5a8ab0b25057afe43c90af53902ec1410d5230c2
-  data.tar.gz: f57b1b142e88bac02e52d893f1824fd63e19cacf
+  metadata.gz: 28d891511f5e9885528269e401d43946f8eab6e9
+  data.tar.gz: 7c4d7828f39a16ccecc075387792ca63eba21879
 SHA512:
-  metadata.gz: 141d7d76f6209efe1b7beb56673094e1ed549dbf572dfef923524b4ab295ea35741b6961e4b655a612b91b0f52dabd8f81c47674fc74ca565b3f1c884e7d0e0a
-  data.tar.gz: 3f135b6596edf3a741e9df12d2870a9eafa8a3aded85205c198e17e94a2f651056452de89213be800bf81a54ed5bcfbb0ab38df2cb665786d91cb847b3a43abc
+  metadata.gz: 5e6b1079cc3a9b932f4d44ae19b584ea77e414a2929e193e2e4ff3cb1330dca72bd59eff298222aad07456435da083af7b518a286ee3a728ddbd2ca342e0bc9b
+  data.tar.gz: 94ba8c40b9ef4bbb5e2f04074acfefecd3f0376f2d8217fd6966f89ab36a6d6eb03fa3ca74b116bbb4c98baf679868e5cf5e34f8fc65eddff4e1129e19903c9c

data/lib/boxxspring/abstract.rb

@@ -13,6 +13,10 @@ module Boxxspring
       return @attributes
     end
 
+    def include?( name )
+      @attributes.include?( name )
+    end
+
     def method_missing( method, *arguments, &block )
       result = nil
       if arguments.length == 0 

data/lib/boxxspring/synchronization/orchestrator.rb

@@ -27,8 +27,15 @@ module Boxxspring
           true : false
       end
 
-      def read( key )
-        @provider.get( key )
+      def read( key, options = {} )
+        range = options[ :range ]
+        if range
+          range_start = range[ :start ] || 0
+          range_end   = range[ :end ] || -1
+          @provider.lrange( key, range_start, range_end )
+        else
+          @provider.get( key )
+        end
       end
 
       def write( key, value, options = {} )

data/lib/boxxspring/synchronization/variable.rb

@@ -8,8 +8,8 @@ module Boxxspring
         @name = name
       end
 
-      def read
-        @orchestrator.read( @name )
+      def read( options={} )
+        @orchestrator.read( @name, options )
       end
 
       def write( value, options={} )

data/lib/boxxspring/worker/base.rb

@@ -1,4 +1,4 @@
-module Boxxspring 
+module Boxxspring
 
   module Worker
 
@@ -7,16 +7,22 @@ module Boxxspring
 
     class Base
 
+      #------------------------------------------------------------------------
+      # modules
+
+      include Logging
+
       #------------------------------------------------------------------------
       # class attributes
 
       class_attribute :queue_name
       class_attribute :processor
+      class_attribute :environment
 
       #------------------------------------------------------------------------
       # class methods
 
-      class << self 
+      class << self
 
         public; def process( &block )
           self.processor = block
@@ -27,25 +33,27 @@ module Boxxspring
         end
 
         def queue_url
-          unless @queue_url.present?
-            response = self.queue_interface.create_queue( 
-              queue_name: self.full_queue_name 
+          @queue_url ||= begin
+            response = self.queue_interface.create_queue(
+              queue_name: self.full_queue_name
             )
-            @queue_url = response[ :queue_url ]       
+            response[ :queue_url ]
+          end
+        end
+
+        def environment
+          @environment ||= begin
+            Worker.env == 'development' ?
+              ( ENV[ 'USER' ].underscore || 'development' ) :
+              Worker.env
           end
-          @queue_url
         end
 
         protected; def full_queue_name
-          name = ( Worker.env == 'development' ) ?
-            ( ENV[ 'USER' ] || 'development' ) :
-            Worker.env 
-          name += '-' + 
-                  ( self.queue_name || 
-                    self.name.
-                      underscore.
-                      gsub( /[\/]/, '-' ).
-                      gsub( /_worker\Z/, '' ) )
+          queue_name = self.queue_name ||
+                       self.name.underscore.gsub( /[\/]/, '-' ).
+                         gsub( /_worker\Z/, '' )
+          self.environment + '-' + queue_name
         end
 
       end
@@ -53,11 +61,12 @@ module Boxxspring
       #------------------------------------------------------------------------
       # operations
 
-      def process 
-        messages = self.receive_messages() || [] 
+      def process
+        messages = self.receive_messages() || []
         messages.each do | message |
           if message.present?
             payload = self.payload_from_message( message )
+
             if payload.present?
               begin
                 result = self.process_payload( payload )
@@ -65,43 +74,37 @@ module Boxxspring
                 self.delete_message( message ) unless result == false
               rescue StandardError => error
                 self.logger.error(
-                  "The #{ self.human_name } worker failed to process the " + 
-                  "payload. "
+                  "The #{ self.human_name } failed to process the payload."
                 )
                 self.logger.error( error.message )
-                self.logger.error( error.backtrace.join( "\n" ) )
+                self.logger.info( error.backtrace.join( "\n" ) )
               end
             else
-              # note: messages with invalid payloads are deleted
               self.delete_message( message )
               self.logger.error(
-                "The #{ self.human_name } worker received an invalid payload."
+                "The #{ self.human_name } received an invalid payload."
               )
             end
           end
         end
       end
 
-      protected; def logger
-        @logger ||= Boxxspring::Worker.configuration.logger
-      end
-
       #------------------------------------------------------------------------
       # implementation
 
       protected; def receive_messages
         messages = nil
         begin
-          response = self.class.queue_interface.receive_message( 
+          response = self.class.queue_interface.receive_message(
             queue_url: self.class.queue_url,
             max_number_of_messages: QUEUE_MESSAGE_REQUEST_COUNT,
-            wait_time_seconds: QUEUE_MESSAGE_WAIT_IN_SECONDS 
+            wait_time_seconds: QUEUE_MESSAGE_WAIT_IN_SECONDS
           )
           messages = response[ :messages ]
         rescue StandardError => error
-          raise RuntimeError.new( 
-            "The #{ self.human_name } worker is unable to receive a message " +
-            "from the queue. #{error.message}."
+          raise RuntimeError.new(
+            "The #{ self.human_name } is unable to receive a message " +
+            "from the queue. #{ error.message }."
           )
         end
         messages
@@ -109,54 +112,59 @@ module Boxxspring
 
       protected; def delete_message( message )
         begin
-          self.class.queue_interface.delete_message( 
+          self.class.queue_interface.delete_message(
             queue_url: self.class.queue_url,
             receipt_handle: message[ :receipt_handle ]
           )
         rescue StandardError => error
-          raise RuntimeError.new( 
-            "The #{ self.human_name } worker is unable to delete the " + 
-            "message from the queue. #{error.message}."
+          raise RuntimeError.new(
+            "The #{ self.human_name } is unable to delete the " +
+            "message from the queue. #{ error.message }."
           )
         end
+
         message
       end
 
       protected; def payload_from_message( message )
+        
         payload = message.body
-        if ( payload.present? )
-          payload = JSON.parse( payload ) rescue payload 
-          if ( payload.is_a?( Hash ) && 
-               payload.include?( 'Type' ) && 
-               payload[ 'Type' ] == 'Notification' )
+
+        if payload.present?
+          payload = JSON.parse( payload ) rescue payload
+          if payload.is_a?( Hash ) && payload.include?( 'Type' ) &&
+             payload[ 'Type' ] == 'Notification'
             payload = payload[ 'Message' ]
-            payload = payload.present? ? 
-              ( JSON.parse( payload ) rescue payload ) : 
+            payload = payload.present? ?
+              ( JSON.parse( payload ) rescue payload ) :
               payload
-          end 
+          end
+        else
+          logger.error( "The message lacks a payload." ) 
+          logger.debug( message.inspect )
         end
         payload
       end
 
       protected; def process_payload( payload )
-        if self.class.processor.present? 
+        if self.class.processor.present?
           self.class.processor.call( payload )
         else
-          raise RuntimeError.new( 
+          raise RuntimeError.new(
             "The worker lacks a processor"
           )
         end
       end
 
       protected; def delegate_payload( queue_name, payload )
-        queue_name_prefix = ( Worker.env == 'development' ) ?
-          ( ENV[ 'USER' ] || 'development' ) : Worker.env 
-       queue_name = queue_name_prefix + '-' + queue_name
+        queue_name = self.class.environment + '-' + queue_name
+
         begin
-          response = self.class.queue_interface.create_queue( 
-            queue_name: queue_name 
+          response = self.class.queue_interface.create_queue(
+            queue_name: queue_name
           )
-          queue_url = response[ :queue_url ]       
+          queue_url = response[ :queue_url ]
+
           if queue_url.present?
             self.class.queue_interface.send_message(
               queue_url: queue_url,
@@ -164,18 +172,43 @@ module Boxxspring
             )
           end
         rescue StandardError => error
-          raise RuntimeError.new( 
-            "The #{ self.human_name } worker was unable to delegate the " + 
-            "payload to the queue name '#{ queue_name }'. #{error.message}."
+          raise RuntimeError.new(
+            "The #{ self.human_name } was unable to delegate the " +
+            "payload to the '#{ queue_name }' queue. #{ error.message }."
           )
         end
       end
 
+      # Meta API read & error handling
+      protected; def read_object( property_id=nil, type, id, includes )
+        if property_id.present?
+          endpoint = "/properties/#{ property_id }/#{ type.pluralize }/#{ id }"
+        else
+          endpoint = "/properties/#{ id }"
+        end
+
+        object = operation( endpoint ).include( *includes ).read
+        if error?( object, type.capitalize ) && object.is_a?( Array )
+          object.first
+        else
+          object
+        end
+      end
+
+      protected; def error?( object, object_class )
+        class_name = "Boxxspring::#{ object_class }".constantize
+        !object.is_a?( class_name ) || object.send( :errors ).present?
+      end
+
+      protected; def operation( endpoint )
+        Boxxspring::Operation.new(
+          endpoint,
+          Boxxspring::Worker.configuration.api_credentials.to_hash
+        )
+      end
+
       protected; def human_name
-        self.class.name.  
-          underscore.
-          gsub( /[\/]/, ' ' ).
-          gsub( /_worker\Z/, '' )
+        self.class.name.underscore.gsub('_', ' ')
       end
 
     end

data/lib/boxxspring/worker/configuration.rb

@@ -4,26 +4,10 @@ module Boxxspring
 
   module Worker
 
-    def self.configuration( &block )
-      Configuration.instance().instance_eval( &block ) unless block.nil?
-      Configuration.instance()
-    end
-
-    def self.env 
-      self.configuration.env
-    end
-
     class Configuration < Abstract
 
       include Singleton
 
-      def initialize
-        super( { 
-          env: ENV[ 'WORKERS_ENV' ] || 'development',
-          logger: Logger.new( STDOUT )
-        } )  
-      end
-
       def self.reloadable?
         false
       end
@@ -32,10 +16,31 @@ module Boxxspring
         configuration.each_pair do | name, value |
           self.send( "@#{name}", value )
         end
-      end  
+      end
+
+      def get_token( options = {} )
+
+        token_details = Unimatrix::Authorization::ClientCredentialsGrant.new(
+          client_id: ENV[ 'KEYMAKER_CLIENT' ],
+          client_secret: ENV[ 'KEYMAKER_SECRET' ]
+        ).request_token( options )
+
+        if token_details.include?( 'error' )
+          puts "ERROR: #{ token_details }"
+        end
 
+        if token_details.is_a?( Hash )
+          token_details.symbolize_keys!
+          token_details.update( expires_in: Time.now + token_details[ :expires_in ] ) if token_details[ :expires_in ]
+          api_credentials.access_token =  token_details[ :access_token ]
+          api_credentials.token_expiry =  token_details[ :expires_in ]
+          result = api_credentials
+        else
+          result = token_details
+        end
+        result
+      end
     end
-
   end
 
 end

data/lib/boxxspring/worker/logging.rb

@@ -0,0 +1,95 @@
+module Boxxspring
+
+  module Worker
+
+    module Logging
+
+      PWD = Dir.pwd.freeze
+
+      def logger
+
+        @logger ||= begin
+
+          workers_env = ENV[ 'WORKERS_ENV' ]
+
+          if Worker.configuration.include?( 'logger' )
+
+           logger = Worker.configuration.logger
+
+          else
+
+            if self.log_local? || workers_env == 'test'
+
+              logger = Logger.new( STDOUT )
+
+            else
+
+              group_name = self.log_group_name
+              raise 'A logging group is required. You may need to set LOG_GROUP.' \
+                unless group_name.present?
+
+              worker_name = self.human_name.gsub( ' ','_' )
+
+              if workers_env == 'development'
+                username = ENV[ 'USER' ] || ENV[ 'USERNAME' ]
+                username = username.underscore.dasherize
+
+                group_name = "#{ username }.#{ group_name }"
+              elsif workers_env != 'production'
+                group_name = "#{ workers_env }.#{ group_name }"
+              end
+
+              logger = CloudWatchLogger.new(
+                {
+                  access_key_id: ENV[ 'AWS_ACCESS_KEY_ID' ],
+                  secret_access_key: ENV[ 'AWS_SECRET_ACCESS_KEY' ]
+                },
+                group_name,
+                worker_name
+              )
+
+            end
+
+
+          end
+
+          logger.level = self.log_level
+          logger
+
+        end
+
+      end
+
+      protected; def log_group_name
+        group_name = ENV[ 'LOG_GROUP' ]
+        group_name ||= begin
+          File.open( File.join( PWD, 'GROUP' ), &:readline ) rescue nil
+        end
+        group_name ||= begin
+          name = `git config --get remote.origin.url` rescue nil
+          name.present? ? File.basename( name, '.*' ) : nil
+        end
+        group_name.strip
+      end
+
+      protected; def log_level
+        level = Logger::WARN
+        if ENV[ 'LOG_LEVEL' ].present?
+          level = ENV[ 'LOG_LEVEL' ].upcase
+          raise "An unkown log level was specificed by LOG_LEVEL." \
+            unless [ "INFO", "WARN", "ERROR", "DEBUG", "FATAL" ].include?( level )
+          level = "Logger::#{ level }".constantize
+        end
+        level
+      end
+
+      protected; def log_local?
+        log_local = ENV[ 'LOG_LOCAL' ] || 'false'
+        ( log_local.to_s =~ /^true$/i ) == 0
+      end
+
+    end
+
+  end
+
+end

data/lib/boxxspring/worker/task_base.rb

@@ -18,7 +18,7 @@ module Boxxspring
           self.class.processor.call( task )
         else
           raise RuntimeError.new( 
-            "The #{self.human_name} worker lacks a task processor"
+            "The #{ self.human_name } lacks a task processor"
           )
         end
       end
@@ -27,56 +27,49 @@ module Boxxspring
       # implementation
 
       protected; def process_payload( payload )
-
         result = true
         type_names = self.task_type_name.blank? ? 
-          nil : 
-          [ self.task_type_name ].flatten
+                       nil : [ self.task_type_name ].flatten
         states = self.task_state.blank? ?
-          nil :
-          [ self.task_state ].flatten
-
+                   nil : [ self.task_state ].flatten
         tasks = payload[ 'tasks' ]
-        if ( tasks.present? && tasks.respond_to?( :each ) ) 
+
+        if tasks.present? && tasks.respond_to?( :each )
           tasks.each do | task |
             task_id = task[ 'id' ]
-            if ( type_names.blank? || type_names.include?( task[ 'type_name' ] ) )
+            if type_names.blank? || type_names.include?( task[ 'type_name' ] )
               task = task_read( task[ 'property_id' ], task_id )
+
               if task.is_a?( Boxxspring::Task )
-                if ( states.blank? || states.include?( task.state ) )
+                if states.blank? || states.include?( task.state )
                   self.logger.info(  
-                    "The task (id: #{task.id}) processing has started."
+                    "Task #{ task.id } processing has started."
                   )
                   begin
                     result = self.process_task( task )
-                    message = "The task (id: #{task.id}) processing has ended"
-                    message += " and the message has been retained." if result == false
-                    self.logger.info(  
-                      message
-                    )
+                    message = "Task #{ task.id } processing has ended"
+                    message += " and the message was retained." if result == false
+                    self.logger.info( message )
                   rescue SignalException, StandardError => error
                     if error.is_a?( SignalException )
                       task_state = 'idle'
-                      task_message = "The task (id: #{task.id}) has restarted."
+                      task_message = "Task #{ task.id } has restarted."
                     else
                       task_state = 'failed'
-                      task_message = "The task (id: #{task.id}) processing has failed."
+                      task_message = "Task #{ task.id } processing has failed."
                     end
-                    task = task_write_state(task,
-                                            task_state,
-                                            task_message)
+                    task = task_write_state( task, task_state, task_message )
                     self.logger.error( error.message )
-                    self.logger.error( error.backtrace.join( "\n" ) )
+                    self.logger.info( error.backtrace.join( "\n" ) )
                     raise error if error.is_a?( SignalException )
                   end
                 end
-              elsif task.is_a?(Array) &&
-                    task.first.is_a?(Boxxspring::ForbiddenError)
-                self.logger.error(task.first.message)
+              elsif task.is_a?( Array ) && task.first.respond_to?( :message )
+                self.logger.error( task.first.message )
               else
                 self.logger.error(
-                  "The #{self.human_name} worker is unable to retrieve the " +
-                  "task with the id #{task_id}."
+                  "The #{self.human_name} is unable to retrieve the " +
+                  "task with the id #{task_id}. #{task.inspect}"
                 )
               end
             end
@@ -84,14 +77,13 @@ module Boxxspring
         end
 
         result
-      
       end
 
       protected; def task_read( property_id, task_id )
         # why did this not work?
         # self.task_operation( property_id ).where( id: task_id ).read
         Boxxspring::Operation.new( 
-          "/properties/#{property_id}/tasks/#{task_id}",
+          "/properties/#{ property_id }/tasks/#{ task_id }",
           Worker.configuration.api_credentials.to_hash
         ).read      
       end
@@ -137,15 +129,8 @@ module Boxxspring
         self.delegate_payload( queue_name, payload )
       end
 
-      protected; def operation(endpoint)
-        Boxxspring::Operation.new(
-          endpoint,
-          Boxxspring::Worker.configuration.api_credentials.to_hash
-        )
-      end
-
     end
 
   end
 
-end
+end

data/lib/boxxspring/worker.rb

@@ -0,0 +1,23 @@
+module Boxxspring
+
+  module Worker
+
+    def self.configuration( &block )
+      Configuration.instance().instance_eval( &block ) unless block.nil?
+      Configuration.instance()
+    end
+
+    def self.env 
+      @environment ||= ENV[ 'WORKERS_ENV' ] || 'development'
+    end
+
+    def self.root
+      @root ||= begin 
+        specification = Gem::Specification.find_by_name( 'boxxspring-workers' ) 
+        Pathname.new( specification.gem_dir )
+      end
+    end
+
+  end
+
+end

data/lib/boxxspring-worker-version.rb

@@ -1,5 +1,5 @@
 module Boxxspring
   module Worker
-    VERSION = '1.1.4'
+    VERSION = '1.1.5'
   end
 end

data/lib/boxxspring-worker.rb

@@ -4,6 +4,8 @@ require 'active_support/all'
 $LOAD_PATH.unshift( File.expand_path( '..', File.dirname( __FILE__ ) ) )
 require 'lib/boxxspring/abstract'
 require 'lib/boxxspring/journal'
+require 'lib/boxxspring/worker'
+require 'lib/boxxspring/worker/logging'
 require 'lib/boxxspring/worker/configuration'
 require 'lib/boxxspring/worker/base'
 require 'lib/boxxspring/worker/task_base'

data/lib/tasks/workers.rake

@@ -1,38 +1,34 @@
+require 'boxxspring-worker'
+
 namespace :worker do
 
   descendants = Boxxspring::Worker::Base.descendants
-
-  # remove base class workers
   descendants.delete( Boxxspring::Worker::TaskBase )
 
   descendants.each do | worker_class |
+    worker_name = worker_class.name.underscore.gsub(/_worker\Z/, '')
+    human_name  = worker_class.name.underscore.gsub('_', ' ')
 
-    worker_name = worker_class.
-                    name.
-                    underscore.
-                    gsub( /[\/]/, '-' ). 
-                    gsub( /_worker\Z/, '' )
-
-    desc "#{worker_name.humanize.downcase} worker."
+    desc "#{ human_name }"
     task worker_name.to_sym do
-      spinner = %w{| / - \\}
       worker = worker_class.new
+      logger = worker.logger
+
+      spinner = %w{| / - \\}
       print 'working...  ' 
-      Boxxspring::Worker.configuration.logger.info(
-        "The #{worker_name.humanize.downcase} worker has started." 
-      )
+      logger.info( "The #{ human_name } has started." )
+
       begin
         loop do 
           print "\b" + spinner.rotate!.first
           worker.process
         end
       rescue SystemExit, Interrupt
-        Boxxspring::Worker.configuration.logger.info(
-          "The #{worker_name.humanize.downcase} worker has stopped." 
-        )
+        logger.info( "The #{ human_name } has stopped." )
         puts 'stopped'
         exit 130
       end 
     end
   end
-end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: boxxspring-workers
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.1.5
 platform: ruby
 authors:
 - Kristoph Cichocki-Romanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-15 00:00:00.000000000 Z
+date: 2017-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -111,8 +111,10 @@ files:
 - lib/boxxspring/synchronization/operations.rb
 - lib/boxxspring/synchronization/orchestrator.rb
 - lib/boxxspring/synchronization/variable.rb
+- lib/boxxspring/worker.rb
 - lib/boxxspring/worker/base.rb
 - lib/boxxspring/worker/configuration.rb
+- lib/boxxspring/worker/logging.rb
 - lib/boxxspring/worker/task_base.rb
 - lib/tasks/workers.rake
 homepage: http://bedrocket.com
@@ -135,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Bedrocket Media Ventrures Boxxspring Worker framework.