eventhub-processor 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/lib/eventhub-processor.rb +37 -37
 - data/lib/eventhub/argument_parser.rb +34 -34
 - data/lib/eventhub/base_exception.rb +1 -1
 - data/lib/eventhub/configuration.rb +25 -25
 - data/lib/eventhub/constant.rb +15 -15
 - data/lib/eventhub/hash_extensions.rb +48 -48
 - data/lib/eventhub/heartbeat.rb +77 -77
 - data/lib/eventhub/helper.rb +49 -49
 - data/lib/eventhub/message.rb +138 -138
 - data/lib/eventhub/message_processor.rb +31 -31
 - data/lib/eventhub/no_deadletter_exception.rb +1 -1
 - data/lib/eventhub/processor.rb +269 -268
 - data/lib/eventhub/statistics.rb +47 -47
 - data/lib/eventhub/version.rb +3 -3
 - metadata +17 -5
 - data/lib/eventhub/multi_logger.rb +0 -89
 - data/lib/eventhub/pidfile.rb +0 -21
 - data/lib/eventhub/test.rb +0 -10
 
    
        data/lib/eventhub/statistics.rb
    CHANGED
    
    | 
         @@ -1,47 +1,47 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            class EventHub::Statistics
         
     | 
| 
       2 
     | 
    
         
            -
              attr_reader :messages_successful, :messages_unsuccessful, :messages_average_size, :messages_average_process_time
         
     | 
| 
       3 
     | 
    
         
            -
             
     | 
| 
       4 
     | 
    
         
            -
              def initialize
         
     | 
| 
       5 
     | 
    
         
            -
                @messages_successful = 0
         
     | 
| 
       6 
     | 
    
         
            -
                @messages_unsuccessful = 0
         
     | 
| 
       7 
     | 
    
         
            -
                @messages_average_size = 0
         
     | 
| 
       8 
     | 
    
         
            -
                @messages_average_process_time = 0
         
     | 
| 
       9 
     | 
    
         
            -
                @messages_total_process_time = 0
         
     | 
| 
       10 
     | 
    
         
            -
              end
         
     | 
| 
       11 
     | 
    
         
            -
             
     | 
| 
       12 
     | 
    
         
            -
             
     | 
| 
       13 
     | 
    
         
            -
              def measure(size, &block)
         
     | 
| 
       14 
     | 
    
         
            -
                begin
         
     | 
| 
       15 
     | 
    
         
            -
                  start = Time.now
         
     | 
| 
       16 
     | 
    
         
            -
                  yield
         
     | 
| 
       17 
     | 
    
         
            -
                  success(Time.now - start, size)
         
     | 
| 
       18 
     | 
    
         
            -
                rescue
         
     | 
| 
       19 
     | 
    
         
            -
                  failure
         
     | 
| 
       20 
     | 
    
         
            -
                  raise
         
     | 
| 
       21 
     | 
    
         
            -
                end
         
     | 
| 
       22 
     | 
    
         
            -
              end
         
     | 
| 
       23 
     | 
    
         
            -
             
     | 
| 
       24 
     | 
    
         
            -
              def success(process_time, size)
         
     | 
| 
       25 
     | 
    
         
            -
                @messages_total_process_time += process_time
         
     | 
| 
       26 
     | 
    
         
            -
                @messages_average_process_time = (messages_total_process_time + process_time) / (messages_successful + 1).to_f
         
     | 
| 
       27 
     | 
    
         
            -
                @messages_average_size = (messages_total_size + size) / (messages_successful + 1).to_f
         
     | 
| 
       28 
     | 
    
         
            -
                @messages_successful += 1
         
     | 
| 
       29 
     | 
    
         
            -
              end
         
     | 
| 
       30 
     | 
    
         
            -
             
     | 
| 
       31 
     | 
    
         
            -
              def failure
         
     | 
| 
       32 
     | 
    
         
            -
                @messages_unsuccessful += 1
         
     | 
| 
       33 
     | 
    
         
            -
              end
         
     | 
| 
       34 
     | 
    
         
            -
             
     | 
| 
       35 
     | 
    
         
            -
              def messages_total
         
     | 
| 
       36 
     | 
    
         
            -
                messages_unsuccessful + messages_successful
         
     | 
| 
       37 
     | 
    
         
            -
              end
         
     | 
| 
       38 
     | 
    
         
            -
             
     | 
| 
       39 
     | 
    
         
            -
              def messages_total_process_time
         
     | 
| 
       40 
     | 
    
         
            -
                messages_average_process_time * messages_successful
         
     | 
| 
       41 
     | 
    
         
            -
              end
         
     | 
| 
       42 
     | 
    
         
            -
             
     | 
| 
       43 
     | 
    
         
            -
              def messages_total_size
         
     | 
| 
       44 
     | 
    
         
            -
                messages_average_size * messages_successful
         
     | 
| 
       45 
     | 
    
         
            -
              end
         
     | 
| 
       46 
     | 
    
         
            -
             
     | 
| 
       47 
     | 
    
         
            -
            end
         
     | 
| 
      
 1 
     | 
    
         
            +
            class EventHub::Statistics
         
     | 
| 
      
 2 
     | 
    
         
            +
              attr_reader :messages_successful, :messages_unsuccessful, :messages_average_size, :messages_average_process_time
         
     | 
| 
      
 3 
     | 
    
         
            +
             
     | 
| 
      
 4 
     | 
    
         
            +
              def initialize
         
     | 
| 
      
 5 
     | 
    
         
            +
                @messages_successful = 0
         
     | 
| 
      
 6 
     | 
    
         
            +
                @messages_unsuccessful = 0
         
     | 
| 
      
 7 
     | 
    
         
            +
                @messages_average_size = 0
         
     | 
| 
      
 8 
     | 
    
         
            +
                @messages_average_process_time = 0
         
     | 
| 
      
 9 
     | 
    
         
            +
                @messages_total_process_time = 0
         
     | 
| 
      
 10 
     | 
    
         
            +
              end
         
     | 
| 
      
 11 
     | 
    
         
            +
             
     | 
| 
      
 12 
     | 
    
         
            +
             
     | 
| 
      
 13 
     | 
    
         
            +
              def measure(size, &block)
         
     | 
| 
      
 14 
     | 
    
         
            +
                begin
         
     | 
| 
      
 15 
     | 
    
         
            +
                  start = Time.now
         
     | 
| 
      
 16 
     | 
    
         
            +
                  yield
         
     | 
| 
      
 17 
     | 
    
         
            +
                  success(Time.now - start, size)
         
     | 
| 
      
 18 
     | 
    
         
            +
                rescue
         
     | 
| 
      
 19 
     | 
    
         
            +
                  failure
         
     | 
| 
      
 20 
     | 
    
         
            +
                  raise
         
     | 
| 
      
 21 
     | 
    
         
            +
                end
         
     | 
| 
      
 22 
     | 
    
         
            +
              end
         
     | 
| 
      
 23 
     | 
    
         
            +
             
     | 
| 
      
 24 
     | 
    
         
            +
              def success(process_time, size)
         
     | 
| 
      
 25 
     | 
    
         
            +
                @messages_total_process_time += process_time
         
     | 
| 
      
 26 
     | 
    
         
            +
                @messages_average_process_time = (messages_total_process_time + process_time) / (messages_successful + 1).to_f
         
     | 
| 
      
 27 
     | 
    
         
            +
                @messages_average_size = (messages_total_size + size) / (messages_successful + 1).to_f
         
     | 
| 
      
 28 
     | 
    
         
            +
                @messages_successful += 1
         
     | 
| 
      
 29 
     | 
    
         
            +
              end
         
     | 
| 
      
 30 
     | 
    
         
            +
             
     | 
| 
      
 31 
     | 
    
         
            +
              def failure
         
     | 
| 
      
 32 
     | 
    
         
            +
                @messages_unsuccessful += 1
         
     | 
| 
      
 33 
     | 
    
         
            +
              end
         
     | 
| 
      
 34 
     | 
    
         
            +
             
     | 
| 
      
 35 
     | 
    
         
            +
              def messages_total
         
     | 
| 
      
 36 
     | 
    
         
            +
                messages_unsuccessful + messages_successful
         
     | 
| 
      
 37 
     | 
    
         
            +
              end
         
     | 
| 
      
 38 
     | 
    
         
            +
             
     | 
| 
      
 39 
     | 
    
         
            +
              def messages_total_process_time
         
     | 
| 
      
 40 
     | 
    
         
            +
                messages_average_process_time * messages_successful
         
     | 
| 
      
 41 
     | 
    
         
            +
              end
         
     | 
| 
      
 42 
     | 
    
         
            +
             
     | 
| 
      
 43 
     | 
    
         
            +
              def messages_total_size
         
     | 
| 
      
 44 
     | 
    
         
            +
                messages_average_size * messages_successful
         
     | 
| 
      
 45 
     | 
    
         
            +
              end
         
     | 
| 
      
 46 
     | 
    
         
            +
             
     | 
| 
      
 47 
     | 
    
         
            +
            end
         
     | 
    
        data/lib/eventhub/version.rb
    CHANGED
    
    | 
         @@ -1,3 +1,3 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            module EventHub
         
     | 
| 
       2 
     | 
    
         
            -
              VERSION = "0. 
     | 
| 
       3 
     | 
    
         
            -
            end
         
     | 
| 
      
 1 
     | 
    
         
            +
            module EventHub
         
     | 
| 
      
 2 
     | 
    
         
            +
              VERSION = "0.4.0"
         
     | 
| 
      
 3 
     | 
    
         
            +
            end
         
     | 
    
        metadata
    CHANGED
    
    | 
         @@ -1,14 +1,14 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            --- !ruby/object:Gem::Specification
         
     | 
| 
       2 
2 
     | 
    
         
             
            name: eventhub-processor
         
     | 
| 
       3 
3 
     | 
    
         
             
            version: !ruby/object:Gem::Version
         
     | 
| 
       4 
     | 
    
         
            -
              version: 0. 
     | 
| 
      
 4 
     | 
    
         
            +
              version: 0.4.0
         
     | 
| 
       5 
5 
     | 
    
         
             
            platform: ruby
         
     | 
| 
       6 
6 
     | 
    
         
             
            authors:
         
     | 
| 
       7 
7 
     | 
    
         
             
            - Thomas Steiner
         
     | 
| 
       8 
8 
     | 
    
         
             
            autorequire: 
         
     | 
| 
       9 
9 
     | 
    
         
             
            bindir: bin
         
     | 
| 
       10 
10 
     | 
    
         
             
            cert_chain: []
         
     | 
| 
       11 
     | 
    
         
            -
            date: 2014-11- 
     | 
| 
      
 11 
     | 
    
         
            +
            date: 2014-11-20 00:00:00.000000000 Z
         
     | 
| 
       12 
12 
     | 
    
         
             
            dependencies:
         
     | 
| 
       13 
13 
     | 
    
         
             
            - !ruby/object:Gem::Dependency
         
     | 
| 
       14 
14 
     | 
    
         
             
              name: bundler
         
     | 
| 
         @@ -138,6 +138,20 @@ dependencies: 
     | 
|
| 
       138 
138 
     | 
    
         
             
                - - ">="
         
     | 
| 
       139 
139 
     | 
    
         
             
                  - !ruby/object:Gem::Version
         
     | 
| 
       140 
140 
     | 
    
         
             
                    version: '2.1'
         
     | 
| 
      
 141 
     | 
    
         
            +
            - !ruby/object:Gem::Dependency
         
     | 
| 
      
 142 
     | 
    
         
            +
              name: eventhub-components
         
     | 
| 
      
 143 
     | 
    
         
            +
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
      
 144 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 145 
     | 
    
         
            +
                - - ">="
         
     | 
| 
      
 146 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 147 
     | 
    
         
            +
                    version: 0.0.5
         
     | 
| 
      
 148 
     | 
    
         
            +
              type: :runtime
         
     | 
| 
      
 149 
     | 
    
         
            +
              prerelease: false
         
     | 
| 
      
 150 
     | 
    
         
            +
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
      
 151 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 152 
     | 
    
         
            +
                - - ">="
         
     | 
| 
      
 153 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 154 
     | 
    
         
            +
                    version: 0.0.5
         
     | 
| 
       141 
155 
     | 
    
         
             
            description: Gem to build Event Hub processors
         
     | 
| 
       142 
156 
     | 
    
         
             
            email:
         
     | 
| 
       143 
157 
     | 
    
         
             
            - thomas.steiner@ikey.ch
         
     | 
| 
         @@ -156,12 +170,9 @@ files: 
     | 
|
| 
       156 
170 
     | 
    
         
             
            - lib/eventhub/helper.rb
         
     | 
| 
       157 
171 
     | 
    
         
             
            - lib/eventhub/message.rb
         
     | 
| 
       158 
172 
     | 
    
         
             
            - lib/eventhub/message_processor.rb
         
     | 
| 
       159 
     | 
    
         
            -
            - lib/eventhub/multi_logger.rb
         
     | 
| 
       160 
173 
     | 
    
         
             
            - lib/eventhub/no_deadletter_exception.rb
         
     | 
| 
       161 
     | 
    
         
            -
            - lib/eventhub/pidfile.rb
         
     | 
| 
       162 
174 
     | 
    
         
             
            - lib/eventhub/processor.rb
         
     | 
| 
       163 
175 
     | 
    
         
             
            - lib/eventhub/statistics.rb
         
     | 
| 
       164 
     | 
    
         
            -
            - lib/eventhub/test.rb
         
     | 
| 
       165 
176 
     | 
    
         
             
            - lib/eventhub/version.rb
         
     | 
| 
       166 
177 
     | 
    
         
             
            homepage: http://github.com/thomis/eventhub-processor
         
     | 
| 
       167 
178 
     | 
    
         
             
            licenses:
         
     | 
| 
         @@ -188,3 +199,4 @@ signing_key: 
     | 
|
| 
       188 
199 
     | 
    
         
             
            specification_version: 4
         
     | 
| 
       189 
200 
     | 
    
         
             
            summary: Gem to build Event Hub processors
         
     | 
| 
       190 
201 
     | 
    
         
             
            test_files: []
         
     | 
| 
      
 202 
     | 
    
         
            +
            has_rdoc: 
         
     | 
| 
         @@ -1,89 +0,0 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            require 'logger'
         
     | 
| 
       2 
     | 
    
         
            -
             
     | 
| 
       3 
     | 
    
         
            -
            # format adaptation
         
     | 
| 
       4 
     | 
    
         
            -
            class Logger
         
     | 
| 
       5 
     | 
    
         
            -
              class Formatter
         
     | 
| 
       6 
     | 
    
         
            -
                def call(severity, time, progname, msg)
         
     | 
| 
       7 
     | 
    
         
            -
                  time_in_string = "#{time.strftime("%Y-%m-%d %H:%M:%S")}.#{"%04d" % (time.usec/100)}"
         
     | 
| 
       8 
     | 
    
         
            -
                  [time_in_string,Process.pid,severity,msg].join("\t") + "\n"
         
     | 
| 
       9 
     | 
    
         
            -
                end
         
     | 
| 
       10 
     | 
    
         
            -
              end
         
     | 
| 
       11 
     | 
    
         
            -
             
     | 
| 
       12 
     | 
    
         
            -
            end
         
     | 
| 
       13 
     | 
    
         
            -
             
     | 
| 
       14 
     | 
    
         
            -
             
     | 
| 
       15 
     | 
    
         
            -
            module EventHub
         
     | 
| 
       16 
     | 
    
         
            -
             
     | 
| 
       17 
     | 
    
         
            -
              class MultiLogger
         
     | 
| 
       18 
     | 
    
         
            -
             
     | 
| 
       19 
     | 
    
         
            -
                MAX_EXCEPTIONS_FILES = 500
         
     | 
| 
       20 
     | 
    
         
            -
             
     | 
| 
       21 
     | 
    
         
            -
                attr_accessor :folder, :devices
         
     | 
| 
       22 
     | 
    
         
            -
             
     | 
| 
       23 
     | 
    
         
            -
                def initialize(folder = nil)
         
     | 
| 
       24 
     | 
    
         
            -
                  @folder_base = folder || Dir.pwd
         
     | 
| 
       25 
     | 
    
         
            -
                  @folder_base.chomp!('/')
         
     | 
| 
       26 
     | 
    
         
            -
                  @folder            = [@folder_base,'logs'].join('/')
         
     | 
| 
       27 
     | 
    
         
            -
                  @folder_exceptions = [@folder_base,'exceptions'].join('/')
         
     | 
| 
       28 
     | 
    
         
            -
             
     | 
| 
       29 
     | 
    
         
            -
                  @devices = []
         
     | 
| 
       30 
     | 
    
         
            -
             
     | 
| 
       31 
     | 
    
         
            -
                  FileUtils.makedirs(@folder)
         
     | 
| 
       32 
     | 
    
         
            -
                end
         
     | 
| 
       33 
     | 
    
         
            -
             
     | 
| 
       34 
     | 
    
         
            -
                def add_device(device)
         
     | 
| 
       35 
     | 
    
         
            -
                  @devices << device
         
     | 
| 
       36 
     | 
    
         
            -
                end
         
     | 
| 
       37 
     | 
    
         
            -
             
     | 
| 
       38 
     | 
    
         
            -
                def save_detailed_error(feedback,message=nil)
         
     | 
| 
       39 
     | 
    
         
            -
                	time = Time.now
         
     | 
| 
       40 
     | 
    
         
            -
                  stamp = "#{time.strftime("%Y%m%d_%H%M%S")}_#{"%03d" % (time.usec/1000)}"
         
     | 
| 
       41 
     | 
    
         
            -
                  filename = "#{stamp}.log"
         
     | 
| 
       42 
     | 
    
         
            -
             
     | 
| 
       43 
     | 
    
         
            -
                	FileUtils.makedirs(@folder_exceptions)
         
     | 
| 
       44 
     | 
    
         
            -
             
     | 
| 
       45 
     | 
    
         
            -
                  # check max exception log files
         
     | 
| 
       46 
     | 
    
         
            -
                  exception_files = Dir.glob(@folder_exceptions + '/*.log')
         
     | 
| 
       47 
     | 
    
         
            -
                  if exception_files.size > MAX_EXCEPTIONS_FILES
         
     | 
| 
       48 
     | 
    
         
            -
                    exception_files.reverse[MAX_EXCEPTIONS_FILES..-1].each do |file|
         
     | 
| 
       49 
     | 
    
         
            -
                      begin
         
     | 
| 
       50 
     | 
    
         
            -
                        File.delete(file)
         
     | 
| 
       51 
     | 
    
         
            -
                        File.delete(File.dirname(file) + '/' + File.basename(file,".*") + '.msg.raw')
         
     | 
| 
       52 
     | 
    
         
            -
                      rescue
         
     | 
| 
       53 
     | 
    
         
            -
                      end
         
     | 
| 
       54 
     | 
    
         
            -
                    end
         
     | 
| 
       55 
     | 
    
         
            -
                  end
         
     | 
| 
       56 
     | 
    
         
            -
             
     | 
| 
       57 
     | 
    
         
            -
                  File.open("#{@folder_exceptions}/#{filename}","w") do |output|
         
     | 
| 
       58 
     | 
    
         
            -
                    output.write("#{feedback}\n\n")
         
     | 
| 
       59 
     | 
    
         
            -
                    output.write("Exception: #{feedback.class.to_s}\n\n")
         
     | 
| 
       60 
     | 
    
         
            -
                    output.write("Call Stack:\n")
         
     | 
| 
       61 
     | 
    
         
            -
                    feedback.backtrace.each do |line|
         
     | 
| 
       62 
     | 
    
         
            -
                      output.write("#{line}\n")
         
     | 
| 
       63 
     | 
    
         
            -
                    end
         
     | 
| 
       64 
     | 
    
         
            -
                  end
         
     | 
| 
       65 
     | 
    
         
            -
             
     | 
| 
       66 
     | 
    
         
            -
                  # save message if provided
         
     | 
| 
       67 
     | 
    
         
            -
                  if message
         
     | 
| 
       68 
     | 
    
         
            -
              	    File.open("#{@folder_exceptions}/#{stamp}.msg.raw","wb") do |output|
         
     | 
| 
       69 
     | 
    
         
            -
              	    	output.write(message)
         
     | 
| 
       70 
     | 
    
         
            -
              	    end
         
     | 
| 
       71 
     | 
    
         
            -
                	end
         
     | 
| 
       72 
     | 
    
         
            -
             
     | 
| 
       73 
     | 
    
         
            -
                  return stamp
         
     | 
| 
       74 
     | 
    
         
            -
                end
         
     | 
| 
       75 
     | 
    
         
            -
             
     | 
| 
       76 
     | 
    
         
            -
                %w(log debug info warn error).each do |m|
         
     | 
| 
       77 
     | 
    
         
            -
                  define_method(m) do |*args|
         
     | 
| 
       78 
     | 
    
         
            -
                      @devices.map { |d| d.send(m, *args) }
         
     | 
| 
       79 
     | 
    
         
            -
                  end
         
     | 
| 
       80 
     | 
    
         
            -
                end
         
     | 
| 
       81 
     | 
    
         
            -
             
     | 
| 
       82 
     | 
    
         
            -
              end
         
     | 
| 
       83 
     | 
    
         
            -
             
     | 
| 
       84 
     | 
    
         
            -
            end
         
     | 
| 
       85 
     | 
    
         
            -
             
     | 
| 
       86 
     | 
    
         
            -
             
     | 
| 
       87 
     | 
    
         
            -
             
     | 
| 
       88 
     | 
    
         
            -
             
     | 
| 
       89 
     | 
    
         
            -
             
     | 
    
        data/lib/eventhub/pidfile.rb
    DELETED
    
    | 
         @@ -1,21 +0,0 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            class EventHub::Pidfile
         
     | 
| 
       2 
     | 
    
         
            -
              attr_reader :file
         
     | 
| 
       3 
     | 
    
         
            -
              def initialize(file)
         
     | 
| 
       4 
     | 
    
         
            -
                @file = file
         
     | 
| 
       5 
     | 
    
         
            -
              end
         
     | 
| 
       6 
     | 
    
         
            -
             
     | 
| 
       7 
     | 
    
         
            -
              # write the pid to the file specified in the initializer
         
     | 
| 
       8 
     | 
    
         
            -
              def write(pid)
         
     | 
| 
       9 
     | 
    
         
            -
                FileUtils.makedirs(File.dirname(file))
         
     | 
| 
       10 
     | 
    
         
            -
                IO.write(file, pid.to_s)
         
     | 
| 
       11 
     | 
    
         
            -
              end
         
     | 
| 
       12 
     | 
    
         
            -
             
     | 
| 
       13 
     | 
    
         
            -
              # Try to delete file, ignore all errors
         
     | 
| 
       14 
     | 
    
         
            -
              def delete
         
     | 
| 
       15 
     | 
    
         
            -
                begin
         
     | 
| 
       16 
     | 
    
         
            -
                  File.delete(file)
         
     | 
| 
       17 
     | 
    
         
            -
                rescue
         
     | 
| 
       18 
     | 
    
         
            -
                  # ignore
         
     | 
| 
       19 
     | 
    
         
            -
                end
         
     | 
| 
       20 
     | 
    
         
            -
              end
         
     | 
| 
       21 
     | 
    
         
            -
            end
         
     |