opener-daemons 0.4.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 92495f6b7af2acc1f0f83462c71f21a52bb25f4f
-  data.tar.gz: e6fea06acac2ee6bf859a76768a68549f927aff0
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OWIyYmNmYWY3MzVjMmYzNzAwYzIzMTk4ZDlhMzAwYThhMjE4ZTdlNw==
+  data.tar.gz: !binary |-
+    MzMzNDQwOTg0OTgwYWVkZjMxNGI2ODU4MWE1YmYxZDUxMjQyY2Q5Ng==
 SHA512:
-  metadata.gz: ad2cd39279379f1323008b65284893bd6ce81ff16dc5b931496747c3b35c931c1235315134cd3778a0e005b20de0f9e0258d56ec06c391a99e56f0cb33d93e5f
-  data.tar.gz: c4cad8ceb7fb4195ba8e7786288d4ccbfb30145a7e30425fb34e91a3fd2ea889cdef563a9bf5afa2a6f46ea549d41dfe452b08f9ff4b13eb656de19a25e68058
+  metadata.gz: !binary |-
+    YzdkZDFmNTZmZTYzZmQ1ZTMzODNkMmY0YWQ5MTFmYTg1ZDM1MGE1Y2QxMzc2
+    YzdkMDczNjk1NTc3NjVmNzc2OTEzYzBkOWJmMzg4MjA4N2NlNjc5ZDIzNDAx
+    OGFhN2E2ZGM0N2JjZjA0ZmNhOTNjMDU4OWQ3MjMyNWMzMGZjMmY=
+  data.tar.gz: !binary |-
+    YjBhYzJjZTAwOTZkNjg3MTQ3MjRmZDU0YjFjYzY2ZjlmYjE5N2MzMWRiYTUz
+    OGRjZjE5MGFlN2QwNzNlMGRlYmRhNzViZWUzMDNlZmZhYjA2NWE1YmY3OTdi
+    NWRmYTFhZWY5ZDhkYjNmYTAwYmI1NzY3MjI5MjQ1Nzk1YWQyYmQ=

data/lib/opener/daemons/daemon.rb

@@ -2,10 +2,13 @@ require 'thread'
 require 'opener/daemons/sqs'
 require 'json'
 
+Encoding.default_internal = Encoding::UTF_8
+Encoding.default_external = Encoding::UTF_8
+
 module Opener
   module Daemons
     class Daemon
-      attr_reader :batch_size,
+      attr_reader :batch_size, :buffer_size, :sleep_interval,
                   :input_queue, :output_queue,
                   :input_buffer, :output_buffer,
                   :klass,
@@ -14,27 +17,34 @@ module Opener
       attr_accessor :threads, :thread_counts
 
       def initialize(klass, options={})
-        @input_queue  = Opener::Daemons::SQS.find(options.fetch(:input_queue))
-        @output_queue = Opener::Daemons::SQS.find(options.fetch(:output_queue))
 
-        @threads = {}
-        @threads[:readers] = []
-        @threads[:workers] = []
-        @threads[:writers] = []
+        @threads = {:readers=>[], :workers=>[], :writers=>[], :reporters=>[]}
+        @thread_counts = {:readers => options.fetch(:readers, 1),
+                          :workers => options.fetch(:workers, 5),
+                          :writers => options.fetch(:writers, 1)}
 
-        @thread_counts = {}
-        @thread_counts[:readers] = options.fetch(:readers, 1)
-        @thread_counts[:workers] = options.fetch(:workers, 5)
-        @thread_counts[:writers] = options.fetch(:writers, 1)
+        @relentless = options.fetch(:relentless, false)
+        @sleep_interval = options.fetch(:sleep_interval, 5)
 
-        @batch_size = options.fetch(:batch_size, 10)
+        # Initialize queues
+        @input_queue  = Opener::Daemons::SQS.find(options.fetch(:input_queue))
+        if options[:output_queue]
+          @output_queue = Opener::Daemons::SQS.find(options[:output_queue])
+        end
 
+        # Initialize Buffers
         @input_buffer  = Queue.new
         @output_buffer = Queue.new
 
+        # Batch and Buffer size for a smooth flow.
+        @batch_size = options.fetch(:batch_size, 10)
+        @buffer_size = options[:buffer_size]
+
+        # Working component
         @klass = klass
 
         script_name = File.basename($0, ".rb")
+
         @logger = Logger.new(options.fetch(:log, STDOUT))
         @logger.level = if options.fetch(:debug, false)
                           Logger::DEBUG
@@ -42,22 +52,19 @@ module Opener
                           Logger::INFO
                         end
 
+        logger.debug(options.to_json)
       end
 
       def buffer_new_messages
-        if input_buffer.size > buffer_size
-          #logger.debug "Maximum input buffer size reached"
-          return
-        end
-
-        if output_buffer.size > buffer_size
-          #logger.debug "Maximum output buffer size reached"
-          return
-        end
+        return if input_buffer.size > buffer_size
+        return if output_buffer.size > buffer_size
 
         messages = input_queue.receive_messages(batch_size)
 
-        return if messages.nil?
+        if messages.nil?
+          sleep(sleep_interval)
+          return
+        end
         messages.each do |message|
           input_buffer << message
         end
@@ -65,21 +72,30 @@ module Opener
 
       def start
         Thread.abort_on_exception = true
-        #
-        # Load Readers
-        #
+
+        start_readers
+        start_workers
+        start_writers
+        start_reporters
+
+        threads[:readers].each(&:join)
+        threads[:workers].each(&:join)
+        threads[:writers].each(&:join)
+        threads[:reporters].each(&:join)
+      end
+
+      def start_readers
         thread_counts[:readers].times do |t|
           threads[:readers] << Thread.new do
-            logger.info "Producer #{t+1} ready for action..."
+            logger.info "Reader #{t+1} ready for action..."
             loop do
               buffer_new_messages
             end
           end
         end
+      end
 
-        #
-        # Load Workers
-        #
+      def start_workers
         thread_counts[:workers].times do |t|
           threads[:workers] << Thread.new do
             logger.info "Worker #{t+1} launching..."
@@ -96,53 +112,67 @@ module Opener
                   raise "The component returned an empty response."
                 end
               rescue Exception => e
-                logger.error(e)
-                output = input
+                if relentless?
+                  raise
+                else
+                  logger.error(e)
+                  output = input
+                end
               end
+
               output_buffer.push({ :output=>output,
-                                 :handle=>message[:receipt_handle]})
+                                   :handle=>message[:receipt_handle]})
             end
           end
         end
+      end
 
-        #
-        # Load Writers
-        #
+      def start_writers
         thread_counts[:writers].times do |t|
           threads[:writers] << Thread.new do
             logger.info "Pusher #{t+1} ready for action..."
             loop do
               message = output_buffer.pop
 
-              payload = {:input=>message[:output]}.to_json
-              output_queue.send_message(payload)
+              payload = {:input=>message[:output].force_encoding("UTF-8")}.to_json
+              output_queue.send_message(payload) if output_queue
               input_queue.delete_message(message[:handle])
             end
           end
         end
+      end
 
-        reporter = Thread.new do
+      def start_reporters
+        threads[:reporters] << Thread.new do
           loop do
-            logger.debug "input buffer: #{input_buffer.size} \t output buffer: #{output_buffer.size}"
+            log = {:buffers=>{:input=>input_buffer.size}}
+            log[:buffers][:output] = output_buffer.size if output_buffer
 
-            thread_types = [:readers, :workers, :writers]
+            logger.debug log.to_json
+            sleep(2)
+          end
+        end
+
+        threads[:reporters] << Thread.new do
+          loop do
+            thread_types = threads.keys - [:reporters]
             thread_counts = thread_types.map do |type|
               threads[type].select{|thread| thread.status}.count
             end
             zip = thread_types.zip(thread_counts)
             logger.debug "active thread counts: #{zip}"
 
-            sleep(2)
+            sleep(10)
           end
         end
-
-        threads[:readers].each(&:join)
-        threads[:workers].each(&:join)
-        threads[:writers].each(&:join)
       end
 
       def buffer_size
-        4 * batch_size
+        @buffer_size ||= (4 * batch_size)
+      end
+
+      def relentless?
+        @relentless
       end
 
     end

data/lib/opener/daemons/opt_parser.rb

@@ -59,10 +59,18 @@ module Opener
             options[:output_queue] = v
           end
 
-          opts.on("-b", "--batch-size BATCH_SIZE", Integer, "Request x messages at once where x is between 1 and 10") do |v|
+          opts.on("--batch-size ITEMS", Integer, "Request x messages at once where x is between 1 and 10") do |v|
             options[:batch_size] = v
           end
 
+          opts.on("--buffer-size ITEMS", Integer, "Size of input and output buffer. Defaults to 4 * batch-size") do |v|
+            options[:buffer_size] = v
+          end
+
+          opts.on("--sleep-interval TIME_IN_SECONDS", Integer, "The interval to sleep when the queue is empty (seconds)") do |v|
+            options[:sleep_interval] = v
+          end
+
           opts.on("-w", "--workers NUMBER", Integer, "number of worker thread") do |v|
             options[:workers] = v
           end
@@ -79,7 +87,7 @@ module Opener
             options[:log] = v
           end
 
-          opts.on("--pid FILENAME", "Filename and path of pidfile. Defaults to /var/run/{basename of current script}.pid") do |v|
+          opts.on("--pidfile FILENAME", "--pid FILENAME", "Filename and path of pidfile. Defaults to /var/run/#{script_name}.pid") do |v|
             options[:pid] = v
           end
 
@@ -91,6 +99,10 @@ module Opener
             options[:debug] = true
           end
 
+          opts.on("--relentless", "Be relentless, fail fast, fail hard, do not continue processing when encountering component errors") do |v|
+            options[:relentless] = true
+          end
+
           opts.separator ""
 
           opts.separator "Common options:"

data/lib/opener/daemons/version.rb

@@ -1,5 +1,5 @@
 module Opener
   module Daemons
-    VERSION = "0.4.2"
+    VERSION = "0.6.0"
   end
 end

metadata

@@ -1,86 +1,87 @@
 --- !ruby/object:Gem::Specification
 name: opener-daemons
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Wilco van Duinkerken
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-07 00:00:00.000000000 Z
+date: 2014-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: spoon
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: spoon
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: bundler
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.5'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rspec
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-description: Daemonize OpeNER components and make them read from an SQS queue. JRuby compatible.
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Daemonize OpeNER components and make them read from an SQS queue. JRuby
+  compatible.
 email:
 - wilco@sparkboxx.com
 executables: []
@@ -103,24 +104,24 @@ homepage: http://opener-project.github.io
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.2.2
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
 specification_version: 4
 summary: Daemonize OpeNER components and make them read from an SQS queue. JRuby compatible.
 test_files: []