opener-daemons 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6d0b2126a8b6322257975cfa14b323e8adc2b764
+  data.tar.gz: 9d5cd048fed57092de5bca35d930f25cc7ec1ae5
+SHA512:
+  metadata.gz: 2e557daafe0982331f859d5c6b0100b1c8cbfedcd8e2256215fc1a4175ceb4628a70c9edef86a962d19bccf769412ee8ea42c2721464fce3d0486689287dea31
+  data.tar.gz: e7d8736c85e18449ba7cf4192bd7840547c05bc12eedcb13651d9db74b302ca6f38e1a519a5a3b432eb114292df25126aeff28d5ccf422edc24f7f385e51bfcf

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in opener-daemons.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Wilco van Duinkerken
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,127 @@
+# Opener::Daemons
+
+This GEM is part of the OpeNER project, which is the NLP toolchain for the rest
+of us. This particular GEM makes is possible that al OpeNER components can
+actually be launched as deamons reading from and push to Amazon SQS queues.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'opener-daemons'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install opener-daemons
+
+
+## SQS
+
+The Opener-daemon GEM uses Amazon SQS service as a message service. In order for
+this to work properly you need an Amazon AWS account and you need to set the
+following 3 environment variables in your shell:
+
+```
+export AWS_ACCESS_KEY_ID='...'
+export AWS_SECRET_ACCESS_KEY='...'
+export AWS_REGION='...' #e.g. eu-west-1
+```
+
+To see how you specify which queues to use, checkout the usage section below.
+
+## Implementation
+
+This Gem is intended for use with other OpeNER components. In order to turn a
+component in to a Daemon you have to do the following:
+
+Add the opener-daemons gem to the gemspec of your component (or the the Gemfile
+if your component is not a gem).
+
+```
+  gem.add_dependency 'opener-daemons'
+```
+
+Create a file in the bin/ directory of the component. Following the OpeNER
+naming conventions that will be something like this (e.g. the
+language-identifier). This file provides you with the option to launch a daemon
+from the command line.
+
+    touch bin/language-identifier-daemon
+
+Then add the following lines and replace the language-identifier with your own
+component:
+
+```ruby
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'opener/daemons'
+
+exec_path = File.expand_path("exec/language-identifier.rb")
+Opener::Daemons::Controller.new(:name=>"language-identifier",
+                                :exec_path=>exec_path)
+```
+
+After that you have to create a file that does the actual work in an "exec"
+directory. From the root of your component do this:
+
+```
+mkdir exec
+touch exec/language-identifier.rb
+```
+
+Then copy paste the following code into that file, replacing the
+"language-identifier" parts with your own component.
+
+```ruby
+require 'opener/daemons'
+require 'opener/language_identifier'
+
+options = Opener::OptParser.parse!(ARGV)
+daemon = Opener::Daemon.new(Opener::LanguageIdentifier, options)
+daemon.start
+```
+
+Now you should be able to launch yourself a LanguageIdentifier daemon. Check out
+the exact usage of the daemon by typing:
+
+```
+bin/language-identifier-daemon -h
+```
+
+## Usage
+
+Once you implemented the daemon you can use the -h option to get usage
+information. It will look like this, with the "language-identifier" strings
+replaced by your own component.
+
+```
+Usage: language-identifier.rb <start|stop|restart> [options]
+
+Specific options:
+    -i, --input INPUT_QUEUE_NAME     Input queue name
+    -o, --output OUTPUT_QUEUE_NAME   Output queue name
+    -b, --batch-size BATCH_SIZE      Request x messages at once where x is between 1 and 10
+    -w, --workers NUMBER             number of worker thread
+    -r, --readers NUMBER             number of reader threads
+    -p, --writers NUMBER             number of writer / pusher threads
+        --log FILENAME               Filename and path of logfile. Defaults to STDOUT
+        --pid FILENAME               Filename and path of pidfile. Defaults to /var/run/{filename}.rb
+        --pidpath DIRNAME            Directory where to put the PID file. Is Overwritten by --pid if that option is present
+        --debug                      Turn on debug log level
+
+Common options:
+    -h, --help                       Show this message
+```
+
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/opener-daemons/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/opener/daemons/controller.rb

@@ -0,0 +1,162 @@
+#
+# Original Idea by Charles Nutter
+# Copied from: https://gist.github.com/ik5/448884
+# Then adjusted.
+#
+
+require 'rubygems'
+require 'opener/daemons'
+require 'spoon'
+
+module Opener
+  module Daemons
+    class Controller
+      attr_reader :name, :exec
+
+      def initialize(options={})
+        @exec = options.fetch(:exec_path)
+        @name      = determine_name(options[:name])
+        read_commandline
+      end
+
+      def determine_name(name)
+        return identify(name) unless name.nil?
+        get_name_from_exec_path
+      end
+
+      def get_name_from_exec_path
+        File.basename(exec, ".rb")
+      end
+
+      def read_commandline
+        args = ARGV.dup
+        @options = Opener::Daemons::OptParser.parse!(args)
+
+        if ARGV[0] == 'start'
+          start
+        elsif ARGV[0] == 'stop'
+          stop
+        elsif ARGV[0] == 'restart'
+          stop
+          start
+        elsif ARGV[0] == '-h'
+          Opener::Daemons::OptParser.parse!(ARGV)
+        else
+          puts "Usage: #{name} <start|stop|restart> [options]"
+          puts "Or for help use: #{name} -h"
+          exit!
+        end
+      end
+
+      def pid_path
+        return @pid_path unless @pid_path.nil?
+        @pid_path = if @options[:pid]
+                      File.expand_path(@options[:pid])
+                    elsif @options[:pidpath]
+                      File.expand_path(File.join(@options[:pidpath], "#{name}.pid"))
+                    else
+                      "/var/run/#{File.basename($0, ".rb")}.pid"
+                    end
+      end
+
+      def create_pid(pid)
+        begin
+          open(pid_path, 'w') do |f|
+            f.puts pid
+          end
+        rescue => e
+          STDERR.puts "Error: Unable to open #{pid_path} for writing:\n\t" +
+            "(#{e.class}) #{e.message}"
+          exit!
+        end
+      end
+
+      def get_pid
+        pid = false
+        begin
+          open(pid_path, 'r') do |f|
+            pid = f.readline
+            pid = pid.to_s.gsub(/[^0-9]/,'')
+          end
+        rescue => e
+          STDOUT.puts "Info: Unable to open #{pid_path} for reading:\n\t" +
+            "(#{e.class}) #{e.message}"
+        end
+
+
+        if pid
+          return pid.to_i
+        else
+          return nil
+        end
+      end
+
+      def remove_pidfile
+        begin
+          File.unlink(pid_path)
+        rescue => e
+          STDERR.puts "ERROR: Unable to unlink #{path}:\n\t" +
+            "(#{e.class}) #{e.message}"
+          exit
+        end
+      end
+
+      def process_exists?
+        begin
+          pid = get_pid
+          return false unless pid
+          Process.kill(0, pid)
+          true
+        rescue Errno::ESRCH, TypeError # "PID is NOT running or is zombied
+          false
+        rescue Errno::EPERM
+          STDERR.puts "No permission to query #{pid}!";
+          false
+        rescue => e
+          STDERR.puts "Error: Unable to determine status for pid: #{pid}.\n\t" +
+            "(#{e.class}) #{e.message}"
+          false
+        end
+      end
+
+      def stop
+        begin
+          pid = get_pid
+          STDOUT.puts "Stopping pid: #{pid}"
+          while true do
+            Process.kill("TERM", pid)
+            Process.wait(pid)
+            sleep(0.1)
+          end
+        rescue Errno::ESRCH # no more process to kill
+          remove_pidfile
+          STDOUT.puts 'Stopped the process'
+        rescue => e
+          STDERR.puts "Unable to terminate process: (#{e.class}) #{e.message}"
+        end
+      end
+
+      def start
+        if process_exists?
+          STDERR.puts "Error: The process #{name} already running. Restarting the process"
+          stop
+        end
+
+        STDOUT.puts "Starting DAEMON"
+        pid = Spoon.spawnp exec, *ARGV
+        STDOUT.puts "Started DAEMON"
+        create_pid(pid)
+        begin
+          Process.setsid
+        rescue Errno::EPERM => e
+          STDERR.puts "Process.setsid not permitted on this platform, not critical. Continuing normal operations.\n\t (#{e.class}) #{e.message}"
+        end
+        File::umask(0)
+      end
+
+      def identify(string)
+        string.gsub(/\W/,"-").gsub("--","-")
+      end
+    end
+  end
+end

data/lib/opener/daemons/daemon.rb

@@ -0,0 +1,135 @@
+require 'thread'
+require 'opener/daemons/sqs'
+
+module Opener
+  module Daemons
+    class Daemon
+      attr_reader :batch_size,
+                  :input_queue, :output_queue,
+                  :input_buffer, :output_buffer,
+                  :klass,
+                  :logger
+
+      attr_accessor :threads, :thread_counts
+
+      def initialize(klass, options={})
+        @input_queue  = Opener::Daemons::SQS.find(options.fetch(:input_queue))
+        @output_queue = Opener::Daemons::SQS.find(options.fetch(:output_queue))
+
+        @threads = {}
+        @threads[:writers] = []
+        @threads[:readers] = []
+        @threads[:workers] = []
+
+        @thread_counts = {}
+        @thread_counts[:workers] = options.fetch(:workers, 1)
+        @thread_counts[:readers] = options.fetch(:readers, 5)
+        @thread_counts[:writers] = options.fetch(:writers, 1)
+
+        @batch_size = options.fetch(:batch_size, 10)
+
+        @input_buffer  = Queue.new
+        @output_buffer = Queue.new
+
+        @klass = klass
+
+        script_name = File.basename($0, ".rb")
+        @logger = Logger.new(options.fetch(:log, "#{script_name}.log"))
+        @logger.level = if options.fetch(:debug, false)
+                          Logger::DEBUG
+                        else
+                          Logger::INFO
+                        end
+      end
+
+      def buffer_new_messages
+        if input_buffer.size > buffer_size
+          #logger.debug "Maximum input buffer size reached"
+          return
+        end
+
+        if output_buffer.size > buffer_size
+          #logger.debug "Maximum output buffer size reached"
+          return
+        end
+
+        messages = input_queue.receive_messages(batch_size)
+
+        return if messages.nil?
+        messages.each do |message|
+          input_buffer << message
+        end
+      end
+
+      def start
+        Thread.abort_on_exception = true
+        #
+        # Load Readers
+        #
+        thread_counts[:readers].times do |t|
+          threads[:readers] << Thread.new do
+            logger.info "Producer #{t+1} ready for action..."
+            loop do
+              buffer_new_messages
+            end
+          end
+        end
+
+        #
+        # Load Workers
+        #
+        thread_counts[:workers].times do |t|
+          threads[:workers] << Thread.new do
+            logger.info "Worker #{t+1} launching..."
+            identifier = klass.new
+            loop do
+              message = input_buffer.pop
+
+              input = JSON.parse(message[:body])["input"]
+              begin
+                output = identifier.run(input)
+              rescue Exception => e
+                logger.error(e)
+                output = input
+              end
+              output_buffer.push({ :output=>output,
+                                 :handle=>message[:receipt_handle]})
+            end
+          end
+        end
+
+        #
+        # Load Writers
+        #
+        thread_counts[:writers].times do |t|
+          threads[:writers] << Thread.new do
+            logger.info "Pusher #{t+1} ready for action..."
+            loop do
+              message = output_buffer.pop
+
+              payload = {:input=>message[:output]}.to_json
+              output_queue.send_message(payload)
+              input_queue.delete_message(message[:handle])
+            end
+          end
+        end
+
+        reporter = Thread.new do
+          loop do
+            logger.debug "input buffer: #{input_buffer.size} \t output buffer: #{output_buffer.size}"
+            sleep(2)
+          end
+        end
+
+        threads[:writers].each(&:join)
+        threads[:readers].each(&:join)
+        threads[:workers].each(&:join)
+      end
+
+      def buffer_size
+        4 * batch_size
+      end
+
+    end
+  end
+end

data/lib/opener/daemons/opt_parser.rb

@@ -0,0 +1,79 @@
+require 'optparse'
+
+module Opener
+  module Daemons
+    class OptParser
+
+      def self.parse(args)
+        process(args, :parse)
+      end
+
+      def self.parse!(args)
+        process(args, :parse!)
+      end
+
+      def self.process(args, call)
+        options = {}
+
+        OptionParser.new do |opts|
+          opts.banner = "Usage: language-identifier.rb <start|stop|restart> [options]"
+          opts.separator ""
+          opts.separator "Specific options:"
+
+          opts.on("-i", "--input INPUT_QUEUE_NAME", "Input queue name") do |v|
+            options[:input_queue] = v
+          end
+
+          opts.on("-o", "--output OUTPUT_QUEUE_NAME", "Output queue name") do |v|
+            options[:output_queue] = v
+          end
+
+          opts.on("-b", "--batch-size BATCH_SIZE", Integer, "Request x messages at once where x is between 1 and 10") do |v|
+            options[:batch_size] = v
+          end
+
+          opts.on("-w", "--workers NUMBER", Integer, "number of worker thread") do |v|
+            options[:workers] = v
+          end
+
+          opts.on("-r", "--readers NUMBER", Integer, "number of reader threads") do |v|
+            options[:readers] = v
+          end
+
+          opts.on("-p", "--writers NUMBER", Integer, "number of writer / pusher threads") do |v|
+            options[:writers] = v
+          end
+
+          opts.on("--log FILENAME", "Filename and path of logfile. Defaults to STDOUT") do |v|
+            options[:log] = v
+          end
+
+          opts.on("--pid FILENAME", "Filename and path of pidfile. Defaults to /var/run/{basename of current script}.pid") do |v|
+            options[:pid] = v
+          end
+
+          opts.on("--pidpath DIRNAME", "Directory where to put the PID file. Is Overwritten by --pid if that option is present") do |v|
+            options[:pidpath] = v
+          end
+
+          opts.on("--debug", "Turn on debug log level") do |v|
+            options[:debug] = true
+          end
+
+          opts.separator ""
+
+          opts.separator "Common options:"
+
+          # No argument, shows at tail.  This will print an options summary.
+          # Try it and see!
+          opts.on_tail("-h", "--help", "Show this message") do
+            puts opts
+            exit
+          end
+        end.send(call, args)
+
+        return options
+      end
+    end
+  end
+end

data/lib/opener/daemons/sqs.rb

@@ -0,0 +1,33 @@
+require 'aws-sdk-core'
+
+module Opener
+  module Daemons
+    class SQS
+      attr_reader :sqs, :name, :url
+
+      def self.find(name)
+        new(name)
+      end
+
+      def initialize(name)
+        @sqs = Aws::SQS.new
+        @name = name
+        @url = sqs.get_queue_url(:queue_name=>name)[:queue_url]
+      end
+
+      def send_message(message)
+        sqs.send_message(:queue_url=>url, :message_body=>message)
+      end
+
+      def delete_message(handle)
+        sqs.delete_message(:queue_url=>url, :receipt_handle=>handle)
+      end
+
+      def receive_messages(limit)
+        result = sqs.receive_message(:queue_url=>url,
+                                     :max_number_of_messages=>limit)[:messages]
+      end
+
+    end
+  end
+end

data/lib/opener/daemons/version.rb

@@ -0,0 +1,5 @@
+module Opener
+  module Daemons
+    VERSION = "0.1.0"
+  end
+end

data/lib/opener/daemons.rb

@@ -0,0 +1,10 @@
+require "opener/daemons/version"
+require "opener/daemons/sqs"
+require "opener/daemons/daemon"
+require "opener/daemons/opt_parser"
+require "opener/daemons/controller"
+
+module Opener
+  module Daemons
+  end
+end

data/opener-daemons.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'opener/daemons/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "opener-daemons"
+  spec.version       = Opener::Daemons::VERSION
+  spec.authors       = ["Wilco van Duinkerken"]
+  spec.email         = ["wilco@sparkboxx.com"]
+  spec.summary       = %q{Daemonize OpeNER components and make them read from an SQS queue. JRuby compatible.}
+  spec.description   = spec.summary
+  spec.homepage      = "http://opener-project.github.io"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+
+  spec.add_dependency 'aws-sdk-core'
+  spec.add_dependency 'spoon'
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: opener-daemons
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Wilco van Duinkerken
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-05-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: spoon
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: Daemonize OpeNER components and make them read from an SQS queue. JRuby compatible.
+email:
+- wilco@sparkboxx.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/opener/daemons.rb
+- lib/opener/daemons/controller.rb
+- lib/opener/daemons/daemon.rb
+- lib/opener/daemons/opt_parser.rb
+- lib/opener/daemons/sqs.rb
+- lib/opener/daemons/version.rb
+- opener-daemons.gemspec
+homepage: http://opener-project.github.io
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.2.2
+signing_key:
+specification_version: 4
+summary: Daemonize OpeNER components and make them read from an SQS queue. JRuby compatible.
+test_files: []