opener-daemons 1.3.0 → 2.0

data/lib/opener/daemons/pidfile.rb

@@ -0,0 +1,85 @@
+module Opener
+  module Daemons
+    ##
+    # Class for writing and retrieving PID files as well as managing the
+    # associated process.
+    #
+    # @!attribute [r] path
+    #  The path to store the PID in.
+    #  @return [String]
+    #
+    class Pidfile
+      attr_reader :path
+
+      ##
+      # @param [String] path
+      #
+      def initialize(path)
+        @path = path
+      end
+
+      ##
+      # Writes the given process ID to `@path`.
+      #
+      # @param [Fixnum] id
+      #
+      def write(id)
+        File.open(path, 'w') do |handle|
+          handle.write(id.to_s)
+        end
+
+      # Kill the process immediately if we couldn't write the PID
+      rescue Errno::ENOENT, Errno::EPERM => error
+        Process.kill('KILL', id)
+      end
+
+      ##
+      # Reads and returns the process ID.
+      #
+      # @return [Fixnum]
+      #
+      def read
+        return File.read(path).to_i
+      end
+
+      ##
+      # Removes the associated file, if it exists.
+      #
+      def unlink
+        File.unlink(path) if File.file?(path)
+      end
+
+      ##
+      # Terminates the process by sending it the TERM signal and waits for it to
+      # shut down.
+      #
+      def terminate
+        id = read
+
+        begin
+          Process.kill('TERM', id)
+          Process.wait(id)
+        rescue Errno::ESRCH, Errno::ECHILD
+          # Process terminated, yay. Any other error is re-raised.
+        end
+      end
+
+      ##
+      # Returns `true` if the associated process is alive.
+      #
+      # @return [TrueClass|FalseClass]
+      #
+      def alive?
+        id = read
+
+        begin
+          Process.kill(0, id)
+
+          return true
+        rescue Errno::ESRCH, Errno::EPERM
+          return false
+        end
+      end
+    end # Pidfile
+  end # Daemons
+end # Opener

data/lib/opener/daemons/syslog.rb

@@ -0,0 +1,58 @@
+module Opener
+  module Daemons
+    ##
+    # Wrapper around `Syslog` that makes it easier to disable loggers and to log
+    # custom key/value pairs in message.
+    #
+    module Syslog
+      ##
+      # Configures Syslog.
+      #
+      # @see [Syslog.open]
+      # @param [String] name The name of the program.
+      # @param [Fixnum] facility
+      #
+      def self.open(name, facility = nil)
+        ::Syslog.open(name, facility) if Daemons.syslog?
+      end
+
+      # methods defined using define_method() are slower than those defined
+      # using eval() on JRuby and Rubinius.
+      [:debug, :err, :fatal, :info, :warn, :unknown].each do |method|
+        eval <<-EOF, nil, __FILE__, __LINE__ + 1
+        def self.#{method}(message, meta = {})
+          add(:#{method}, message, meta)
+        end
+        EOF
+      end
+
+      ##
+      # Adds a new log message.
+      #
+      # @example
+      #  add(:info, 'Testing Syslog', :user_id => 19)
+      #
+      # @param [Symbol] type The type of log message to add, corresponds to the
+      #  logger method that will be called.
+      #
+      # @param [String] message The message to log.
+      # @param [Hash] meta Extra meta data to log.
+      #
+      def self.add(type, message, meta = {})
+        return unless Daemons.syslog?
+
+        pairs = make_pairs(meta)
+
+        ::Syslog.send(type, "#{pairs} #{message}".strip)
+      end
+
+      ##
+      # @param [Hash] meta
+      # @return [String]
+      #
+      def self.make_pairs(meta)
+        return meta.map { |(key, value)| "#{key}=#{value.inspect}" }.join(' ')
+      end
+    end # Syslog
+  end # Daemons
+end # Opener

data/lib/opener/daemons/uploader.rb

@@ -0,0 +1,50 @@
+module Opener
+  module Daemons
+    ##
+    # Class for uploading KAF documents to Amazon S3.
+    #
+    class Uploader
+      ##
+      # Uploads the given KAF document.
+      #
+      # @param [String] identifier
+      # @param [String] document
+      # @param [Hash] metadata description
+      #
+      # @return [AWS::S3::S3Object]
+      #
+      def upload(identifier, document, metadata = {})
+        object = create(
+          "#{identifier}.xml",
+          document,
+          :metadata     => metadata,
+          :content_type => 'application/xml'
+        )
+
+        return object
+      end
+
+      ##
+      # @param [Array] args
+      # @return [AWS::S3::S3Object]
+      #
+      def create(*args)
+        return bucket.objects.create(*args)
+      end
+
+      ##
+      # @return [AWS::S3.new]
+      #
+      def s3
+        return @s3 ||= AWS::S3.new
+      end
+
+      ##
+      # @return [AWS::S3::Bucket]
+      #
+      def bucket
+        return @bucket ||= s3.buckets[Daemons.output_bucket]
+      end
+    end # Uploader
+  end # Daemons
+end # Opener

data/lib/opener/daemons/version.rb

@@ -1,5 +1,5 @@
 module Opener
   module Daemons
-    VERSION = "1.3.0"
-  end
-end
+    VERSION = '2.0'
+  end # Daemons
+end # Opener

data/lib/opener/daemons/worker.rb

@@ -0,0 +1,104 @@
+module Opener
+  module Daemons
+    ##
+    # Downlods a KAF document, passes it to a component and submits the output
+    # to a callback URL or a default queue. Each Worker instance runs in an
+    # isolated thread
+    #
+    # @!attribute [r] config
+    #  @return [Opener::Daemons::Configuration]
+    #
+    # @!attribute [r] uploader
+    #  @return [Opener::Daemons::Uploader]
+    #
+    # @!attribute [r] downloader
+    #  @return [Opener::Daemons::Downloader]
+    #
+    # @!attribute [r] callback_handler
+    #  @return [Opener::CallbackHandler]
+    #
+    class Worker < Oni::Worker
+      attr_reader :config, :uploader, :downloader, :callback_handler
+
+      include NewRelic::Agent::Instrumentation::ControllerInstrumentation
+      include NewRelic::Agent::MethodTracer
+
+      ##
+      # @param [Opener::Daemons::Configuration] config
+      #
+      def initialize(config)
+        @config           = config
+        @downloader       = Downloader.new
+        @uploader         = Uploader.new
+        @callback_handler = CallbackHandler.new
+      end
+
+      ##
+      # Processes a document.
+      #
+      # @raise [Oni::WrappedError]
+      #
+      def process
+        add_newrelic_attributes
+
+        input    = downloader.download(config.input_url)
+        instance = config.component.new
+        output   = instance.run(input)
+        object   = uploader.upload(config.identifier, output, config.metadata)
+
+        Syslog.info(
+          "Wrote output to s3://#{Daemons.output_bucket}/#{object.key}",
+          config.metadata
+        )
+
+        submit_callbacks(object)
+
+      rescue Exception => error
+        raise Oni::WrappedError.from(
+          error,
+          :input_url => config.input_url,
+          :callbacks => config.callbacks,
+          :metadata  => config.metadata
+        )
+      end
+
+      ##
+      # Sends the object's URL to the next callback URL.
+      #
+      # @param [AWS::S3::S3Object] object
+      #
+      def submit_callbacks(object)
+        urls      = config.callbacks.dup
+        next_url  = urls.shift
+        input_url = object.url_for(:read, :expires => 3600)
+
+        callback_handler.post(
+          next_url,
+          :input_url  => input_url,
+          :identifier => config.identifier,
+          :callbacks  => urls,
+          :metadata   => config.metadata
+        )
+
+        Syslog.info("Submitted response to #{next_url}", config.metadata)
+      end
+
+      private
+
+      def add_newrelic_attributes
+        if Daemons.newrelic?
+          NewRelic::Agent.add_custom_attributes(
+            :input_url => config.input_url,
+            :callbacks => config.callbacks.join(', '),
+            :metadata  => config.metadata
+          )
+        end
+      end
+
+      if Daemons.newrelic?
+        add_transaction_tracer(:process, :category => :task)
+        add_method_tracer(:download_input)
+      end
+    end # Worker
+  end # Daemons
+end # Opener

data/lib/opener/daemons.rb

@@ -1,16 +1,31 @@
-require "opener/daemons/version"
-require "opener/daemons/sqs"
-require "opener/daemons/s3"
-require "opener/daemons/daemon"
-require "opener/daemons/opt_parser"
-require "opener/daemons/controller"
+# https://github.com/nahi/httpclient/issues/230
+gem 'httpclient', '= 2.5.2'
+
+require 'timeout'
+require 'syslog'
+require 'logger'
+require 'securerandom'
+
+require 'aws-sdk'
+require 'httpclient'
+require 'slop'
+require 'oni'
+require 'oni/daemons/sqs'
 require 'opener/callback_handler'
+require 'new_relic/control'
+require 'rollbar'
+require 'json-schema'
 
-require "dotenv"
-env_file = File.expand_path("~/.opener-daemons-env")
-Dotenv.load(env_file)
+require_relative 'daemons/version'
+require_relative 'daemons/daemons'
+require_relative 'daemons/option_parser'
+require_relative 'daemons/controller'
+require_relative 'daemons/pidfile'
+require_relative 'daemons/configuration'
+require_relative 'daemons/downloader'
+require_relative 'daemons/uploader'
+require_relative 'daemons/syslog'
 
-module Opener
-  module Daemons
-  end
-end
+require_relative 'daemons/mapper'
+require_relative 'daemons/worker'
+require_relative 'daemons/daemon'

data/opener-daemons.gemspec

@@ -1,29 +1,36 @@
 require File.expand_path('../lib/opener/daemons/version', __FILE__)
 
 Gem::Specification.new do |spec|
-  spec.name        = "opener-daemons"
+  spec.name        = 'opener-daemons'
   spec.version     = Opener::Daemons::VERSION
-  spec.authors     = ["Wilco van Duinkerken"]
-  spec.email       = ["wilco@sparkboxx.com"]
-  spec.summary     = %q{Daemonize OpeNER components and make them read from an SQS queue. JRuby compatible.}
+  spec.authors     = ['Wilco van Duinkerken', 'Olery']
+  spec.email       = ['wilco@sparkboxx.com', 'development@olery.com']
+  spec.summary     = 'Toolkit for turning OpeNER components into daemons'
   spec.description = spec.summary
-  spec.homepage    = "http://opener-project.github.io"
-  spec.license     = "Apache 2.0"
+  spec.homepage    = 'http://opener-project.github.io'
+  spec.license     = 'Apache 2.0'
+
+  spec.required_ruby_version = '>= 1.9.2'
 
   spec.files = Dir.glob([
     'lib/**/*',
     'LICENSE.txt',
     '*.gemspec',
     'README.md'
-  ]).select { |file| File.basename(file) }
+  ]).select { |file| File.file?(file) }
+
+  spec.add_dependency 'aws-sdk', '~> 1.0'
+  spec.add_dependency 'slop', '~> 3.0'
+  spec.add_dependency 'opener-callback-handler'
+  spec.add_dependency 'newrelic_rpm'
+  spec.add_dependency 'json-schema'
+  spec.add_dependency 'rollbar', '~> 1.0'
+  spec.add_dependency 'oni', '~> 3.0'
 
-  spec.add_dependency 'aws-sdk'
-  spec.add_dependency 'spoon'
-  spec.add_dependency 'dotenv'
-  spec.add_dependency "opener-callback-handler"
-  spec.add_dependency "httpclient"
+  # https://github.com/nahi/httpclient/issues/230
+  spec.add_dependency 'httpclient', '= 2.5.2'
 
-  spec.add_development_dependency "bundler", "~> 1.5"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
 end

metadata

@@ -1,166 +1,217 @@
 --- !ruby/object:Gem::Specification
 name: opener-daemons
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: '2.0'
 platform: ruby
 authors:
 - Wilco van Duinkerken
-autorequire:
+- Olery
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-22 00:00:00.000000000 Z
+date: 2014-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
+        version: '3.0'
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: spoon
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: opener-callback-handler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: dotenv
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: newrelic_rpm
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: opener-callback-handler
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: json-schema
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: httpclient
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rollbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
+        version: '1.0'
   type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: oni
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: httpclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: 2.5.2
+  type: :runtime
   prerelease: false
-  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.5.2
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rspec
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-description: Daemonize OpeNER components and make them read from an SQS queue. JRuby compatible.
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Toolkit for turning OpeNER components into daemons
 email:
 - wilco@sparkboxx.com
+- development@olery.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE.txt
+- README.md
 - lib/opener/daemons.rb
+- lib/opener/daemons/configuration.rb
 - lib/opener/daemons/controller.rb
 - lib/opener/daemons/daemon.rb
-- lib/opener/daemons/opt_parser.rb
-- lib/opener/daemons/s3.rb
-- lib/opener/daemons/sqs.rb
+- lib/opener/daemons/daemons.rb
+- lib/opener/daemons/downloader.rb
+- lib/opener/daemons/mapper.rb
+- lib/opener/daemons/option_parser.rb
+- lib/opener/daemons/pidfile.rb
+- lib/opener/daemons/syslog.rb
+- lib/opener/daemons/uploader.rb
 - lib/opener/daemons/version.rb
-- LICENSE.txt
+- lib/opener/daemons/worker.rb
 - opener-daemons.gemspec
-- README.md
 homepage: http://opener-project.github.io
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.1.9
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
 specification_version: 4
-summary: Daemonize OpeNER components and make them read from an SQS queue. JRuby compatible.
+summary: Toolkit for turning OpeNER components into daemons
 test_files: []
+has_rdoc: