opener-daemons 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 983e879dcaa50627ad0893e28d50d70f69fd77c3
-  data.tar.gz: 9c6cf436bc1c7aee5a78d3b62adcd220df334b02
+  metadata.gz: db9deedcb0722c93dce8dbf63951426670ccfc31
+  data.tar.gz: 0c915961e981220c36f5668d89420d12eea6a615
 SHA512:
-  metadata.gz: 77b310b5014d001ab149822d64fba94e1b0553ac95609b6f56ef7e77cba507792db17163c0832aa79cf24faa8117497c758c2853f86ae862c63f79f7158f8eef
-  data.tar.gz: 4cdab83d98edf62ce78cd9a2003d61c6b44bcd060235be307bf4f7023b24e293863ac8c7432b8b60b05a99256b874e686d0b73635a4386964f392d46e93e3d40
+  metadata.gz: bd96be27c2fd2c3a312afdb304b903262bbfee59d3a97a46c492aae903d744928e132afbfd91cca2069d2cf68fd3ea587537269a0268ad520146c17f59e4924c
+  data.tar.gz: e460109a333e013dfbfe54228348f030475914406d8cc7faed4af9e1196cf36f04d8b07b55df2902161c6e6a82a18e957dd94a7fe0caf8e0fe70cebf47422392

data/lib/opener/daemons/daemon.rb

@@ -13,8 +13,10 @@ module Opener
       attr_reader :batch_size, :buffer_size, :sleep_interval,
                   :input_queue, :output_queue,
                   :input_buffer, :output_buffer,
+                  :bucket_name, :bucket_dir, :file_suffix,
                   :klass,
-                  :logger
+                  :logger,
+                  :script_name
 
       attr_accessor :threads, :thread_counts
 
@@ -33,6 +35,12 @@ module Opener
         if options[:output_queue]
           @output_queue = Opener::Daemons::SQS.find(options[:output_queue])
         end
+        
+        # Get bucket name and other bucket options, if any.
+        if @bucket_name = options[:bucket_name]
+           @bucket_dir  = options.fetch(:bucket_dir, nil)
+           @file_suffix = options.fetch(:file_suffix, nil)
+         end
 
         # Initialize Buffers
         @input_buffer  = Queue.new
@@ -45,7 +53,7 @@ module Opener
         # Working component
         @klass = klass
 
-        script_name = File.basename($0, ".rb")
+        @script_name = File.basename($0, ".rb")
 
         @logger = Logger.new(options.fetch(:log, STDOUT))
         @logger.level = if options.fetch(:debug, false)
@@ -103,8 +111,8 @@ module Opener
             identifier = klass.new
             loop do
               message = input_buffer.pop
-
-              input = message[:body]["input"]
+              
+              input = get_input(message[:body])
               input,* = input if input.kind_of?(Array)
 
               begin
@@ -138,7 +146,15 @@ module Opener
               message = output_buffer.pop
               callbacks = extract_callbacks(message[:body]["callbacks[]"])
               handler = Opener::CallbackHandler.new
-              message[:body][:input] = message[:output].force_encoding("UTF-8")
+              
+              if bucket_name
+                filename = [message[:body]["request_id"], script_name, Time.now.to_i].join("-")
+                s3 = Opener::Daemons::S3.new(bucket_name, message[:output].force_encoding("UTF-8"), filename,  bucket_dir, file_suffix)
+                s3.upload
+                message[:body][:input_url] = s3.url
+              else
+                message[:body][:input] = message[:output].force_encoding("UTF-8")
+              end
               
               
               unless callbacks.empty?
@@ -203,7 +219,11 @@ module Opener
 
         return callbacks
       end
-
+      
+      def get_input(body)
+        return body.delete("input") if body["input"]
+        return HTTPClient.new.get(body.delete("input_url")).body if body["input_url"]
+      end
     end
   end
 end

data/lib/opener/daemons/opt_parser.rb

@@ -147,6 +147,18 @@ module Opener
           opts.on("--relentless", "Be relentless, fail fast, fail hard, do not continue processing when encountering component errors") do |v|
             options[:relentless] = true
           end
+          
+          opts.on("--bucket-name BUCKET_NAME", "The name of the bucket where the output should be stored.") do |v|
+            options[:bucket_name] = v
+          end
+          
+          opts.on("--bucket-dir BUCKET_DIR", "The directory that the output should be stored in. Requires --bucket-name option.") do |v|
+            options[:bucket_dir] = v
+          end
+          
+          opts.on("--file-suffix FILE_SUFFIX", "The suffix of the filename. Requires --bucket-name option.") do |v|
+            options[:bucket_name] = v
+          end
 
           opts.separator ""
 

data/lib/opener/daemons/s3.rb

@@ -0,0 +1,34 @@
+require 'aws-sdk'
+
+module Opener
+  module Daemons
+    class S3
+      attr_reader :s3_client, :bucket_name, :content, :filename, :directory, :url
+
+      def initialize(bucket_name, content, filename, directory = nil, filename_suffix = nil)
+        @s3_client   = AWS::S3.new
+        @bucket_name = bucket_name
+        @filename    = [filename, filename_suffix].compact.reject{|e| e.empty?}.join("-")
+        @content     = content
+        @directory   = directory
+      end
+      
+      def upload
+        @filename = File.join(directory, filename) if directory
+        bucket = s3_client.buckets[bucket_name]
+        object = bucket.objects["#{filename}.kaf"]
+
+        object.write(content)
+        
+        @url = object.url_for(
+          :read,
+          :secure                       => false,
+          :force_path_style             => false,
+          :response_content_type        => "application/xml",
+          :response_content_disposition => "attachment",
+          :expires                      => 7 * 24 * 60 * 60 # 7 days
+        )
+      end
+    end
+  end
+end

data/lib/opener/daemons/version.rb

@@ -1,5 +1,5 @@
 module Opener
   module Daemons
-    VERSION = "1.2.0"
+    VERSION = "1.3.0"
   end
 end

data/lib/opener/daemons.rb

@@ -1,5 +1,6 @@
 require "opener/daemons/version"
 require "opener/daemons/sqs"
+require "opener/daemons/s3"
 require "opener/daemons/daemon"
 require "opener/daemons/opt_parser"
 require "opener/daemons/controller"

data/opener-daemons.gemspec

@@ -17,10 +17,11 @@ Gem::Specification.new do |spec|
     'README.md'
   ]).select { |file| File.basename(file) }
 
-  spec.add_dependency 'aws-sdk-core'
+  spec.add_dependency 'aws-sdk'
   spec.add_dependency 'spoon'
   spec.add_dependency 'dotenv'
   spec.add_dependency "opener-callback-handler"
+  spec.add_dependency "httpclient"
 
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: opener-daemons
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Wilco van Duinkerken
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-09-16 00:00:00.000000000 Z
+date: 2014-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-core
+  name: aws-sdk
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -66,6 +66,20 @@ dependencies:
         version: '0'
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: bundler
   version_requirements: !ruby/object:Gem::Requirement
@@ -119,6 +133,7 @@ files:
 - lib/opener/daemons/controller.rb
 - lib/opener/daemons/daemon.rb
 - lib/opener/daemons/opt_parser.rb
+- lib/opener/daemons/s3.rb
 - lib/opener/daemons/sqs.rb
 - lib/opener/daemons/version.rb
 - LICENSE.txt
@@ -149,4 +164,3 @@ signing_key:
 specification_version: 4
 summary: Daemonize OpeNER components and make them read from an SQS queue. JRuby compatible.
 test_files: []
-has_rdoc: