bulk-processor 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 163fb73da29263963c492e14b85b35290e2f8d64
-  data.tar.gz: 352d9b5659f885f238a922361d6f341496ff36f1
+  metadata.gz: 90877b508066f5c70c78f247b6b29a3110c87c3c
+  data.tar.gz: ab09632f364999e0138f781b86341b5f87333bd3
 SHA512:
-  metadata.gz: f908072ce3303676c8c8440dff4f8d47646571be81ea71bc668e523f6bce7e94d5e4553c3f2a31a1d9eeba797a630b47a6c010b9bf97d495f575cdf844a1f4df
-  data.tar.gz: af56ed66e8a44edcabd9962f16ba0cfc89e2d784285506ecc34ec792f636592009e302a970403eb2dca45f21e77bc944718d18108cfd0018905d1f81cc243aad
+  metadata.gz: 101c55e1a96ced37167a87dd8b4f2d8933e4b19461e43331d77534c53d52ce199a438155701395f1d0b628b8857057d36bcf4f3b5966b522e9283361670d2c7c
+  data.tar.gz: bfd29f4c4a09c8ee2fcbeeb73e00891557eabcd0cc1f0ac8c3f9e08b8bf5745a58b6133d223cc843c4995ae86b34f5d9ac0edba9dcea999924fa809184ad6d79

data/README.md

@@ -28,18 +28,39 @@ Or install it yourself as:
 
 Bulk processor requires the following configuration
 
+#### Back end: ActiveJob
+
 ```ruby
+BulkProcessor.back_end = :active_job
 BulkProcessor.queue_adapter = <adapter>
-BulkProcessor.temp_directory = '/tmp'
-BulkProcessor.aws.access_key_id = 'my-aws-access-key'
-BulkProcessor.aws.secret_access_key = 'my-aws-secret'
-BulkProcessor.aws.bucket = 'my-s3-bucket'
 ```
 
 The default queue_adapter is `:inline`, which skips queueing and processes synchronously. Since
 this is backed by ActiveJob, all of the adapters in [ActiveJob::QueueAdapters]( http://api.rubyonrails.org/classes/ActiveJob/QueueAdapters.html ) are supported,
 including `:resque`.
 
+#### Back end: Dynosaur
+
+```ruby
+BulkProcessor.back_end = :dynosaur
+BulkProcessor.heroku.api_key = 'my-heroku-api-key'
+BulkProcessor.heroku.app_name = 'my-heroku-app-name'
+```
+
+```ruby
+# Rakefile
+require 'bulk_processor/tasks'
+```
+
+#### AWS S3
+
+```ruby
+BulkProcessor.temp_directory = '/tmp'
+BulkProcessor.aws.access_key_id = 'my-aws-access-key'
+BulkProcessor.aws.secret_access_key = 'my-aws-secret'
+BulkProcessor.aws.bucket = 'my-s3-bucket'
+```
+
 The CSV file passed to BulkProcessor will be persisted on AWS S3 so that the job
 can access it. This requires configuring AWS credentials, the S3 bucket in which
 to store the file, and a local temp directory to hold the file locally.

data/bulk-processor.gemspec

@@ -23,6 +23,7 @@ success or failure report
 
   spec.add_runtime_dependency 'activejob', '~> 4'
   spec.add_runtime_dependency 'aws-sdk', '~> 2.1'
+  spec.add_runtime_dependency 'dynosaur', '~> 0.2.1'
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'pry-byebug', '~> 3'

data/lib/bulk_processor.rb

@@ -1,3 +1,6 @@
+require 'bulk_processor/back_end'
+require 'bulk_processor/back_end/active_job'
+require 'bulk_processor/back_end/dynosaur'
 require 'bulk_processor/config'
 require 'bulk_processor/job'
 require 'bulk_processor/s3_file'
@@ -43,7 +46,7 @@ class BulkProcessor
     )
 
     if csv.valid?
-      perform_later(file_class, encoded_contents)
+      start_backend(file_class, encoded_contents)
     else
       errors.concat(csv.errors)
     end
@@ -54,10 +57,11 @@ class BulkProcessor
 
   attr_reader :key, :stream, :processor_class, :payload
 
-  def perform_later(file_class, contents)
+  def start_backend(file_class, contents)
     file = file_class.new(key)
     file.write(contents)
-    Job.perform_later(processor_class.name, payload, file_class.name, key)
+    BackEnd.start(processor_class: processor_class, payload: payload,
+                  file_class: file_class, key: key)
   rescue Exception
     # Clean up the file, which is treated as a lock, if we bail out of here
     # unexpectedly.

data/lib/bulk_processor/back_end.rb

@@ -0,0 +1,23 @@
+class BulkProcessor
+  module BackEnd
+    class << self
+      def start(processor_class:, payload:, file_class:, key:)
+        back_end = back_end_class.new(
+          processor_class: processor_class,
+          payload: payload,
+          file_class: file_class,
+          key: key
+        )
+        back_end.start
+      end
+
+      private
+
+      def back_end_class
+        back_end = BulkProcessor.config.back_end
+        classified = back_end.to_s.split('_').collect(&:capitalize).join
+        BulkProcessor::BackEnd.const_get(classified)
+      end
+    end
+  end
+end

data/lib/bulk_processor/back_end/active_job.rb

@@ -0,0 +1,20 @@
+class BulkProcessor
+  module BackEnd
+    class ActiveJob
+      def initialize(processor_class:, payload:, file_class:, key:)
+        @processor_class = processor_class
+        @payload = payload
+        @file_class = file_class
+        @key = key
+      end
+
+      def start
+        Job.perform_later(processor_class.name, payload.to_json, file_class.name, key)
+      end
+
+      private
+
+      attr_reader :processor_class, :payload, :file_class, :key
+    end
+  end
+end

data/lib/bulk_processor/back_end/dynosaur.rb

@@ -0,0 +1,34 @@
+require 'dynosaur'
+
+class BulkProcessor
+  module BackEnd
+    class Dynosaur
+      def initialize(processor_class:, payload:, file_class:, key:)
+        @processor_class = processor_class
+        @payload = payload
+        @file_class = file_class
+        @key = key
+        configure_dynosaur
+      end
+
+      def start
+        args = {
+          task: 'bulk_processor:start',
+          args: [processor_class.name, payload.to_json, file_class.name, key]
+        }
+        ::Dynosaur::Process::Heroku.new(args).start
+      end
+
+      private
+
+      attr_reader :processor_class, :payload, :file_class, :key
+
+      def configure_dynosaur
+        ::Dynosaur::Client::HerokuClient.configure do |config|
+          config.api_key = BulkProcessor.config.heroku.api_key
+          config.app_name = BulkProcessor.config.heroku.app_name
+        end
+      end
+    end
+  end
+end

data/lib/bulk_processor/config.rb

@@ -2,7 +2,7 @@ class BulkProcessor
   # Store configuration data set by clients
   class Config
     attr_reader :queue_adapter
-    attr_accessor :temp_directory
+    attr_accessor :back_end, :temp_directory
 
     def queue_adapter=(adapter)
       ActiveJob::Base.queue_adapter = @queue_adapter = adapter
@@ -11,5 +11,9 @@ class BulkProcessor
     def aws
       @aws ||= Struct.new(:access_key_id, :secret_access_key, :bucket).new
     end
+
+    def heroku
+      @heroku ||= Struct.new(:api_key, :app_name).new
+    end
   end
 end

data/lib/bulk_processor/job.rb

@@ -7,6 +7,7 @@ class BulkProcessor
 
     def perform(processor_class, payload, file_class, key)
       file = file_class.constantize.new(key)
+      payload = payload.nil? ? nil : JSON.parse(payload)
       file.open do |f|
         csv = CSV.parse(f.read, headers: true)
         processor = processor_class.constantize.new(csv, payload: payload)

data/lib/bulk_processor/tasks.rb

@@ -0,0 +1,23 @@
+require 'rake'
+
+class BulkProcessor
+  class Tasks
+    include Rake::DSL
+
+    def install_tasks
+      namespace :bulk_processor do
+        desc 'Start processing a CSV file'
+        task :start, [:processor_class, :payload, :file_class, :key] => :environment do |_task, args|
+          Job.new.perform(
+            args[:processor_class],
+            args[:payload],
+            args[:file_class],
+            args[:key]
+          )
+        end
+      end
+    end
+  end
+end
+
+BulkProcessor::Tasks.new.install_tasks

data/lib/bulk_processor/version.rb

@@ -1,3 +1,3 @@
 class BulkProcessor
-  VERSION = '0.4.0'.freeze
+  VERSION = '0.5.0'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bulk-processor
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Tom Collier, Justin Richard
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: dynosaur
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -115,6 +129,9 @@ files:
 - bin/setup
 - bulk-processor.gemspec
 - lib/bulk_processor.rb
+- lib/bulk_processor/back_end.rb
+- lib/bulk_processor/back_end/active_job.rb
+- lib/bulk_processor/back_end/dynosaur.rb
 - lib/bulk_processor/config.rb
 - lib/bulk_processor/csv_processor.rb
 - lib/bulk_processor/csv_processor/no_op_handler.rb
@@ -124,6 +141,7 @@ files:
 - lib/bulk_processor/job.rb
 - lib/bulk_processor/s3_file.rb
 - lib/bulk_processor/stream_encoder.rb
+- lib/bulk_processor/tasks.rb
 - lib/bulk_processor/validated_csv.rb
 - lib/bulk_processor/version.rb
 homepage: