bulk-processor 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77320b807b3cd9862490408058611d9b461cf83f
-  data.tar.gz: 203e565ab7f722c6f639527b4065e2e5f495aa57
+  metadata.gz: f5d0cde0e86097416d0234ead33d2836f4e08c7a
+  data.tar.gz: d176ea1309c0e618ccf718fd8c4320a2b3948831
 SHA512:
-  metadata.gz: 4f89dd796184485f44d0018a9819cf42ec5c147ff20693c63e7055b43fd2e0807e5b268f622e8555157e1ac36b01207c9b23fe6f0212eb2a744924b6e3533d91
-  data.tar.gz: 2373584252697f040d460070a93958cc944c2f0045948233125f4e0ee06d39daf5080191dd0f07e6ad0432e461c7947e01e821a5e81e7dbc994c558571a1da44
+  metadata.gz: 5dfea7cabd9b210116fd21644d34740793f785df799b954c79bdc452f0ec11155575465f0d9ac6bd6b4c1fc2df2a8ca96902c142c99ff15c6a5182761e970f55
+  data.tar.gz: 9b8dc6e9fcd908b6bf0ddc2e6e9c4a36c89ba398ccb11016ed2668c188339b953a4e8c4c58bf8c8025b7ae1b3903170871617fd8e6a808a6d68164ede85af3aa

data/README.md

@@ -30,6 +30,15 @@ Bulk processor requires the following configuration
 
 #### Back end: ActiveJob
 
+Include the `activejob` and back-end queueing gems in your Gemfile, e.g.
+
+```ruby
+# Gemfile
+gem 'activejob'
+gem 'bulk-processor'
+gem 'resque'
+```
+
 ```ruby
 BulkProcessor.back_end = :active_job
 BulkProcessor.queue_adapter = <adapter>
@@ -41,6 +50,14 @@ including `:resque`.
 
 #### Back end: Dynosaur
 
+Include the `dynosaur` gem in your Gemfile, e.g.
+
+```ruby
+# Gemfile
+gem 'dynosaur'
+gem 'resque'
+```
+
 ```ruby
 BulkProcessor.back_end = :dynosaur
 BulkProcessor.heroku.api_key = 'my-heroku-api-key'
@@ -49,7 +66,12 @@ BulkProcessor.heroku.app_name = 'my-heroku-app-name'
 
 ```ruby
 # Rakefile
-require 'bulk_processor/tasks'
+require 'bulk_processor/back_end/dynosaur/tasks'
+
+# If you do not already have an :enivronment rake task, create a no-op one as
+# Dynosaur tasks depend on it.
+task :environment
+end
 ```
 
 #### AWS S3

data/bulk-processor.gemspec

@@ -21,12 +21,12 @@ success or failure report
   spec.require_paths = ['lib']
   spec.required_ruby_version = '>= 2.1'
 
-  spec.add_runtime_dependency 'activejob', '~> 4'
   spec.add_runtime_dependency 'aws-sdk', '~> 2.1'
-  spec.add_runtime_dependency 'dynosaur', '~> 0.2.1'
   spec.add_runtime_dependency 'rack', '~> 1.5'
 
+  spec.add_development_dependency 'activejob', '~> 4'
   spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'dynosaur', '~> 0.2.1'
   spec.add_development_dependency 'pry-byebug', '~> 3'
   spec.add_development_dependency 'rake', '~> 10.4'
   spec.add_development_dependency 'rspec', '~> 3.3'

data/lib/bulk_processor/back_end/active_job/process_csv_job.rb

@@ -0,0 +1,18 @@
+class BulkProcessor
+  module BackEnd
+    class ActiveJob
+      # ActiveJob to handle processing the CSV in the background
+      class ProcessCSVJob < ::ActiveJob::Base
+        queue_as 'bulk_processor'
+
+        def perform(processor_class, payload, key)
+          BulkProcessor::ProcessCSV.new(
+            processor_class.constantize,
+            PayloadSerializer.deserialize(payload),
+            key
+          ).perform
+        end
+      end
+    end
+  end
+end

data/lib/bulk_processor/back_end/active_job/split_csv_job.rb

@@ -0,0 +1,19 @@
+class BulkProcessor
+  module BackEnd
+    class ActiveJob
+      # ActiveJob to handle processing the CSV in the background
+      class SplitCSVJob < ::ActiveJob::Base
+        queue_as 'bulk_processor'
+
+        def perform(processor_class, payload, key, num_chunks)
+          BulkProcessor::SplitCSV.new(
+            processor_class.constantize,
+            PayloadSerializer.deserialize(payload),
+            key,
+            num_chunks
+          ).perform
+        end
+      end
+    end
+  end
+end

data/lib/bulk_processor/back_end/active_job.rb

@@ -1,20 +1,25 @@
+require 'active_job'
+
+require_relative 'active_job/process_csv_job'
+require_relative 'active_job/split_csv_job'
+
 class BulkProcessor
   module BackEnd
     # Execute jobs via ActiveJob, e.g. Resque
     class ActiveJob
       def initialize(processor_class:, payload:, key:)
-        @processor_class = processor_class
-        @payload = payload
+        @processor_class = processor_class.name
+        @payload = PayloadSerializer.serialize(payload)
         @key = key
       end
 
       def start
-        Job::ProcessCSV.perform_later(processor_class.name, payload, key)
+        ActiveJob::ProcessCSVJob.perform_later(processor_class, payload, key)
       end
 
       def split(num_processes)
-        Job::SplitCSV.perform_later(processor_class.name, payload,
-                                    key, num_processes)
+        ActiveJob::SplitCSVJob.perform_later(processor_class, payload, key,
+                                             num_processes)
       end
 
       private

data/lib/bulk_processor/back_end/dynosaur/process_csv_task.rb

@@ -0,0 +1,26 @@
+require 'rake'
+
+class BulkProcessor
+  module BackEnd
+    class ActiveJob
+      class ProcessCSVTask
+        include Rake::DSL
+
+        def install_task
+          namespace :bulk_processor do
+            desc 'Start processing a CSV file'
+            task :start, [:processor_class, :payload, :key] => :environment do |_task, args|
+              BulkProcessor::ProcessCSV.new(
+                args[:processor_class].constantize,
+                PayloadSerializer.deserialize(args[:payload]),
+                args[:key]
+              ).perform
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+BulkProcessor::BackEnd::ActiveJob::ProcessCSVTask.new.install_task

data/lib/bulk_processor/back_end/dynosaur/split_csv_task.rb

@@ -0,0 +1,27 @@
+require 'rake'
+
+class BulkProcessor
+  module BackEnd
+    class ActiveJob
+      class SplitCSVTask
+        include Rake::DSL
+
+        def install_task
+          namespace :bulk_processor do
+            desc 'Split a CSV file and process each piece'
+            task :split, [:processor_class, :payload, :key, :num_chunks] => :environment do |_task, args|
+              BulkProcessor::SplitCSV.new(
+                args[:processor_class].constantize,
+                PayloadSerializer.deserialize(args[:payload]),
+                args[:key],
+                args[:num_chunks].to_i
+              ).perform
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+BulkProcessor::BackEnd::ActiveJob::SplitCSVTask.new.install_task

data/lib/bulk_processor/back_end/dynosaur/tasks.rb

@@ -0,0 +1,2 @@
+require_relative 'process_csv_task'
+require_relative 'split_csv_task'

data/lib/bulk_processor/back_end/dynosaur.rb

@@ -1,12 +1,14 @@
 require 'dynosaur'
 
+require_relative 'dynosaur/tasks'
+
 class BulkProcessor
   module BackEnd
     # Execute jobs via rake tasks that will spawn a new Heroku dyno
     class Dynosaur
       def initialize(processor_class:, payload:, key:)
-        @processor_class = processor_class
-        @payload = payload
+        @processor_class = processor_class.name
+        @payload = PayloadSerializer.serialize(payload)
         @key = key
         configure_dynosaur
       end
@@ -14,7 +16,7 @@ class BulkProcessor
       def start
         args = {
           task: 'bulk_processor:start',
-          args: [processor_class.name, payload, key]
+          args: [processor_class, payload, key]
         }
         ::Dynosaur::Process::Heroku.new(args).start
       end
@@ -22,7 +24,7 @@ class BulkProcessor
       def split(num_processes)
         args = {
           task: 'bulk_processor:split',
-          args: [processor_class.name, payload, key, num_processes]
+          args: [processor_class, payload, key, num_processes.to_s]
         }
         ::Dynosaur::Process::Heroku.new(args).start
       end

data/lib/bulk_processor/back_end.rb

@@ -4,7 +4,7 @@ class BulkProcessor
       def start(processor_class:, payload:, key:, num_processes: 1)
         back_end = back_end_class.new(
           processor_class: processor_class,
-          payload: PayloadSerializer.serialize(payload),
+          payload: payload,
           key: key
         )
         num_processes > 1 ? back_end.split(num_processes) : back_end.start

data/lib/bulk_processor/config.rb

@@ -1,9 +1,17 @@
 class BulkProcessor
   # Store configuration data set by clients
   class Config
-    attr_reader :queue_adapter
+    attr_reader :back_end, :queue_adapter
     attr_writer :file_class
-    attr_accessor :back_end, :temp_directory
+    attr_accessor :temp_directory
+
+    def back_end=(back_end)
+      require_relative "back_end/#{back_end}"
+      @back_end = back_end
+    rescue LoadError => error
+      puts error.message
+      raise ArgumentError, "Invalid back-end: #{back_end}"
+    end
 
     def queue_adapter=(adapter)
       ActiveJob::Base.queue_adapter = @queue_adapter = adapter

data/lib/bulk_processor/process_csv.rb

@@ -0,0 +1,24 @@
+class BulkProcessor
+  class ProcessCSV
+    def initialize(processor_class, payload, key)
+      @processor_class = processor_class
+      @payload = payload
+      @key = key
+    end
+
+    def perform
+      file = BulkProcessor.config.file_class.new(key)
+      file.open do |f|
+        csv = CSV.parse(f.read, headers: true)
+        processor = processor_class.new(csv, payload: payload.merge('key' => key))
+        processor.start
+      end
+    ensure
+      file.try(:delete)
+    end
+
+    private
+
+    attr_reader :processor_class, :payload, :key
+  end
+end

data/lib/bulk_processor/split_csv.rb

@@ -0,0 +1,46 @@
+class BulkProcessor
+  class SplitCSV
+    def initialize(processor_class, payload, key, num_chunks)
+      @processor_class = processor_class
+      @payload = payload
+      @key = key
+      @num_chunks = num_chunks
+    end
+
+    def perform
+      splitter = FileSplitter.new(key: key, row_chunker: row_chunker)
+      keys = splitter.split!
+      keys.each do |key|
+        BackEnd.start(processor_class: processor_class, payload: payload, key: key)
+      end
+    rescue Exception => error
+      handle_error(error)
+      raise
+    ensure
+      BulkProcessor.config.file_class.new(key).delete
+    end
+
+    private
+
+    attr_reader :processor_class, :payload, :key, :num_chunks
+
+    def row_chunker
+      if processor_class.respond_to?(:boundary_column)
+        boundary_column = processor_class.boundary_column
+        RowChunker::Boundary.new(num_chunks, boundary_column: boundary_column)
+      else
+        RowChunker::Balanced.new(num_chunks)
+      end
+    end
+
+    def handle_error(error)
+      if processor_class.respond_to?(:handler_class)
+        handler = processor_class.handler_class.new(
+          payload: payload.merge('key' => key),
+          results: []
+        )
+        handler.fail!(error)
+      end
+    end
+  end
+end

data/lib/bulk_processor/version.rb

@@ -1,3 +1,3 @@
 class BulkProcessor
-  VERSION = '0.6.0'.freeze
+  VERSION = '0.7.0'.freeze
 end

data/lib/bulk_processor.rb

@@ -1,14 +1,12 @@
 require 'bulk_processor/back_end'
-require 'bulk_processor/back_end/active_job'
-require 'bulk_processor/back_end/dynosaur'
 require 'bulk_processor/config'
 require 'bulk_processor/file_splitter'
-require 'bulk_processor/job/process_csv'
-require 'bulk_processor/job/split_csv'
 require 'bulk_processor/payload_serializer'
+require 'bulk_processor/process_csv'
 require 'bulk_processor/row_chunker/balanced'
 require 'bulk_processor/row_chunker/boundary'
 require 'bulk_processor/s3_file'
+require 'bulk_processor/split_csv'
 require 'bulk_processor/stream_encoder'
 require 'bulk_processor/validated_csv'
 require 'bulk_processor/version'

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: bulk-processor
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Tom Collier, Justin Richard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-22 00:00:00.000000000 Z
+date: 2016-01-26 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: activejob
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -39,33 +25,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.1'
 - !ruby/object:Gem::Dependency
-  name: dynosaur
+  name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: '1.5'
 - !ruby/object:Gem::Dependency
-  name: rack
+  name: activejob
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
-  type: :runtime
+        version: '4'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: dynosaur
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -145,7 +145,12 @@ files:
 - lib/bulk_processor.rb
 - lib/bulk_processor/back_end.rb
 - lib/bulk_processor/back_end/active_job.rb
+- lib/bulk_processor/back_end/active_job/process_csv_job.rb
+- lib/bulk_processor/back_end/active_job/split_csv_job.rb
 - lib/bulk_processor/back_end/dynosaur.rb
+- lib/bulk_processor/back_end/dynosaur/process_csv_task.rb
+- lib/bulk_processor/back_end/dynosaur/split_csv_task.rb
+- lib/bulk_processor/back_end/dynosaur/tasks.rb
 - lib/bulk_processor/config.rb
 - lib/bulk_processor/csv_processor.rb
 - lib/bulk_processor/csv_processor/no_op_handler.rb
@@ -153,14 +158,13 @@ files:
 - lib/bulk_processor/csv_processor/result.rb
 - lib/bulk_processor/csv_processor/row_processor.rb
 - lib/bulk_processor/file_splitter.rb
-- lib/bulk_processor/job/process_csv.rb
-- lib/bulk_processor/job/split_csv.rb
 - lib/bulk_processor/payload_serializer.rb
+- lib/bulk_processor/process_csv.rb
 - lib/bulk_processor/row_chunker/balanced.rb
 - lib/bulk_processor/row_chunker/boundary.rb
 - lib/bulk_processor/s3_file.rb
+- lib/bulk_processor/split_csv.rb
 - lib/bulk_processor/stream_encoder.rb
-- lib/bulk_processor/tasks.rb
 - lib/bulk_processor/validated_csv.rb
 - lib/bulk_processor/version.rb
 homepage: 

data/lib/bulk_processor/job/process_csv.rb

@@ -1,22 +0,0 @@
-require 'active_job'
-
-class BulkProcessor
-  # ActiveJob to handle processing the CSV in the background
-  module Job
-    class ProcessCSV < ActiveJob::Base
-      queue_as 'bulk_processor'
-
-      def perform(processor_class, payload, key)
-        file = BulkProcessor.config.file_class.new(key)
-        payload = PayloadSerializer.deserialize(payload).merge('key' => key)
-        file.open do |f|
-          csv = CSV.parse(f.read, headers: true)
-          processor = processor_class.constantize.new(csv, payload: payload)
-          processor.start
-        end
-      ensure
-        file.try(:delete)
-      end
-    end
-  end
-end

data/lib/bulk_processor/job/split_csv.rb

@@ -1,41 +0,0 @@
-require 'active_job'
-
-class BulkProcessor
-  # ActiveJob to handle processing the CSV in the background
-  module Job
-    class SplitCSV < ActiveJob::Base
-      queue_as 'bulk_processor'
-
-      def perform(processor_class, payload, key, num_chunks)
-        processor_class = processor_class.constantize
-        chunker = row_chunker(processor_class, num_chunks)
-        payload = PayloadSerializer.deserialize(payload)
-        splitter = FileSplitter.new(key: key, row_chunker: chunker)
-        keys = splitter.split!
-        keys.each do |key|
-          BackEnd.start(processor_class: processor_class, payload: payload, key: key)
-        end
-      rescue Exception => error
-        if processor_class.respond_to?(:handler_class)
-          payload = payload.merge('key' => key)
-          handler = processor_class.handler_class.new(payload: payload, results: [])
-          handler.fail!(error)
-        end
-        raise
-      ensure
-        BulkProcessor.config.file_class.new(key).delete
-      end
-
-      private
-
-      def row_chunker(processor_class, num_chunks)
-        if processor_class.respond_to?(:boundary_column)
-          boundary_column = processor_class.boundary_column
-          RowChunker::Boundary.new(num_chunks, boundary_column: boundary_column)
-        else
-          RowChunker::Balanced.new(num_chunks)
-        end
-      end
-    end
-  end
-end

data/lib/bulk_processor/tasks.rb

@@ -1,32 +0,0 @@
-require 'rake'
-
-class BulkProcessor
-  class Tasks
-    include Rake::DSL
-
-    def install_tasks
-      namespace :bulk_processor do
-        desc 'Start processing a CSV file'
-        task :start, [:processor_class, :payload, :key] => :environment do |_task, args|
-          Job::ProcessCSV.new.perform(
-            args[:processor_class],
-            args[:payload],
-            args[:key]
-          )
-        end
-
-        desc 'Split a CSV file and process each piece'
-        task :split, [:processor_class, :payload, :key, :num_chunks] => :environment do |_task, args|
-          Job::SplitCSV.new.perform(
-            args[:processor_class],
-            args[:payload],
-            args[:key],
-            args[:num_chunks]
-          )
-        end
-      end
-    end
-  end
-end
-
-BulkProcessor::Tasks.new.install_tasks