exel 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07a61d1ecdfd5d0d957caa0dd064631be8c3e546
-  data.tar.gz: 364c194ce4cb48938f1cc863b6bdf0a5b7e74e47
+  metadata.gz: 85ab889713fbe1368ad2e1e3186f16fb1fe39ac5
+  data.tar.gz: ed633de60e0cdbf125bfe5a1cfd21d110341c43e
 SHA512:
-  metadata.gz: d004472bd8b4b3c6e496377c89cb261abcaa59fad5284009ecca65bee86dff428545af4ab27cd47ca74becbcdc9bb0a3d81962a67cf81f6437b5195a990cdf5f
-  data.tar.gz: debffc94d7f7fd53064f2b8f5317a68f3cbcaa4e961d19c6c2af630a31c39d79b4b42c111947e361a3f5a838853ff29d4a9814fb0892609da994f4c433517b85
+  metadata.gz: 3e739773310941eac8d09ac6793bb5d9b296c1cb22b7015f1c467ed4c14cbe12a3d92ca30cdb7fc39d7e676a12af30f2502bdd3a1dc395c2cce67673abbaed20
+  data.tar.gz: 316c744a8f814dec1191bbfd7964a28bb7c2628ae7bb2726ce0884a4d8392b71338364aa14be51a7c6524a9489cd7c1089a29e86801544dfa08e65576f6d8986

data/.codeclimate.yml

@@ -0,0 +1,16 @@
+---
+engines:
+  duplication:
+    enabled: true
+    config:
+      languages:
+      - ruby
+  fixme:
+    enabled: true
+  rubocop:
+    enabled: true
+ratings:
+  paths:
+  - "**.rb"
+exclude_paths:
+- spec/**/*

data/.rubocop.yml

@@ -15,3 +15,7 @@ Style/SpaceInsideHashLiteralBraces:
 
 RSpec/DescribedClass:
   Enabled: false
+
+Metrics/ModuleLength:
+  Exclude:
+    - 'spec/**/*'

data/.rubocop_todo.yml

@@ -6,16 +6,6 @@
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 1
-# Configuration parameters: CountComments.
-Metrics/MethodLength:
-  Max: 11
-
-# Offense count: 2
-# Configuration parameters: CountComments.
-Metrics/ModuleLength:
-  Max: 166
-
 # Offense count: 65
 # Configuration parameters: CustomTransform, IgnoredWords.
 RSpec/ExampleWording:

data/README.md

@@ -3,7 +3,15 @@
 [![Code Climate](https://codeclimate.com/github/47colborne/exel/badges/gpa.svg)](https://codeclimate.com/github/47colborne/exel)
 [![Build Status](https://snap-ci.com/47colborne/exel/branch/master/build_image)](https://snap-ci.com/47colborne/exel/branch/master)
 
-TODO: Write a gem description
+EXEL is the Elastic eXEcution Language, a simple Ruby DSL for creating processing jobs that can be run on a single machine, or scaled up to run on dozens of machines with no changes to the job itself. To run a job on more than one machine, simply install EXEL async and remote provider gems to integrate with your preferred platforms. The currently implemented providers so far are:
+
+**Async Providers**
+
+* [exel-sidekiq](https://github.com/47colborne/exel-sidekiq)
+
+**Remote Providers**
+
+* [exel-s3](https://github.com/47colborne/exel-s3)
 
 ## Installation
 
@@ -21,7 +29,62 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+### Processors
+
+A processor can be any class that provides the following interface:
+
+    class MyProcessor
+        def initialize(context)
+            # typically context is assigned to @context here
+        end
+        
+        def process(block)
+            # do your work here
+        end
+    end
+
+Processors are initialized immediately before ```#process``` is called, allowing them to set up any state that they need from the context. The ```#process``` method is where your processing logic will be implemented. Processors should be focused on performing one particular aspect of the processing that you want to accomplish, allowing your job to be composed of a sequence of small processing steps. If a block was given in the call to ```process``` in the job DSL, it will be passed as the argument to ```#process``` and can be run with: ```block.run(@context)```
+
+### The Context
+
+The ```Context``` class has a Hash-like interface and acts as shared storage for the various processors that make up a job. Processors take their expected inputs from the context, and place any resulting outputs there for subsequent processors to access. Values are typically placed in the context through the following means:
+
+* Initial context set up before the job is run
+* Arguments passed to processors in the job DSL
+* Outputs assigned by processors during processing
+
+If you use EXEL with an async provider, such as [exel-sidekiq](https://github.com/47colborne/exel-sidekiq), and a remote provider, such as [exel-s3](https://github.com/47colborne/exel-s3), a context switch will occur when the ```async``` command is executed. Context shifts involve serializing the context and uploading it via the remote provider, then downloading and deserializing it when the async block is eventually run. This allows the processors to pass the results of their process through the sequence of processors in the job, without having to be concerned with when, where, or how those processors will be run.
+
+### Supported Commands
+
+* ```process``` Execute the given processor class (specified by the ```:with``` option), given the current context and any additional arguments provided
+* ```split``` Split the input data into 1000 line chunks and run the given block for each chunk. Assumes that the input data is a CSV formatted file referenced by ```context[:resource]```. When each block is run, ```context[:resource]``` will reference to the chunk file.
+* ```async``` Asynchronously run the given block. Uses the configured async provider to execute the block.
+
+### Example job
+
+    EXEL::Job.define :example_job do
+        # Download a large CSV data file
+        process with: FTPDownloader, host: ftp.example.com, path: context[:file_path]
+        
+        # split it into smaller 1000 line files
+        split do
+            # for each file asynchronously run the following sequence of processors
+            async do  
+                process with: RecordLoader # convert each row of data into your domain model
+                process with: SomeProcessor # apply some additional processing to each record
+                process with: RecordSaver # write this batch of records to your database
+                process with: ExternalServiceProcessor # interact with some service, ex: updating a search index
+            end
+        end
+    end
+
+Elsewhere in your application, you could run this job as follows:
+
+    def run_example_job(file_path)
+        context = EXEL::Context.new(file_path: file_path, user: 'username')
+        EXEL::Job.run(:example_job, context)
+    end
 
 ## Contributing
 

data/lib/exel/context.rb

@@ -8,9 +8,12 @@ module EXEL
       @table = initial_context
     end
 
+    def deep_dup
+      Context.deserialize(serialize)
+    end
+
     def serialize
-      remotized_table = @table.each_with_object({}) { |(key, value), acc| acc[key] = EXEL::Value.remotize(value) }
-      EXEL::Value.remotize(serialized_context(remotized_table))
+      EXEL::Value.remotize(serialized_context)
     end
 
     def self.deserialize(uri)
@@ -50,13 +53,17 @@ module EXEL
 
     private
 
-    def serialized_context(table)
+    def serialized_context
       file = Tempfile.new(SecureRandom.uuid, encoding: 'ascii-8bit')
-      file.write(Marshal.dump(Context.new(table)))
+      file.write(Marshal.dump(Context.new(remotized_table)))
       file.rewind
       file
     end
 
+    def remotized_table
+      @table.each_with_object({}) { |(key, value), acc| acc[key] = EXEL::Value.remotize(value) }
+    end
+
     def get_deferred(value)
       if deferred?(value)
         value = value.get(self)

data/lib/exel/processors/split_processor.rb

@@ -7,44 +7,33 @@ module EXEL
     class SplitProcessor
       include EXEL::ProcessorHelper
 
-      attr_accessor :chunk_size, :file_name, :block
+      attr_accessor :file_name, :block
 
       DEFAULT_CHUNK_SIZE = 1000
 
       def initialize(context)
-        @chunk_size = DEFAULT_CHUNK_SIZE
         @buffer = []
         @tempfile_count = 0
         @context = context
-
         @file = context[:resource]
-        @file_name = filename(@file)
-        @csv_options = context[:csv_options] || {col_sep: ','}
 
         log_prefix_with '[SplitProcessor]'
       end
 
       def process(callback)
         log_process do
-          begin
-            CSV.foreach(@file.path, @csv_options) do |line|
-              process_line(line, callback)
-            end
-          rescue CSV::MalformedCSVError => e
-            log_error "CSV::MalformedCSVError => #{e.message}"
-          end
-          process_line(:eof, callback)
-          File.delete(@file.path)
+          process_file(callback)
+          finish(callback)
         end
       end
 
       def process_line(line, callback)
         if line == :eof
-          flush_buffer callback
+          flush_buffer(callback)
         else
           @buffer << CSV.generate_line(line)
 
-          flush_buffer callback if buffer_full?
+          flush_buffer(callback) if buffer_full?
         end
       end
 
@@ -54,31 +43,51 @@ module EXEL
         chunk.write(content)
         chunk.rewind
 
-        log_info "Generated chunk # #{@tempfile_count} for file #{@file_name} in #{chunk.path}"
+        log_info "Generated chunk # #{@tempfile_count} for file #{filename(@file)} in #{chunk.path}"
         chunk
       end
 
-      def chunk_filename
-        "#{@file_name}_#{@tempfile_count}_"
-      end
+      private
 
-      def filename(file)
-        file_name_with_extension = file.path.split('/').last
-        file_name_with_extension.split('.').first
-      end
+      def process_file(callback)
+        csv_options = @context[:csv_options] || {col_sep: ','}
 
-      private
+        CSV.foreach(@file.path, csv_options) do |line|
+          process_line(line, callback)
+        end
+      rescue CSV::MalformedCSVError => e
+        log_error "CSV::MalformedCSVError => #{e.message}"
+      end
 
       def flush_buffer(callback)
         unless @buffer.empty?
           file = generate_chunk(@buffer.join(''))
           callback.run(@context.merge!(resource: file))
         end
+
         @buffer = []
       end
 
       def buffer_full?
-        @buffer.size == @chunk_size
+        @buffer.size == chunk_size
+      end
+
+      def chunk_size
+        DEFAULT_CHUNK_SIZE
+      end
+
+      def chunk_filename
+        "#{filename(@file)}_#{@tempfile_count}_"
+      end
+
+      def filename(file)
+        file_name_with_extension = file.path.split('/').last
+        file_name_with_extension.split('.').first
+      end
+
+      def finish(callback)
+        process_line(:eof, callback)
+        File.delete(@file.path)
       end
     end
   end

data/lib/exel/providers/threaded_async_provider.rb

@@ -6,7 +6,7 @@ module EXEL
       end
 
       def do_async(block)
-        Thread.new { block.start(@context) }
+        Thread.new { block.start(@context.deep_dup) }
       end
     end
   end

data/lib/exel/version.rb

@@ -1,3 +1,3 @@
 module EXEL
-  VERSION = '1.0.0'
+  VERSION = '1.0.1'
 end

data/spec/exel/context_spec.rb

@@ -9,6 +9,19 @@ module EXEL
       end
     end
 
+    describe '#deep_dup' do
+      it 'returns a deep copy of itself' do
+        context[:a] = {nested: []}
+
+        dup = context.deep_dup
+        expect(context).to eq(dup)
+        expect(context).to_not be_equal(dup)
+
+        dup[:a][:nested] << 1
+        expect(context[:a][:nested]).to be_empty
+      end
+    end
+
     describe '#serialize' do
       before { allow(Value).to receive(:upload) }
 

data/spec/exel/processors/split_processor_spec.rb

@@ -43,7 +43,7 @@ module EXEL
           {input: 4, chunks: %W(0\n1\n 2\n3\n)}
         ].each do |data|
           it "should produce #{data[:chunks].size} chunks with #{data[:input]} input lines" do
-            splitter.chunk_size = 2
+            allow(splitter).to receive(:chunk_size).and_return(2)
 
             data[:chunks].each do |chunk|
               expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
@@ -67,10 +67,8 @@ module EXEL
 
         it 'should create a file with a unique name' do
           3.times do |i|
-            index = i + 1
-            file = splitter.generate_chunk("#{index}")
-            file_name = splitter.filename(file)
-            expect(file_name).to include("text_#{index}_")
+            file = splitter.generate_chunk('content')
+            expect(file.path).to include("text_#{i + 1}_")
           end
         end
       end
@@ -83,7 +81,7 @@ module EXEL
           content << line
         end
 
-        StringIO.new content
+        StringIO.new(content)
       end
     end
   end

data/spec/exel/providers/threaded_async_provider_spec.rb

@@ -0,0 +1,53 @@
+module EXEL
+  module Providers
+    class ContextMutatingProcessor
+      def initialize(context)
+        @context = context
+      end
+
+      def process(_block)
+        @context[:array] << @context[:arg]
+      end
+    end
+
+    describe ThreadedAsyncProvider do
+      subject { described_class.new(context) }
+      let(:context) { EXEL::Context.new }
+
+      describe '#do_async' do
+        let(:dsl_block) { instance_double(ASTNode) }
+
+        it 'runs the block in a new thread' do
+          expect(dsl_block).to receive(:start).with(context)
+          expect(Thread).to receive(:new).and_yield
+
+          subject.do_async(dsl_block)
+        end
+
+        it 'passes a copy of the context to each thread' do
+          context[:array] = []
+          complete = 0
+
+          EXEL::Job.define :thread_test do
+            async do
+              process with: ContextMutatingProcessor, arg: 1
+              complete += 1
+            end
+
+            async do
+              process with: ContextMutatingProcessor, arg: 2
+              complete += 1
+            end
+          end
+
+          EXEL::Job.run(:thread_test, context)
+
+          start_time = Time.now
+          sleep 0.1 while complete < 2 && Time.now - start_time < 2
+
+          expect(context[:array]).to be_empty
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: exel
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - yroo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-16 00:00:00.000000000 Z
+date: 2016-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -171,6 +171,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".codeclimate.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
@@ -209,8 +210,8 @@ files:
 - spec/exel/null_instruction_spec.rb
 - spec/exel/processors/async_processor_spec.rb
 - spec/exel/processors/split_processor_spec.rb
-- spec/exel/providers/local_async_provider_spec.rb
 - spec/exel/providers/local_file_provider_spec.rb
+- spec/exel/providers/threaded_async_provider_spec.rb
 - spec/exel/sequence_node_spec.rb
 - spec/exel/value_spec.rb
 - spec/exel_spec.rb
@@ -250,8 +251,8 @@ test_files:
 - spec/exel/null_instruction_spec.rb
 - spec/exel/processors/async_processor_spec.rb
 - spec/exel/processors/split_processor_spec.rb
-- spec/exel/providers/local_async_provider_spec.rb
 - spec/exel/providers/local_file_provider_spec.rb
+- spec/exel/providers/threaded_async_provider_spec.rb
 - spec/exel/sequence_node_spec.rb
 - spec/exel/value_spec.rb
 - spec/exel_spec.rb

data/spec/exel/providers/local_async_provider_spec.rb

@@ -1,19 +0,0 @@
-module EXEL
-  module Providers
-    describe ThreadedAsyncProvider do
-      subject { described_class.new(context) }
-      let(:context) { EXEL::Context.new }
-
-      describe '#do_async' do
-        let(:dsl_block) { instance_double(ASTNode) }
-
-        it 'runs the block in a new thread' do
-          expect(dsl_block).to receive(:start).with(context)
-          expect(Thread).to receive(:new).and_yield
-
-          subject.do_async(dsl_block)
-        end
-      end
-    end
-  end
-end