exel 0.9.0 → 1.0.0

metadata

@@ -1,115 +1,115 @@
 --- !ruby/object:Gem::Specification
 name: exel
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 1.0.0
 platform: ruby
 authors:
 - yroo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-22 00:00:00.000000000 Z
+date: 2015-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
-  type: :runtime
+        version: '1.6'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
-  name: sidekiq
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
-  type: :runtime
+        version: '10'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '10'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '3'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '2'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '4'
 - !ruby/object:Gem::Dependency
-  name: guard
+  name: guard-rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1'
 - !ruby/object:Gem::Dependency
-  name: guard-rspec
+  name: terminal-notifier
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '1'
 - !ruby/object:Gem::Dependency
-  name: terminal-notifier
+  name: terminal-notifier-guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -123,7 +123,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: terminal-notifier-guard
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -136,6 +150,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec-focused
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A DSL for defining jobs that can be run in a highly scalable manner
 email:
 - dev@yroo.com
@@ -145,6 +173,8 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
 - Gemfile
 - Guardfile
 - LICENSE.txt
@@ -156,9 +186,6 @@ files:
 - lib/exel/context.rb
 - lib/exel/deferred_context_value.rb
 - lib/exel/error/job_termination.rb
-- lib/exel/execution_worker.rb
-- lib/exel/handlers/s3_handler.rb
-- lib/exel/handlers/sidekiq_handler.rb
 - lib/exel/instruction.rb
 - lib/exel/instruction_node.rb
 - lib/exel/job.rb
@@ -167,15 +194,14 @@ files:
 - lib/exel/processor_helper.rb
 - lib/exel/processors/async_processor.rb
 - lib/exel/processors/split_processor.rb
-- lib/exel/resource.rb
+- lib/exel/providers/local_file_provider.rb
+- lib/exel/providers/threaded_async_provider.rb
 - lib/exel/sequence_node.rb
+- lib/exel/value.rb
 - lib/exel/version.rb
 - spec/exel/ast_node_spec.rb
 - spec/exel/context_spec.rb
 - spec/exel/deferred_context_value_spec.rb
-- spec/exel/execution_worker_spec.rb
-- spec/exel/handlers/s3_handler_spec.rb
-- spec/exel/handlers/sidekiq_handler_spec.rb
 - spec/exel/instruction_node_spec.rb
 - spec/exel/instruction_spec.rb
 - spec/exel/job_spec.rb
@@ -183,8 +209,11 @@ files:
 - spec/exel/null_instruction_spec.rb
 - spec/exel/processors/async_processor_spec.rb
 - spec/exel/processors/split_processor_spec.rb
-- spec/exel/resource_spec.rb
+- spec/exel/providers/local_async_provider_spec.rb
+- spec/exel/providers/local_file_provider_spec.rb
 - spec/exel/sequence_node_spec.rb
+- spec/exel/value_spec.rb
+- spec/exel_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/47colborne/exel
 licenses:
@@ -206,7 +235,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: EXEL, the Elastic eXEcution Language
@@ -214,9 +243,6 @@ test_files:
 - spec/exel/ast_node_spec.rb
 - spec/exel/context_spec.rb
 - spec/exel/deferred_context_value_spec.rb
-- spec/exel/execution_worker_spec.rb
-- spec/exel/handlers/s3_handler_spec.rb
-- spec/exel/handlers/sidekiq_handler_spec.rb
 - spec/exel/instruction_node_spec.rb
 - spec/exel/instruction_spec.rb
 - spec/exel/job_spec.rb
@@ -224,6 +250,9 @@ test_files:
 - spec/exel/null_instruction_spec.rb
 - spec/exel/processors/async_processor_spec.rb
 - spec/exel/processors/split_processor_spec.rb
-- spec/exel/resource_spec.rb
+- spec/exel/providers/local_async_provider_spec.rb
+- spec/exel/providers/local_file_provider_spec.rb
 - spec/exel/sequence_node_spec.rb
+- spec/exel/value_spec.rb
+- spec/exel_spec.rb
 - spec/spec_helper.rb

data/lib/exel/execution_worker.rb

@@ -1,13 +0,0 @@
-require 'sidekiq'
-
-module EXEL
-  class ExecutionWorker
-    include Sidekiq::Worker
-
-    def perform(context_uri)
-      context = Context.deserialize(context_uri)
-      block = context[:_block]
-      block.start(context)
-    end
-  end
-end

data/lib/exel/handlers/s3_handler.rb

@@ -1,43 +0,0 @@
-require 'aws-sdk-resources'
-require 'tempfile'
-
-module EXEL
-  module Handlers
-    class S3Handler
-      def upload(file)
-        filename = get_filename(file)
-        obj = get_object(filename)
-        obj.upload_file(file)
-        file.close
-
-        "s3://#{filename}"
-      end
-
-      def download(uri)
-        filename = uri.partition('://').last
-        obj = get_object(filename)
-        file = Tempfile.new(filename, encoding: Encoding::ASCII_8BIT)
-        obj.get(response_target: file)
-        file.set_encoding(Encoding::UTF_8)
-        file
-      end
-
-      def get_object(filename)
-        s3 = Aws::S3::Resource.new(
-            credentials: Aws::Credentials.new(
-                EXEL.configuration[:aws][:access_key_id],
-                EXEL.configuration[:aws][:secret_access_key]
-            ),
-            region: 'us-east-1'
-        )
-        s3.bucket(EXEL.configuration[:s3_bucket]).object(filename)
-      end
-
-      private
-
-      def get_filename(file)
-        file.path.split('/').last
-      end
-    end
-  end
-end

data/lib/exel/handlers/sidekiq_handler.rb

@@ -1,21 +0,0 @@
-require 'sidekiq'
-
-module EXEL
-  module Handlers
-    class SidekiqHandler
-      def initialize(context)
-        @context = context
-      end
-
-      def do_async(block)
-        @context[:_block] = block
-
-        push_args = {'class' => ExecutionWorker, 'args' => [@context.serialize]}
-        push_args['queue'] = @context[:queue] if @context[:queue]
-        push_args['retry'] = @context[:retry] if @context[:retry]
-
-        Sidekiq::Client.push(push_args)
-      end
-    end
-  end
-end

data/spec/exel/execution_worker_spec.rb

@@ -1,13 +0,0 @@
-module EXEL
-  describe ExecutionWorker do
-    it 'should run the given block with the deserialized context' do
-      dsl_block = instance_double(SequenceNode)
-      context = Context.new(test1: 'foo', test2: 2, _block: dsl_block)
-      context_uri = 'test uri'
-
-      expect(Context).to receive(:deserialize).with(context_uri).and_return(context)
-      expect(dsl_block).to receive(:start).with(context)
-      ExecutionWorker.new.perform(context_uri)
-    end
-  end
-end

data/spec/exel/handlers/s3_handler_spec.rb

@@ -1,49 +0,0 @@
-module EXEL
-  module Handlers
-    describe S3Handler do
-      subject(:handler) { S3Handler.new }
-
-      describe '#get_object' do
-        before do
-          EXEL.configure { |config| config[:s3_bucket] = 'bucket' }
-        end
-
-        it 'should have the correct bucket and file names' do
-          file_name = 'abc.txt'
-          s3_obj = handler.get_object(file_name)
-          expect(s3_obj.bucket_name).to eq('bucket')
-          expect(s3_obj.key).to eq(file_name)
-        end
-      end
-
-      describe '#upload' do
-        let(:file) { double(path: '/path/to/abc.txt', close: nil) }
-
-        it 'should upload a file to s3' do
-          expect_any_instance_of(Aws::S3::Object).to receive(:upload_file).with(file)
-
-          handler.upload(file)
-        end
-
-        it 'should return the file URI of the uploaded file' do
-          allow_any_instance_of(Aws::S3::Object).to receive(:upload_file).with(file)
-          expect(handler.upload(file)).to eq('s3://abc.txt')
-        end
-      end
-
-      describe '#download' do
-        it 'should download the file from s3' do
-          file = double(:file)
-          s3_object = double(:s3_object)
-
-          expect(handler).to receive(:get_object).with('abc.txt').and_return(s3_object)
-          expect(Tempfile).to receive(:new).with('abc.txt', encoding: Encoding::ASCII_8BIT).and_return(file)
-          expect(s3_object).to receive(:get).with(hash_including(response_target: file)).and_return(file)
-          expect(file).to receive(:set_encoding).with(Encoding::UTF_8)
-
-          expect(handler.download('s3://abc.txt')).to eq(file)
-        end
-      end
-    end
-  end
-end

data/spec/exel/handlers/sidekiq_handler_spec.rb

@@ -1,54 +0,0 @@
-module EXEL
-  module Handlers
-    describe SidekiqHandler do
-      subject(:handler) { SidekiqHandler.new(context) }
-      let(:callback) { instance_double(SequenceNode) }
-      let(:context) { EXEL::Context.new }
-      let(:serialized_context_uri) { 'context_uri' }
-
-      describe '#do_async' do
-        before do
-          allow(context).to receive(:serialize).and_return(serialized_context_uri)
-        end
-
-        it 'should add the callback to the context before serializing it' do
-          expect(context).to receive(:[]=).with(:_block, callback).ordered
-          expect(context).to receive(:serialize).ordered
-          handler.do_async(callback)
-        end
-
-        context 'with queue name' do
-          let(:context) { EXEL::Context.new(queue: 'import_processor') }
-
-          it 'should push the execution worker to the given queue' do
-            expect(Sidekiq::Client).to receive(:push).with('queue' => context[:queue], 'class' => ExecutionWorker, 'args' => [serialized_context_uri])
-            handler.do_async(callback)
-          end
-        end
-
-        context 'without queue name' do
-          it 'should push the execution worker to the default queue' do
-            expect(Sidekiq::Client).to receive(:push).with('class' => ExecutionWorker, 'args' => [serialized_context_uri])
-            handler.do_async(callback)
-          end
-        end
-
-        context 'with retries specified' do
-          let(:context) { EXEL::Context.new(retry: 1) }
-
-          it 'should push the execution worker with a specified number of retries' do
-            expect(Sidekiq::Client).to receive(:push).with('retry' => context[:retry], 'class' => ExecutionWorker, 'args' => [serialized_context_uri])
-            handler.do_async(callback)
-          end
-        end
-
-        context 'with no retries specified' do
-          it 'should push the execution worker with no specified number of retries' do
-            expect(Sidekiq::Client).to receive(:push).with('class' => ExecutionWorker, 'args' => [serialized_context_uri])
-            handler.do_async(callback)
-          end
-        end
-      end
-    end
-  end
-end

data/spec/exel/resource_spec.rb

@@ -1,51 +0,0 @@
-module EXEL
-  describe Resource do
-    let(:s3_uri) { 's3://test_file.csv' }
-
-    describe '.remotize' do
-      context 'when passed in value is not a file' do
-        it 'should return the value' do
-          expect(Resource.remotize('test string')).to eq('test string')
-        end
-      end
-
-      context 'when the passed in value is a file' do
-        [File, Tempfile].each do |file_class|
-          context "with a #{file_class}" do
-            before do
-              @file = instance_double(file_class)
-              allow(@file).to receive(:is_a?) { |klass| klass == file_class }
-            end
-
-            it 'should upload the file to S3' do
-              expect_any_instance_of(Handlers::S3Handler).to receive(:upload).with(@file)
-              Resource.remotize(@file)
-            end
-
-            it 'should return a remote file URI' do
-              allow_any_instance_of(Handlers::S3Handler).to receive(:upload).with(@file).and_return(s3_uri)
-              expect(Resource.remotize(@file)).to eq(s3_uri)
-            end
-          end
-        end
-      end
-    end
-
-    describe '.localize' do
-      context 'with a localized value' do
-        it 'should return the value' do
-          expect(Resource.localize('test string')).to eq('test string')
-        end
-      end
-
-      context 'with a remote file' do
-        it 'should return the downloaded file' do
-          file = double(:file)
-          expect_any_instance_of(Handlers::S3Handler).to receive(:download).with(s3_uri).and_return(file)
-
-          expect(Resource.localize(s3_uri)).to eq(file)
-        end
-      end
-    end
-  end
-end