exel 0.0.1 → 0.9.0

data/spec/exel/processors/async_processor_spec.rb

@@ -0,0 +1,16 @@
+module EXEL
+  module Processors
+    describe AsyncProcessor do
+      subject(:processor) { AsyncProcessor.new(context) }
+      let(:context) { EXEL::Context.new }
+      let(:block) { instance_double(SequenceNode) }
+
+      describe '#process' do
+        it 'should call do_async on the async handler' do
+          expect(processor.handler).to receive(:do_async).with(block)
+          processor.process(block)
+        end
+      end
+    end
+  end
+end

data/spec/exel/processors/split_processor_spec.rb

@@ -0,0 +1,90 @@
+  module EXEL
+    module Processors
+      describe SplitProcessor do
+        let(:chunk_file) { instance_double(File) }
+        let(:file) { create_file(1) }
+        let(:context) { Context.new(resource: file) }
+        let(:callback) { instance_double(SequenceNode) }
+        subject(:splitter) { SplitProcessor.new(context) }
+
+        before do
+          allow_any_instance_of(StringIO).to receive(:path).and_return('/text.txt')
+          allow(File).to receive(:delete)
+        end
+
+        describe '#process' do
+          let(:file) { create_file(3) }
+
+          it 'should process file with 3 lines line by line' do
+            allow(CSV).to receive(:foreach).and_yield("line0").and_yield("line1").and_yield("line2")
+
+            3.times do |i|
+              expect(splitter).to receive(:process_line).with("line#{i}", callback)
+            end
+            expect(splitter).to receive(:process_line).with(:eof, callback)
+
+            expect(File).to receive(:delete).with(file.path)
+
+            splitter.process(callback)
+          end
+
+          it 'should abort parsing the csv file if it is malformed' do
+            allow(CSV).to receive(:foreach).and_raise(CSV::MalformedCSVError)
+            expect(splitter).to receive(:process_line).with(:eof, callback)
+
+            splitter.process(callback)
+          end
+        end
+
+        describe '#process_line' do
+          [
+              {input: 1, chunks: %W(0\n)},
+              {input: 3, chunks: %W(0\n1\n 2\n)},
+              {input: 4, chunks: %W(0\n1\n 2\n3\n)}
+          ].each do |data|
+            it "should produce #{data[:chunks].size} chunks with #{data[:input]} input lines" do
+              splitter.chunk_size = 2
+
+              data[:chunks].each do |chunk|
+                expect(splitter).to receive(:generate_chunk).with(chunk).and_return(chunk_file)
+                expect(callback).to receive(:run).with(context) do
+                  expect(context[:resource]).to eq(chunk_file)
+                end
+              end
+
+              data[:input].times { |i| splitter.process_line([i.to_s], callback) }
+              splitter.process_line(:eof, callback)
+            end
+          end
+        end
+
+        describe '#generate_chunk' do
+          it 'should create a file with the contents of the given string' do
+            file = splitter.generate_chunk('abc')
+            content = file.read
+            expect(content).to eq('abc')
+          end
+
+          it 'should create a file with a unique name' do
+            3.times do |i|
+              index = i + 1
+              file = splitter.generate_chunk("#{index}")
+              file_name = splitter.filename(file)
+              expect(file_name).to include("text_#{index}_")
+            end
+          end
+        end
+
+        def create_file(lines)
+          content = ''
+
+          lines.times do |i|
+            line = CSV.generate_line(["line#{i}"])
+            content << line
+          end
+
+          StringIO.new content
+        end
+      end
+    end
+  end

data/spec/exel/resource_spec.rb

@@ -0,0 +1,51 @@
+module EXEL
+  describe Resource do
+    let(:s3_uri) { 's3://test_file.csv' }
+
+    describe '.remotize' do
+      context 'when passed in value is not a file' do
+        it 'should return the value' do
+          expect(Resource.remotize('test string')).to eq('test string')
+        end
+      end
+
+      context 'when the passed in value is a file' do
+        [File, Tempfile].each do |file_class|
+          context "with a #{file_class}" do
+            before do
+              @file = instance_double(file_class)
+              allow(@file).to receive(:is_a?) { |klass| klass == file_class }
+            end
+
+            it 'should upload the file to S3' do
+              expect_any_instance_of(Handlers::S3Handler).to receive(:upload).with(@file)
+              Resource.remotize(@file)
+            end
+
+            it 'should return a remote file URI' do
+              allow_any_instance_of(Handlers::S3Handler).to receive(:upload).with(@file).and_return(s3_uri)
+              expect(Resource.remotize(@file)).to eq(s3_uri)
+            end
+          end
+        end
+      end
+    end
+
+    describe '.localize' do
+      context 'with a localized value' do
+        it 'should return the value' do
+          expect(Resource.localize('test string')).to eq('test string')
+        end
+      end
+
+      context 'with a remote file' do
+        it 'should return the downloaded file' do
+          file = double(:file)
+          expect_any_instance_of(Handlers::S3Handler).to receive(:download).with(s3_uri).and_return(file)
+
+          expect(Resource.localize(s3_uri)).to eq(file)
+        end
+      end
+    end
+  end
+end

data/spec/exel/sequence_node_spec.rb

@@ -0,0 +1,24 @@
+module EXEL
+  describe SequenceNode do
+    let(:context) { {} }
+
+    def build_tree
+      @node_2 = instance_double(ASTNode)
+      @node_3 = instance_double(ASTNode)
+      @node_1 = SequenceNode.new(@node_2, @node_3)
+    end
+
+    it { is_expected.to be_kind_of(ASTNode) }
+
+    describe '#run' do
+      before { build_tree }
+
+      it 'should run each child node in sequence' do
+        expect(@node_2).to receive(:run).with(context).once.ordered
+        expect(@node_3).to receive(:run).with(context).once.ordered
+
+        @node_1.run(context)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+Dir[File.expand_path('../../lib/**/*.rb', __FILE__)].each { |f| require f }
+
+EXEL.logger = nil
+
+EXEL.configure do |config|
+  config[:aws] = {}
+end

metadata

@@ -1,57 +1,141 @@
 --- !ruby/object:Gem::Specification
 name: exel
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.9.0
 platform: ruby
 authors:
 - yroo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-14 00:00:00.000000000 Z
+date: 2015-11-22 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: terminal-notifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: terminal-notifier-guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
 description: A DSL for defining jobs that can be run in a highly scalable manner
 email:
 - dev@yroo.com
@@ -59,8 +143,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
+- ".gitignore"
+- ".rspec"
 - Gemfile
 - Guardfile
 - LICENSE.txt
@@ -68,7 +152,40 @@ files:
 - Rakefile
 - exel.gemspec
 - lib/exel.rb
+- lib/exel/ast_node.rb
+- lib/exel/context.rb
+- lib/exel/deferred_context_value.rb
+- lib/exel/error/job_termination.rb
+- lib/exel/execution_worker.rb
+- lib/exel/handlers/s3_handler.rb
+- lib/exel/handlers/sidekiq_handler.rb
+- lib/exel/instruction.rb
+- lib/exel/instruction_node.rb
+- lib/exel/job.rb
+- lib/exel/logging.rb
+- lib/exel/null_instruction.rb
+- lib/exel/processor_helper.rb
+- lib/exel/processors/async_processor.rb
+- lib/exel/processors/split_processor.rb
+- lib/exel/resource.rb
+- lib/exel/sequence_node.rb
 - lib/exel/version.rb
+- spec/exel/ast_node_spec.rb
+- spec/exel/context_spec.rb
+- spec/exel/deferred_context_value_spec.rb
+- spec/exel/execution_worker_spec.rb
+- spec/exel/handlers/s3_handler_spec.rb
+- spec/exel/handlers/sidekiq_handler_spec.rb
+- spec/exel/instruction_node_spec.rb
+- spec/exel/instruction_spec.rb
+- spec/exel/job_spec.rb
+- spec/exel/logging_spec.rb
+- spec/exel/null_instruction_spec.rb
+- spec/exel/processors/async_processor_spec.rb
+- spec/exel/processors/split_processor_spec.rb
+- spec/exel/resource_spec.rb
+- spec/exel/sequence_node_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/47colborne/exel
 licenses:
 - MIT
@@ -79,18 +196,34 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: EXEL, the Elastic eXEcution Language
-test_files: []
+test_files:
+- spec/exel/ast_node_spec.rb
+- spec/exel/context_spec.rb
+- spec/exel/deferred_context_value_spec.rb
+- spec/exel/execution_worker_spec.rb
+- spec/exel/handlers/s3_handler_spec.rb
+- spec/exel/handlers/sidekiq_handler_spec.rb
+- spec/exel/instruction_node_spec.rb
+- spec/exel/instruction_spec.rb
+- spec/exel/job_spec.rb
+- spec/exel/logging_spec.rb
+- spec/exel/null_instruction_spec.rb
+- spec/exel/processors/async_processor_spec.rb
+- spec/exel/processors/split_processor_spec.rb
+- spec/exel/resource_spec.rb
+- spec/exel/sequence_node_spec.rb
+- spec/spec_helper.rb