exel 0.0.1 → 0.9.0

data/spec/exel/ast_node_spec.rb

@@ -0,0 +1,52 @@
+module EXEL
+  describe ASTNode do
+    let(:context) { {} }
+
+    def build_tree
+      @node_3 = ASTNode.new(instruction)
+      @node_4 = ASTNode.new(instruction)
+      @node_2 = ASTNode.new(instruction, [@node_3, @node_4])
+      @node_5 = ASTNode.new(instruction)
+      @node_1 = ASTNode.new(instruction, [@node_2, @node_5])
+    end
+
+    def instruction
+      instance_double(Instruction, execute: nil)
+    end
+
+    describe '#start' do
+      class TestNode < ASTNode
+      end
+
+      context 'when an JobTermination error bubbles up' do
+        it 'should ensure the process fails silently' do
+          node = TestNode.new(instruction)
+          allow(node).to receive(:run).and_raise(EXEL::Error::JobTermination, 'Error')
+          expect(EXEL.logger).to receive(:error).with('JobTerminationError: Error')
+          expect { node.start(context) }.to_not raise_error
+        end
+      end
+    end
+
+    describe '#run' do
+      class TestNode < ASTNode;
+      end
+
+      it 'should raise an error if not implemented' do
+        expect { TestNode.new(instruction).run(context) }.to raise_error 'EXEL::TestNode does not implement #process'
+      end
+    end
+
+    describe '#add_child' do
+      it 'should add the given node to its children' do
+        root = ASTNode.new(instruction)
+        child_node = ASTNode.new(instruction)
+        child_node2 = ASTNode.new(instruction)
+        root.add_child(child_node)
+        root.add_child(child_node2)
+
+        expect(root.children).to eq([child_node, child_node2])
+      end
+    end
+  end
+end

data/spec/exel/context_spec.rb

@@ -0,0 +1,151 @@
+module EXEL
+  describe Context do
+    subject(:context) { EXEL::Context.new(test1: 'foo', test2: 2) }
+
+    describe '#initialize' do
+      it 'should be able to initialize with a hash' do
+        expect(context.table[:test1]).to eq('foo')
+        expect(context.table[:test2]).to eq(2)
+      end
+    end
+
+    describe '#serialize' do
+      let(:handler) { instance_double(Handlers::S3Handler, upload: nil) }
+
+      before do
+        allow(Handlers::S3Handler).to receive(:new).and_return(handler) #TODO don't stub new
+      end
+
+      it 'should write the serialized context to a file and upload it' do
+        expect(Resource).to receive(:remotize).with(context[:test1]).and_return('remote_value1')
+        expect(Resource).to receive(:remotize).with(context[:test2]).and_return('remote_value2')
+
+        expect(SecureRandom).to receive(:uuid).and_return('uuid')
+
+        expect(handler).to receive(:upload) do |file|
+          expect(file.read).to eq(Marshal.dump(Context.new(test1: 'remote_value1', test2: 'remote_value2')))
+          expect(file.path).to include('uuid')
+          'file_uri'
+        end
+
+        expect(context.serialize).to eq('file_uri')
+      end
+
+      it 'should not mutate the current context' do
+        original_table = context.table.dup
+        context.serialize
+        expect(context.table).to eq(original_table)
+      end
+    end
+
+    describe '.deserialize' do
+      it 'should deserialize a given uri' do
+        uri = 'test_uri'
+        file = StringIO.new(Marshal.dump(context))
+        expect_any_instance_of(Handlers::S3Handler).to receive(:download).with(uri).and_return(file)
+
+        expect(Context.deserialize(uri)).to eq(context)
+
+        expect(file).to be_closed
+      end
+    end
+
+    describe '#[]' do
+      subject(:context) { EXEL::Context.new(key: Resource.remotize('value')) }
+
+      it 'should return localized values' do
+        expect(context[:key]).to eq('value')
+      end
+
+      it 'should store the localized value' do
+        context[:key]
+        expect(context.table[:key]).to eq('value')
+      end
+
+      context 'DeferredContextValue object' do
+        context 'at the top level' do
+          it 'should return the lookup value from the context' do
+            deferred_context_value = DeferredContextValue.new[:key]
+            context[:deferred_value] = deferred_context_value
+            expect(context[:deferred_value]).to eq(context[:key])
+          end
+        end
+        context 'in array' do
+          it 'should return the lookup value from the context' do
+            deferred_context_value = DeferredContextValue.new[:key]
+            context[:array] = [1, 2, deferred_context_value]
+            expect(context[:array]).to eq([1, 2, context[:key]])
+          end
+        end
+
+        context 'in hash' do
+          it 'should return the lookup value from the context' do
+            deferred_context_value = DeferredContextValue.new[:key]
+            context[:hash] = {hash_key: deferred_context_value}
+            expect(context[:hash]).to eq({hash_key: context[:key]})
+          end
+        end
+
+        context 'in nested arrays and hashes' do
+          it 'should lookup a deferred context value in a hash nested in an array' do
+            deferred_context_value = DeferredContextValue.new[:key]
+            context[:nested] = [{}, {hash_key: deferred_context_value}]
+            expect(context[:nested]).to eq([{}, {hash_key: context[:key]}])
+          end
+          it 'should lookup a deferred context value in an array nested in a hash' do
+            deferred_context_value = DeferredContextValue.new[:key]
+            context[:nested] = {hash_key: [1, deferred_context_value]}
+            expect(context[:nested]).to eq({hash_key: [1, context[:key]]})
+          end
+        end
+      end
+    end
+
+    describe '#[]=' do
+      it 'should add the key/value pair to table' do
+        context[:new_key] = 'new_value'
+        expect(context.table[:new_key]).to eq('new_value')
+      end
+    end
+
+    describe '#delete' do
+      it 'should delete the key/value pair from the table' do
+        context[:key] = 'value'
+        context[:key2] = 'value2'
+        context.delete(:key)
+        expect(context.table.keys).to_not include(:key)
+        expect(context.table.keys).to include(:key2)
+      end
+    end
+
+    describe '#merge!' do
+      it 'should merge given keys and values into the context' do
+        context.table[:overwrite] = 'overwrite'
+        context.table[:existing] = 'existing'
+
+        context.merge!(overwrite: 'changed', new: 'new')
+
+        expect(context.table[:overwrite]).to eq('changed')
+        expect(context.table[:existing]).to eq('existing')
+        expect(context.table[:new]).to eq('new')
+      end
+
+      it 'should return itself' do
+        expect(context.merge!(key: 'value')).to eq(context)
+      end
+    end
+
+    describe '#==' do
+      it { is_expected.to_not eq(nil) }
+
+      it { is_expected.to eq(context) }
+
+      it { is_expected.to_not eq(42) }
+
+      it { is_expected.to_not eq(Context.new(other_key: 'value')) }
+
+      it { is_expected.to eq(context.dup) }
+    end
+
+  end
+end

data/spec/exel/deferred_context_value_spec.rb

@@ -0,0 +1,21 @@
+module EXEL
+  describe DeferredContextValue do
+    subject(:deferred_context_value) { DeferredContextValue.new }
+
+    describe '#[]' do
+      it 'should store passed key in the keys attribute' do
+        deferred_context_value[:top_level_key]['sub_key']
+        expect(deferred_context_value.keys).to eq([:top_level_key, 'sub_key'])
+      end
+    end
+
+    describe '#get' do
+      it 'should look up the value of the keys attribute in the passed-in context' do
+        allow(deferred_context_value).to receive(:keys).and_return([:top_level_key, 'sub_key'])
+        value = 'example_value'
+        context = Context.new(top_level_key: {'sub_key' => value})
+        expect(deferred_context_value.get(context)).to eq(value)
+      end
+    end
+  end
+end

data/spec/exel/execution_worker_spec.rb

@@ -0,0 +1,13 @@
+module EXEL
+  describe ExecutionWorker do
+    it 'should run the given block with the deserialized context' do
+      dsl_block = instance_double(SequenceNode)
+      context = Context.new(test1: 'foo', test2: 2, _block: dsl_block)
+      context_uri = 'test uri'
+
+      expect(Context).to receive(:deserialize).with(context_uri).and_return(context)
+      expect(dsl_block).to receive(:start).with(context)
+      ExecutionWorker.new.perform(context_uri)
+    end
+  end
+end

data/spec/exel/handlers/s3_handler_spec.rb

@@ -0,0 +1,49 @@
+module EXEL
+  module Handlers
+    describe S3Handler do
+      subject(:handler) { S3Handler.new }
+
+      describe '#get_object' do
+        before do
+          EXEL.configure { |config| config[:s3_bucket] = 'bucket' }
+        end
+
+        it 'should have the correct bucket and file names' do
+          file_name = 'abc.txt'
+          s3_obj = handler.get_object(file_name)
+          expect(s3_obj.bucket_name).to eq('bucket')
+          expect(s3_obj.key).to eq(file_name)
+        end
+      end
+
+      describe '#upload' do
+        let(:file) { double(path: '/path/to/abc.txt', close: nil) }
+
+        it 'should upload a file to s3' do
+          expect_any_instance_of(Aws::S3::Object).to receive(:upload_file).with(file)
+
+          handler.upload(file)
+        end
+
+        it 'should return the file URI of the uploaded file' do
+          allow_any_instance_of(Aws::S3::Object).to receive(:upload_file).with(file)
+          expect(handler.upload(file)).to eq('s3://abc.txt')
+        end
+      end
+
+      describe '#download' do
+        it 'should download the file from s3' do
+          file = double(:file)
+          s3_object = double(:s3_object)
+
+          expect(handler).to receive(:get_object).with('abc.txt').and_return(s3_object)
+          expect(Tempfile).to receive(:new).with('abc.txt', encoding: Encoding::ASCII_8BIT).and_return(file)
+          expect(s3_object).to receive(:get).with(hash_including(response_target: file)).and_return(file)
+          expect(file).to receive(:set_encoding).with(Encoding::UTF_8)
+
+          expect(handler.download('s3://abc.txt')).to eq(file)
+        end
+      end
+    end
+  end
+end

data/spec/exel/handlers/sidekiq_handler_spec.rb

@@ -0,0 +1,54 @@
+module EXEL
+  module Handlers
+    describe SidekiqHandler do
+      subject(:handler) { SidekiqHandler.new(context) }
+      let(:callback) { instance_double(SequenceNode) }
+      let(:context) { EXEL::Context.new }
+      let(:serialized_context_uri) { 'context_uri' }
+
+      describe '#do_async' do
+        before do
+          allow(context).to receive(:serialize).and_return(serialized_context_uri)
+        end
+
+        it 'should add the callback to the context before serializing it' do
+          expect(context).to receive(:[]=).with(:_block, callback).ordered
+          expect(context).to receive(:serialize).ordered
+          handler.do_async(callback)
+        end
+
+        context 'with queue name' do
+          let(:context) { EXEL::Context.new(queue: 'import_processor') }
+
+          it 'should push the execution worker to the given queue' do
+            expect(Sidekiq::Client).to receive(:push).with('queue' => context[:queue], 'class' => ExecutionWorker, 'args' => [serialized_context_uri])
+            handler.do_async(callback)
+          end
+        end
+
+        context 'without queue name' do
+          it 'should push the execution worker to the default queue' do
+            expect(Sidekiq::Client).to receive(:push).with('class' => ExecutionWorker, 'args' => [serialized_context_uri])
+            handler.do_async(callback)
+          end
+        end
+
+        context 'with retries specified' do
+          let(:context) { EXEL::Context.new(retry: 1) }
+
+          it 'should push the execution worker with a specified number of retries' do
+            expect(Sidekiq::Client).to receive(:push).with('retry' => context[:retry], 'class' => ExecutionWorker, 'args' => [serialized_context_uri])
+            handler.do_async(callback)
+          end
+        end
+
+        context 'with no retries specified' do
+          it 'should push the execution worker with no specified number of retries' do
+            expect(Sidekiq::Client).to receive(:push).with('class' => ExecutionWorker, 'args' => [serialized_context_uri])
+            handler.do_async(callback)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/exel/instruction_node_spec.rb

@@ -0,0 +1,22 @@
+module EXEL
+  describe InstructionNode do
+    let(:context) { {} }
+    let(:instruction) { instance_double(Instruction, execute: nil) }
+    let(:child) { instance_double(ASTNode) }
+    subject(:node) { InstructionNode.new(instruction, [child]) }
+
+    it { is_expected.to be_kind_of(ASTNode) }
+
+    describe '#run' do
+      it 'should only execute the instruction' do
+        expect(instruction).to receive(:execute).with(context).once
+        node.run(context)
+      end
+
+      it 'should not run it`s children' do
+        expect(child).to_not receive(:run)
+        node.run(context)
+      end
+    end
+  end
+end

data/spec/exel/instruction_spec.rb

@@ -0,0 +1,58 @@
+module EXEL
+  describe Instruction do
+    subject(:instruction) { EXEL::Instruction.new('ins_name', processor_class, args) }
+    let(:processor_class) { double(:processor_class, new: processor_instance) }
+    let(:processor_instance) { double(:processor_instance, process: nil) }
+    let(:args) { {arg1: 'arg_value1', arg2: {}} }
+    let(:context) { {context_key: 'context_value'} }
+
+    describe '#run' do
+      it 'should call process on an instance of the processor class' do
+        expect(processor_class).to receive(:new).and_return(processor_instance)
+        expect(processor_instance).to receive(:process)
+
+        instruction.execute(context)
+      end
+
+      it 'should not pass a copy of the context' do
+        allow(processor_class).to receive(:new) do |context_arg|
+          expect(context_arg).to be(context)
+          processor_instance
+        end
+
+        instruction.execute(context)
+      end
+
+      it 'should add args to the context' do
+        instruction.execute(context)
+        expect(context.keys).to include(*args.keys)
+      end
+
+      context 'with args' do
+        it 'should pass the args to the processor' do
+          expect(processor_class).to receive(:new).with(hash_including(args))
+          instruction.execute(context)
+        end
+      end
+
+      context 'without args' do
+        let(:args) { nil }
+
+        it 'should just pass the context to the processor' do
+          expect(processor_class).to receive(:new).with(context)
+          instruction.execute(context)
+        end
+      end
+
+      context 'with a subtree' do
+        let(:subtree) { double(:subtree) }
+        subject(:instruction) { EXEL::Instruction.new('ins_name', processor_class, args, subtree) }
+
+        it 'should pass the subtree to the processor' do
+          expect(processor_instance).to receive(:process).with(subtree)
+          instruction.execute(context)
+        end
+      end
+    end
+  end
+end

data/spec/exel/job_spec.rb

@@ -0,0 +1,215 @@
+module EXEL
+  describe Job do
+    describe '.define' do
+      let(:ast) { instance_double(SequenceNode, run: nil, start: nil) }
+      let(:block) { Proc.new {} }
+
+      after { Job.registry.clear }
+
+      it 'should register job definitions' do
+        Job.define :test_job, &block
+        expect(Job.registry[:test_job]).to eq(block)
+      end
+
+      it 'should raise an exception if a job name is already in use' do
+        Job.define :test_job, &block
+        expect { Job.define :test_job, &block }.to raise_error 'Job :test_job is already defined'
+      end
+    end
+
+    describe '.run' do
+      let(:ast) { instance_double(SequenceNode, run: nil, start: nil) }
+      let(:context) { instance_double(Context) }
+
+      context 'with a string of DSL code' do
+        it 'should parse the code' do
+          dsl_code = 'code'
+          expect(Job::Parser).to receive(:parse).with(dsl_code).and_return(ast)
+          Job.run(dsl_code, context)
+        end
+
+        it 'should run ast returned by the parser' do
+          allow(Job::Parser).to receive(:parse).and_return(ast)
+          expect(ast).to receive(:start).with(context)
+          Job.run('code', context)
+        end
+      end
+
+      context 'with a job name' do
+        context 'of a defined job' do
+          let(:block) { Proc.new {} }
+
+          before do
+            allow(Job).to receive(:registry).and_return(test_job: block)
+          end
+
+          it 'should run the job' do
+            expect(Job::Parser).to receive(:parse).with(block).and_return(ast)
+            expect(ast).to receive(:start).with(context)
+            Job.run(:test_job, context)
+          end
+        end
+
+        context 'of a undefined job' do
+          it 'should return nil' do
+            expect { Job.run(:test_job, context) }.to raise_error('Job "test_job" not found')
+          end
+        end
+      end
+    end
+
+    describe 'mutation of arguments' do
+      class TestProcessor
+        def initialize(context)
+          context[:array] << context[:arg]
+        end
+
+        def process(callback)
+        end
+      end
+
+      it 'should not persist between runs' do
+        Job.define :test do
+          process with: TestProcessor, array: [], arg: context[:value]
+        end
+
+        context = Context.new(value: 1)
+        Job.run(:test, context)
+        expect(context[:array]).to eq([1])
+
+        context = Context.new(value: 2)
+        Job.run(:test, context)
+        expect(context[:array]).to eq([2])
+      end
+    end
+  end
+
+  describe Job::Parser do
+    let(:parser) { Job::Parser.new }
+    let(:ast) { instance_double(SequenceNode, run: nil) }
+
+    describe '#initialize' do
+      it 'should initialize a sequence node' do
+        expect(parser.ast).to be_kind_of(SequenceNode)
+      end
+    end
+
+    describe '.parse' do
+      let(:parser) { instance_double(Job::Parser, ast: ast, instance_eval: nil) }
+
+      before do
+        allow(Job::Parser).to receive(:new).and_return(parser)
+      end
+
+      context 'given DSL code as a proc' do
+        it 'should eval the code as a block' do
+          dsl_proc = Proc.new {}
+          expect(parser).to receive(:instance_eval) do |*_args, &block|
+            expect(block).to eq(dsl_proc)
+          end
+
+          Job::Parser.parse(dsl_proc)
+        end
+      end
+
+      context 'given DSL code as a string' do
+        it 'should eval the code as a string' do
+          dsl_code = 'code'
+          expect(parser).to receive(:instance_eval).with(dsl_code)
+
+          Job::Parser.parse(dsl_code)
+        end
+      end
+
+      it 'should return the parsed AST' do
+        expect(Job::Parser.parse(Proc.new {})).to eq(ast)
+      end
+    end
+
+    describe '#process' do
+      let(:block) { Proc.new {} }
+
+      before do
+        allow(Job::Parser).to receive(:parse).and_return(ast)
+      end
+
+      context 'without a block' do
+        it 'should create a process instruction' do
+          processor_class = double(:processor_class)
+          expect(Instruction).to receive(:new).with('process', processor_class, {arg1: 'arg1_value'}, nil)
+
+          parser.process with: processor_class, arg1: 'arg1_value'
+        end
+
+        it 'should append an instruction node to the AST with no children' do
+          expect(parser.ast).to receive(:add_child) do |node|
+            expect(node).to be_a_kind_of(InstructionNode)
+            expect(node.instruction.name).to eq('process')
+            expect(node.children).to eq([])
+          end
+
+          parser.process with: double(:processor_class)
+        end
+      end
+
+      context 'with a block' do
+        it 'should pass the parsed subtree to the instruction' do
+          processor_class = double(:processor_class)
+          expect(Job::Parser).to receive(:parse).with(block).and_return(ast)
+          expect(Instruction).to receive(:new).with('process', processor_class, {arg1: 'arg1_value'}, ast)
+
+          parser.process with: processor_class, arg1: 'arg1_value', &block
+        end
+
+        it 'should append an instruction node to the AST with the parsed block as its subtree' do
+          expect(parser.ast).to receive(:add_child) do |node|
+            expect(node).to be_a_kind_of(InstructionNode)
+            expect(node.instruction.name).to eq('process')
+            expect(node.children).to eq([ast])
+          end
+
+          parser.process with: double(:processor_class), &block
+        end
+      end
+    end
+
+    [
+        {method: :async, processor: Processors::AsyncProcessor},
+        {method: :split, processor: Processors::SplitProcessor}
+    ].each do |data|
+      describe "##{data[:method]}" do
+        before do
+          allow(Job::Parser).to receive(:parse).and_return(ast)
+        end
+
+        it "should create a #{data[:method]} instruction" do
+          expect(Instruction).to receive(:new).with(data[:method].to_s, data[:processor], {arg1: 'arg1_value'}, ast)
+          parser.send(data[:method], {arg1: 'arg1_value'}) {}
+        end
+
+        it 'should parse the block given' do
+          block = -> {}
+          expect(Job::Parser).to receive(:parse).with(block).and_return(ast)
+
+          parser.send(data[:method], &block)
+        end
+
+        it 'should add parsed subtree and instruction to the AST' do
+          expect(parser.ast).to receive(:add_child) do |node|
+            expect(node).to be_a_kind_of(InstructionNode)
+            expect(node.instruction.name).to eq(data[:method].to_s)
+            expect(node.children).to eq([ast])
+          end
+
+          parser.send(data[:method]) {}
+        end
+      end
+    end
+
+    describe '#context' do
+      it 'should return a DeferredContextValue' do
+        expect(parser.context).to be_a_kind_of(DeferredContextValue)
+      end
+    end
+  end
+end

data/spec/exel/logging_spec.rb

@@ -0,0 +1,36 @@
+module EXEL
+  describe Logging do
+    before { @restore_logger = Logging.logger }
+    after { Logging.logger = @restore_logger }
+
+    describe '.logger=' do
+      it 'should set a logger' do
+        logger = double(:logger)
+        Logging.logger = logger
+        expect(Logging.logger).to be(logger)
+      end
+
+      it 'should set a null logger when nil given' do
+        expect(Logger).to receive(:new).with('/dev/null')
+        Logging.logger = nil
+      end
+    end
+
+    describe '.logger' do
+      before { Logging.instance_variable_set(:@logger, nil) }
+
+      it 'should initialize the logger on first read if not already set' do
+        EXEL.configure do |config|
+          config[:log_level] = :warn
+          config[:log_filename] = 'log.txt'
+        end
+
+        logger = instance_double(Logger)
+        expect(Logger).to receive(:new).with('log.txt').and_return(logger)
+        expect(logger).to receive(:level=).with(Logger::WARN)
+
+        Logging.logger
+      end
+    end
+  end
+end

data/spec/exel/null_instruction_spec.rb

@@ -0,0 +1,5 @@
+module EXEL
+  describe NullInstruction do
+    it { is_expected.to respond_to :execute }
+  end
+end