exel 0.9.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c5eccac7b009cd6a4063a36cbb53b70de2efa22b
-  data.tar.gz: f6e7dd493f08d09365158095290b0c319907215e
+  metadata.gz: 07a61d1ecdfd5d0d957caa0dd064631be8c3e546
+  data.tar.gz: 364c194ce4cb48938f1cc863b6bdf0a5b7e74e47
 SHA512:
-  metadata.gz: 4135007934f8288843da2852a13cb269214725997069a3704b03ad7f4f8f5df103f7824156f1e959de24ca42d6b1a81e72f3fac1eb9b02e174ec9825b66f3b34
-  data.tar.gz: d605c327290de5f1c71f4a4f552b71f7dc8d2469e146a5814e12581188011e6f462a4b08aa8434d2645e92cebaa76720bf0be14656dce1c0feb99c8e5f0e9ed6
+  metadata.gz: d004472bd8b4b3c6e496377c89cb261abcaa59fad5284009ecca65bee86dff428545af4ab27cd47ca74becbcdc9bb0a3d81962a67cf81f6437b5195a990cdf5f
+  data.tar.gz: debffc94d7f7fd53064f2b8f5317a68f3cbcaa4e961d19c6c2af630a31c39d79b4b42c111947e361a3f5a838853ff29d4a9814fb0892609da994f4c433517b85

data/.rubocop.yml

@@ -0,0 +1,17 @@
+require:
+  - rubocop-rspec
+  - rubocop/rspec/focused
+
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  DisplayStyleGuide: true
+
+Metrics/LineLength:
+  Max: 120
+
+Style/SpaceInsideHashLiteralBraces:
+  EnforcedStyle: no_space
+
+RSpec/DescribedClass:
+  Enabled: false

data/.rubocop_todo.yml

@@ -0,0 +1,43 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2015-12-13 18:06:06 -0500 using RuboCop version 0.35.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 11
+
+# Offense count: 2
+# Configuration parameters: CountComments.
+Metrics/ModuleLength:
+  Max: 166
+
+# Offense count: 65
+# Configuration parameters: CustomTransform, IgnoredWords.
+RSpec/ExampleWording:
+  Exclude:
+    - 'spec/exel/ast_node_spec.rb'
+    - 'spec/exel/context_spec.rb'
+    - 'spec/exel/deferred_context_value_spec.rb'
+    - 'spec/exel/handlers/s3_handler_spec.rb'
+    - 'spec/exel/instruction_node_spec.rb'
+    - 'spec/exel/instruction_spec.rb'
+    - 'spec/exel/job_spec.rb'
+    - 'spec/exel/logging_spec.rb'
+    - 'spec/exel/processors/split_processor_spec.rb'
+    - 'spec/exel/resource_spec.rb'
+    - 'spec/exel/sequence_node_spec.rb'
+
+# Offense count: 1
+RSpec/InstanceVariable:
+  Exclude:
+    - 'spec/exel/logging_spec.rb'
+
+# Offense count: 18
+# Configuration parameters: Exclude.
+Style/Documentation:
+  Enabled: false

data/Guardfile

@@ -1,14 +1,21 @@
-guard :rspec, cmd: 'bundle exec rspec' do
-  require 'guard/rspec/dsl'
-  dsl = Guard::RSpec::Dsl.new(self)
+group :rspec_rubocop, halt_on_fail: true do
+  guard :rspec, cmd: 'bundle exec rspec' do
+    require 'guard/rspec/dsl'
+    dsl = Guard::RSpec::Dsl.new(self)
 
-  # RSpec files
-  rspec = dsl.rspec
-  watch(rspec.spec_helper) { rspec.spec_dir }
-  watch(rspec.spec_support) { rspec.spec_dir }
-  watch(rspec.spec_files)
+    # RSpec files
+    rspec = dsl.rspec
+    watch(rspec.spec_helper) { rspec.spec_dir }
+    watch(rspec.spec_support) { rspec.spec_dir }
+    watch(rspec.spec_files)
 
-  # Ruby files
-  ruby = dsl.ruby
-  dsl.watch_spec_files_for(ruby.lib_files)
+    # Ruby files
+    ruby = dsl.ruby
+    dsl.watch_spec_files_for(ruby.lib_files)
+  end
+
+  guard :rubocop do
+    watch(/.+\.rb$/)
+    watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
+  end
 end

data/README.md

@@ -1,4 +1,7 @@
 # EXEL
+[![Gem Version](https://badge.fury.io/rb/exel.svg)](https://badge.fury.io/rb/exel)
+[![Code Climate](https://codeclimate.com/github/47colborne/exel/badges/gpa.svg)](https://codeclimate.com/github/47colborne/exel)
+[![Build Status](https://snap-ci.com/47colborne/exel/branch/master/build_image)](https://snap-ci.com/47colborne/exel/branch/master)
 
 TODO: Write a gem description
 

data/Rakefile

@@ -1,2 +1 @@
 require 'bundler/gem_tasks'
-

data/exel.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.version       = EXEL::VERSION
   spec.authors       = ['yroo']
   spec.email         = ['dev@yroo.com']
-  spec.summary       = %q{EXEL, the Elastic eXEcution Language}
-  spec.description   = %q{A DSL for defining jobs that can be run in a highly scalable manner}
+  spec.summary       = 'EXEL, the Elastic eXEcution Language'
+  spec.description   = 'A DSL for defining jobs that can be run in a highly scalable manner'
   spec.homepage      = 'https://github.com/47colborne/exel'
   spec.license       = 'MIT'
 
@@ -18,14 +18,15 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'aws-sdk', '~> 2'
-  spec.add_dependency 'sidekiq', '~> 3'
-
   spec.add_development_dependency 'bundler', '~> 1.6'
   spec.add_development_dependency 'rake', '~> 10'
   spec.add_development_dependency 'rspec', '~> 3'
   spec.add_development_dependency 'guard', '~> 2'
   spec.add_development_dependency 'guard-rspec', '~> 4'
+  spec.add_development_dependency 'guard-rubocop', '~> 1'
   spec.add_development_dependency 'terminal-notifier', '~> 1'
   spec.add_development_dependency 'terminal-notifier-guard', '~> 1'
+  spec.add_development_dependency 'rubocop', '~> 0'
+  spec.add_development_dependency 'rubocop-rspec', '~> 1'
+  spec.add_development_dependency 'rubocop-rspec-focused', '~> 0'
 end

data/lib/exel/ast_node.rb

@@ -2,7 +2,7 @@ module EXEL
   class ASTNode
     attr_reader :instruction, :children
 
-    def initialize(instruction, children=[])
+    def initialize(instruction, children = [])
       @instruction = instruction
       @children = children
     end
@@ -12,7 +12,7 @@ module EXEL
     end
 
     def run(_context)
-      raise "#{self.class} does not implement #process"
+      fail "#{self.class} does not implement #process"
     end
 
     def add_child(node)

data/lib/exel/context.rb

@@ -4,26 +4,24 @@ module EXEL
   class Context
     attr_reader :table
 
-    def initialize(initial_context={})
+    def initialize(initial_context = {})
       @table = initial_context
     end
 
     def serialize
-      remotized_table = @table.each_with_object({}) { |(key, value), acc| acc[key] = EXEL::Resource.remotize(value) }
-      file = serialize_context(remotized_table)
-      upload(file)
+      remotized_table = @table.each_with_object({}) { |(key, value), acc| acc[key] = EXEL::Value.remotize(value) }
+      EXEL::Value.remotize(serialized_context(remotized_table))
     end
 
     def self.deserialize(uri)
-      handler = Handlers::S3Handler.new
-      file = handler.download(uri)
+      file = EXEL::Value.localize(uri)
       context = Marshal.load(file.read)
       file.close
       context
     end
 
     def [](key)
-      value = EXEL::Resource.localize(@table[key])
+      value = EXEL::Value.localize(@table[key])
       value = get_deferred(value)
       @table[key] = value
       value
@@ -43,37 +41,36 @@ module EXEL
     end
 
     def ==(other)
-      other.kind_of?(EXEL::Context) && table == other.table
+      other.is_a?(EXEL::Context) && table == other.table
+    end
+
+    def include?(values)
+      @table.merge(values) == @table
     end
 
     private
 
-    def serialize_context(table)
+    def serialized_context(table)
       file = Tempfile.new(SecureRandom.uuid, encoding: 'ascii-8bit')
       file.write(Marshal.dump(Context.new(table)))
       file.rewind
       file
     end
 
-    def upload(file)
-      handler = Handlers::S3Handler.new
-      handler.upload(file)
-    end
-
     def get_deferred(value)
-      if is_deferred?(value)
+      if deferred?(value)
         value = value.get(self)
-      elsif value.kind_of?(Array)
+      elsif value.is_a?(Array)
         value.map! { |v| get_deferred(v) }
-      elsif value.kind_of?(Hash)
+      elsif value.is_a?(Hash)
         value.each { |k, v| value[k] = get_deferred(v) }
       end
 
       value
     end
 
-    def is_deferred?(value)
-      value.kind_of?(DeferredContextValue)
+    def deferred?(value)
+      value.is_a?(DeferredContextValue)
     end
   end
 end

data/lib/exel/deferred_context_value.rb

@@ -12,7 +12,7 @@ module EXEL
     end
 
     def get(context)
-      keys.reduce(context) { |acc, key| acc[key] }
+      keys.reduce(context) { |a, e| a[e] }
     end
   end
-end
+end

data/lib/exel/instruction.rb

@@ -1,8 +1,8 @@
 module EXEL
-  class EXEL::Instruction
+  class Instruction
     attr_reader :name
 
-    def initialize(name, processor_class, args, subtree=nil)
+    def initialize(name, processor_class, args, subtree = nil)
       @name = name
       @processor_class = processor_class
       @args = args || {}

data/lib/exel/job.rb

@@ -2,7 +2,7 @@ module EXEL
   module Job
     class << self
       def define(job_name, &block)
-        raise "Job #{job_name.inspect} is already defined" unless registry[job_name].nil?
+        fail "Job #{job_name.inspect} is already defined" unless registry[job_name].nil?
         registry[job_name] = block
       end
 
@@ -12,7 +12,7 @@ module EXEL
 
       def run(dsl_code_or_name, context = {})
         context = EXEL::Context.new(context) if context.is_a?(Hash)
-        (ast = parse(dsl_code_or_name)) ? ast.start(context) : raise(%(Job "#{dsl_code_or_name}" not found))
+        (ast = parse(dsl_code_or_name)) ? ast.start(context) : fail(%(Job "#{dsl_code_or_name}" not found))
       end
 
       private
@@ -49,11 +49,11 @@ module EXEL
         add_instruction_node('process', processor_class, block, options)
       end
 
-      def async(options={}, &block)
+      def async(options = {}, &block)
         add_instruction_node('async', Processors::AsyncProcessor, block, options)
       end
 
-      def split(options={}, &block)
+      def split(options = {}, &block)
         add_instruction_node('split', Processors::SplitProcessor, block, options)
       end
 
@@ -63,7 +63,7 @@ module EXEL
 
       private
 
-      def add_instruction_node(name, processor, block, args={})
+      def add_instruction_node(name, processor, block, args = {})
         sub_tree = block.nil? ? nil : Parser.parse(block)
         instruction = EXEL::Instruction.new(name, processor, args, sub_tree)
         node = sub_tree.nil? ? InstructionNode.new(instruction) : InstructionNode.new(instruction, [sub_tree])

data/lib/exel/logging.rb

@@ -15,11 +15,11 @@ module EXEL
     end
 
     def self.log_filename
-      EXEL.configuration[:log_filename] || '/dev/null'
+      EXEL.configuration.log_filename || '/dev/null'
     end
 
     def self.log_level
-      level = EXEL.configuration[:log_level] || DEFAULT_LEVEL
+      level = EXEL.configuration.log_level || DEFAULT_LEVEL
       Logger.const_get(level.to_s.upcase)
     end
 
@@ -27,4 +27,4 @@ module EXEL
       @logger = logger || Logger.new('/dev/null')
     end
   end
-end
+end

data/lib/exel/null_instruction.rb

@@ -3,4 +3,4 @@ module EXEL
     def execute(context)
     end
   end
-end
+end

data/lib/exel/processor_helper.rb

@@ -1,6 +1,5 @@
 module EXEL
   module ProcessorHelper
-
     # Helper Methods
 
     def tag(*tags)
@@ -12,7 +11,7 @@ module EXEL
     end
 
     def file_size_in_mb(file)
-      "#{'%.2f' % (file.size.to_f / 1_024_000).round(2)} MB"
+      format('%.2f MB', file.size.to_f / 1_024_000)
     end
 
     # Logging Helpers
@@ -38,10 +37,10 @@ module EXEL
     end
 
     def log_transaction(message = '')
-      transaction_start_time = Time.now.to_f
+      transaction_start_time = Time.now
       log_info "Started at #{transaction_start_time}"
       yield(transaction_start_time)
-      transaction_end_time = Time.now.to_f
+      transaction_end_time = Time.now
       log_info "Finished in #{(transaction_end_time - transaction_start_time).to_i} seconds #{message}"
     end
 
@@ -62,6 +61,5 @@ module EXEL
       time_to_sleep = duration.second.to_f - elapsed_time
       sleep(time_to_sleep) if time_to_sleep > 0
     end
-
   end
 end

data/lib/exel/processors/async_processor.rb

@@ -4,18 +4,18 @@ module EXEL
   module Processors
     class AsyncProcessor
       include EXEL::ProcessorHelper
-      attr_reader :handler
+      attr_reader :provider
 
       def initialize(context)
         @context = context
-        @handler = EXEL::Handlers::SidekiqHandler.new(context)
+        @provider = EXEL.async_provider.new(context)
 
         log_prefix_with '[AsyncProcessor]'
       end
 
       def process(block)
         log_process do
-          @handler.do_async(block)
+          @provider.do_async(block)
           log_info 'call to async completed'
         end
       end

data/lib/exel/providers/local_file_provider.rb

@@ -0,0 +1,18 @@
+module EXEL
+  module Providers
+    class LocalFileProvider
+      def upload(file)
+        "file://#{file.path}"
+      end
+
+      def download(uri)
+        fail 'URI must begin with "file://"' unless uri.start_with? 'file://'
+        File.open(uri.split('file://').last)
+      end
+
+      def self.remote?(uri)
+        uri =~ %r{file://}
+      end
+    end
+  end
+end

data/lib/exel/providers/threaded_async_provider.rb

@@ -0,0 +1,13 @@
+module EXEL
+  module Providers
+    class ThreadedAsyncProvider
+      def initialize(context)
+        @context = context
+      end
+
+      def do_async(block)
+        Thread.new { block.start(@context) }
+      end
+    end
+  end
+end

data/lib/exel/{resource.rb → value.rb}

@@ -1,11 +1,11 @@
 module EXEL
-  class Resource
+  module Value
     def self.remotize(value)
       file?(value) ? upload(value) : value
     end
 
     def self.localize(value)
-      serialized?(value) ? deserialize_file(value) : value
+      remote?(value) ? download(value) : value
     end
 
     class << self
@@ -15,20 +15,16 @@ module EXEL
         value.is_a?(File) || value.is_a?(Tempfile)
       end
 
-      def serialized?(value)
-        value =~ %r{^s3://}
+      def upload(file)
+        EXEL.remote_provider.new.upload(file)
       end
 
-      def deserialize_file(uri)
-        download(uri)
+      def remote?(value)
+        EXEL.remote_provider.remote?(value)
       end
 
       def download(uri)
-        Handlers::S3Handler.new.download(uri)
-      end
-
-      def upload(file)
-        Handlers::S3Handler.new.upload(file)
+        EXEL.remote_provider.new.download(uri)
       end
     end
   end

data/lib/exel/version.rb

@@ -1,3 +1,3 @@
 module EXEL
-  VERSION = '0.9.0'
+  VERSION = '1.0.0'
 end

data/lib/exel.rb

@@ -1,5 +1,6 @@
 require 'exel/version'
 require 'exel/logging'
+require 'ostruct'
 
 module EXEL
   def self.logger
@@ -11,13 +12,21 @@ module EXEL
   end
 
   def self.configuration
-    @config ||= {}
+    @config ||= OpenStruct.new
   end
 
   def self.configure
     yield configuration
   end
 
+  def self.async_provider
+    configuration.async_provider || Providers::ThreadedAsyncProvider
+  end
+
+  def self.remote_provider
+    configuration.remote_provider || Providers::LocalFileProvider
+  end
+
   root = File.expand_path('../..', __FILE__)
   Dir[File.join(root, 'lib/exel/**/*.rb')].each { |file| require file }
 end

data/spec/exel/ast_node_spec.rb

@@ -1,23 +1,14 @@
 module EXEL
   describe ASTNode do
-    let(:context) { {} }
-
-    def build_tree
-      @node_3 = ASTNode.new(instruction)
-      @node_4 = ASTNode.new(instruction)
-      @node_2 = ASTNode.new(instruction, [@node_3, @node_4])
-      @node_5 = ASTNode.new(instruction)
-      @node_1 = ASTNode.new(instruction, [@node_2, @node_5])
-    end
+    let(:context) { instance_double(EXEL::Context) }
 
     def instruction
       instance_double(Instruction, execute: nil)
     end
 
-    describe '#start' do
-      class TestNode < ASTNode
-      end
+    TestNode = Class.new(ASTNode)
 
+    describe '#start' do
       context 'when an JobTermination error bubbles up' do
         it 'should ensure the process fails silently' do
           node = TestNode.new(instruction)
@@ -29,9 +20,6 @@ module EXEL
     end
 
     describe '#run' do
-      class TestNode < ASTNode;
-      end
-
       it 'should raise an error if not implemented' do
         expect { TestNode.new(instruction).run(context) }.to raise_error 'EXEL::TestNode does not implement #process'
       end