exel 0.0.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0c87f88078f409a37472e257ec0f70e31d1c19c4
-  data.tar.gz: 9961dbc04ded3f33b5201e92c276846c26887701
+  metadata.gz: c5eccac7b009cd6a4063a36cbb53b70de2efa22b
+  data.tar.gz: f6e7dd493f08d09365158095290b0c319907215e
 SHA512:
-  metadata.gz: b623b2d623fbe39a0ef54e314339bd1836f6cf21a81b6c860da9955f09cf4b617ce3f8dcba5e521763086246fefebbebbe74913d5cb6ac9fe26c68ca538b9e69
-  data.tar.gz: d1cd09aa61c0e415fd4e4d15328062065aadff978b5667d7c0217dd49b2303957e84e6630f6bd3512665f336ea6f3db689a8dc3043dd0b284f9d5f8f58364cbf
+  metadata.gz: 4135007934f8288843da2852a13cb269214725997069a3704b03ad7f4f8f5df103f7824156f1e959de24ca42d6b1a81e72f3fac1eb9b02e174ec9825b66f3b34
+  data.tar.gz: d605c327290de5f1c71f4a4f552b71f7dc8d2469e146a5814e12581188011e6f462a4b08aa8434d2645e92cebaa76720bf0be14656dce1c0feb99c8e5f0e9ed6

data/.gitignore

@@ -21,4 +21,5 @@ tmp
 *.a
 mkmf.log
 .idea
-*.iml
+*.iml
+.ruby-*

data/.rspec

@@ -1 +1,2 @@
---color
+--color
+--require spec_helper

data/exel.gemspec

@@ -18,7 +18,14 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'aws-sdk', '~> 2'
+  spec.add_dependency 'sidekiq', '~> 3'
+
   spec.add_development_dependency 'bundler', '~> 1.6'
-  spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rake', '~> 10'
+  spec.add_development_dependency 'rspec', '~> 3'
+  spec.add_development_dependency 'guard', '~> 2'
+  spec.add_development_dependency 'guard-rspec', '~> 4'
+  spec.add_development_dependency 'terminal-notifier', '~> 1'
+  spec.add_development_dependency 'terminal-notifier-guard', '~> 1'
 end

data/lib/exel/ast_node.rb

@@ -0,0 +1,30 @@
+module EXEL
+  class ASTNode
+    attr_reader :instruction, :children
+
+    def initialize(instruction, children=[])
+      @instruction = instruction
+      @children = children
+    end
+
+    def start(context)
+      fail_silently { run(context) }
+    end
+
+    def run(_context)
+      raise "#{self.class} does not implement #process"
+    end
+
+    def add_child(node)
+      @children << node
+    end
+
+    private
+
+    def fail_silently(&_block)
+      yield if block_given?
+    rescue EXEL::Error::JobTermination => e
+      EXEL.logger.error "JobTerminationError: #{e.message.chomp}"
+    end
+  end
+end

data/lib/exel/context.rb

@@ -0,0 +1,79 @@
+require 'tempfile'
+
+module EXEL
+  class Context
+    attr_reader :table
+
+    def initialize(initial_context={})
+      @table = initial_context
+    end
+
+    def serialize
+      remotized_table = @table.each_with_object({}) { |(key, value), acc| acc[key] = EXEL::Resource.remotize(value) }
+      file = serialize_context(remotized_table)
+      upload(file)
+    end
+
+    def self.deserialize(uri)
+      handler = Handlers::S3Handler.new
+      file = handler.download(uri)
+      context = Marshal.load(file.read)
+      file.close
+      context
+    end
+
+    def [](key)
+      value = EXEL::Resource.localize(@table[key])
+      value = get_deferred(value)
+      @table[key] = value
+      value
+    end
+
+    def []=(key, value)
+      @table[key] = value
+    end
+
+    def merge!(hash)
+      @table.merge!(hash)
+      self
+    end
+
+    def delete(key)
+      @table.delete(key)
+    end
+
+    def ==(other)
+      other.kind_of?(EXEL::Context) && table == other.table
+    end
+
+    private
+
+    def serialize_context(table)
+      file = Tempfile.new(SecureRandom.uuid, encoding: 'ascii-8bit')
+      file.write(Marshal.dump(Context.new(table)))
+      file.rewind
+      file
+    end
+
+    def upload(file)
+      handler = Handlers::S3Handler.new
+      handler.upload(file)
+    end
+
+    def get_deferred(value)
+      if is_deferred?(value)
+        value = value.get(self)
+      elsif value.kind_of?(Array)
+        value.map! { |v| get_deferred(v) }
+      elsif value.kind_of?(Hash)
+        value.each { |k, v| value[k] = get_deferred(v) }
+      end
+
+      value
+    end
+
+    def is_deferred?(value)
+      value.kind_of?(DeferredContextValue)
+    end
+  end
+end

data/lib/exel/deferred_context_value.rb

@@ -0,0 +1,18 @@
+module EXEL
+  class DeferredContextValue
+    attr_reader :keys
+
+    def initialize
+      @keys = []
+    end
+
+    def [](key)
+      keys << key
+      self
+    end
+
+    def get(context)
+      keys.reduce(context) { |acc, key| acc[key] }
+    end
+  end
+end

data/lib/exel/error/job_termination.rb

@@ -0,0 +1,10 @@
+module EXEL
+  module Error
+    # Inherit from Exception rather then StandardError
+    # because rescue => e will only catch StandardError
+    # and allow the Exception to propagate to the root
+    # of the job
+    class JobTermination < Exception
+    end
+  end
+end

data/lib/exel/execution_worker.rb

@@ -0,0 +1,13 @@
+require 'sidekiq'
+
+module EXEL
+  class ExecutionWorker
+    include Sidekiq::Worker
+
+    def perform(context_uri)
+      context = Context.deserialize(context_uri)
+      block = context[:_block]
+      block.start(context)
+    end
+  end
+end

data/lib/exel/handlers/s3_handler.rb

@@ -0,0 +1,43 @@
+require 'aws-sdk-resources'
+require 'tempfile'
+
+module EXEL
+  module Handlers
+    class S3Handler
+      def upload(file)
+        filename = get_filename(file)
+        obj = get_object(filename)
+        obj.upload_file(file)
+        file.close
+
+        "s3://#{filename}"
+      end
+
+      def download(uri)
+        filename = uri.partition('://').last
+        obj = get_object(filename)
+        file = Tempfile.new(filename, encoding: Encoding::ASCII_8BIT)
+        obj.get(response_target: file)
+        file.set_encoding(Encoding::UTF_8)
+        file
+      end
+
+      def get_object(filename)
+        s3 = Aws::S3::Resource.new(
+            credentials: Aws::Credentials.new(
+                EXEL.configuration[:aws][:access_key_id],
+                EXEL.configuration[:aws][:secret_access_key]
+            ),
+            region: 'us-east-1'
+        )
+        s3.bucket(EXEL.configuration[:s3_bucket]).object(filename)
+      end
+
+      private
+
+      def get_filename(file)
+        file.path.split('/').last
+      end
+    end
+  end
+end

data/lib/exel/handlers/sidekiq_handler.rb

@@ -0,0 +1,21 @@
+require 'sidekiq'
+
+module EXEL
+  module Handlers
+    class SidekiqHandler
+      def initialize(context)
+        @context = context
+      end
+
+      def do_async(block)
+        @context[:_block] = block
+
+        push_args = {'class' => ExecutionWorker, 'args' => [@context.serialize]}
+        push_args['queue'] = @context[:queue] if @context[:queue]
+        push_args['retry'] = @context[:retry] if @context[:retry]
+
+        Sidekiq::Client.push(push_args)
+      end
+    end
+  end
+end

data/lib/exel/instruction.rb

@@ -0,0 +1,17 @@
+module EXEL
+  class EXEL::Instruction
+    attr_reader :name
+
+    def initialize(name, processor_class, args, subtree=nil)
+      @name = name
+      @processor_class = processor_class
+      @args = args || {}
+      @subtree = subtree
+    end
+
+    def execute(context)
+      context.merge!(@args)
+      @processor_class.new(context).process(@subtree)
+    end
+  end
+end

data/lib/exel/instruction_node.rb

@@ -0,0 +1,9 @@
+require_relative './ast_node'
+
+module EXEL
+  class InstructionNode < ASTNode
+    def run(context)
+      @instruction.execute(context)
+    end
+  end
+end

data/lib/exel/job.rb

@@ -0,0 +1,74 @@
+module EXEL
+  module Job
+    class << self
+      def define(job_name, &block)
+        raise "Job #{job_name.inspect} is already defined" unless registry[job_name].nil?
+        registry[job_name] = block
+      end
+
+      def registry
+        @registry ||= {}
+      end
+
+      def run(dsl_code_or_name, context = {})
+        context = EXEL::Context.new(context) if context.is_a?(Hash)
+        (ast = parse(dsl_code_or_name)) ? ast.start(context) : raise(%(Job "#{dsl_code_or_name}" not found))
+      end
+
+      private
+
+      def parse(dsl_code_or_name)
+        if dsl_code_or_name.is_a?(Symbol)
+          job = registry[dsl_code_or_name]
+          Parser.parse(job) if job
+        else
+          Parser.parse(dsl_code_or_name)
+        end
+      end
+    end
+
+    class Parser
+      attr_reader :ast
+
+      def initialize
+        @ast = SequenceNode.new
+      end
+
+      def self.parse(dsl_proc_or_code)
+        parser = Parser.new
+        if dsl_proc_or_code.is_a?(::Proc)
+          parser.instance_eval(&dsl_proc_or_code)
+        else
+          parser.instance_eval(dsl_proc_or_code)
+        end
+        parser.ast
+      end
+
+      def process(options, &block)
+        processor_class = options.delete(:with)
+        add_instruction_node('process', processor_class, block, options)
+      end
+
+      def async(options={}, &block)
+        add_instruction_node('async', Processors::AsyncProcessor, block, options)
+      end
+
+      def split(options={}, &block)
+        add_instruction_node('split', Processors::SplitProcessor, block, options)
+      end
+
+      def context
+        DeferredContextValue.new
+      end
+
+      private
+
+      def add_instruction_node(name, processor, block, args={})
+        sub_tree = block.nil? ? nil : Parser.parse(block)
+        instruction = EXEL::Instruction.new(name, processor, args, sub_tree)
+        node = sub_tree.nil? ? InstructionNode.new(instruction) : InstructionNode.new(instruction, [sub_tree])
+        @ast.add_child(node)
+      end
+    end
+  end
+end

data/lib/exel/logging.rb

@@ -0,0 +1,30 @@
+require 'logger'
+
+module EXEL
+  module Logging
+    DEFAULT_LEVEL = :info
+
+    def self.logger
+      @logger || initialize_logger
+    end
+
+    def self.initialize_logger
+      @logger = Logger.new(log_filename)
+      @logger.level = log_level
+      @logger
+    end
+
+    def self.log_filename
+      EXEL.configuration[:log_filename] || '/dev/null'
+    end
+
+    def self.log_level
+      level = EXEL.configuration[:log_level] || DEFAULT_LEVEL
+      Logger.const_get(level.to_s.upcase)
+    end
+
+    def self.logger=(logger)
+      @logger = logger || Logger.new('/dev/null')
+    end
+  end
+end

data/lib/exel/null_instruction.rb

@@ -0,0 +1,6 @@
+module EXEL
+  class NullInstruction
+    def execute(context)
+    end
+  end
+end

data/lib/exel/processor_helper.rb

@@ -0,0 +1,67 @@
+module EXEL
+  module ProcessorHelper
+
+    # Helper Methods
+
+    def tag(*tags)
+      tags.map { |t| "[#{t}]" }.join('')
+    end
+
+    def timestamp
+      Time.now.strftime('%m/%e/%y %H:%M')
+    end
+
+    def file_size_in_mb(file)
+      "#{'%.2f' % (file.size.to_f / 1_024_000).round(2)} MB"
+    end
+
+    # Logging Helpers
+
+    def log_prefix_with(prefix)
+      @log_prefix = (@context[:log_prefix] || '') + prefix
+    end
+
+    def log_prefix
+      @log_prefix
+    end
+
+    def log_info(message)
+      EXEL.logger.info(log(message))
+    end
+
+    def log_error(message)
+      EXEL.logger.error(log(message))
+    end
+
+    def log(message)
+      "#{log_prefix} #{message}"
+    end
+
+    def log_transaction(message = '')
+      transaction_start_time = Time.now.to_f
+      log_info "Started at #{transaction_start_time}"
+      yield(transaction_start_time)
+      transaction_end_time = Time.now.to_f
+      log_info "Finished in #{(transaction_end_time - transaction_start_time).to_i} seconds #{message}"
+    end
+
+    def log_exception(message = '')
+      yield
+    rescue => e
+      log_error "Exception: #{e.message.chomp} #{message}"
+      log_error e.backtrace.join("\n")
+      raise e
+    end
+
+    def log_process(message = '')
+      log_exception(message) { log_transaction(message) { yield } }
+    end
+
+    def ensure_transaction_duration(duration, start_time)
+      elapsed_time = Time.now.to_f - start_time.to_f
+      time_to_sleep = duration.second.to_f - elapsed_time
+      sleep(time_to_sleep) if time_to_sleep > 0
+    end
+
+  end
+end

data/lib/exel/processors/async_processor.rb

@@ -0,0 +1,24 @@
+require_relative '../processor_helper'
+
+module EXEL
+  module Processors
+    class AsyncProcessor
+      include EXEL::ProcessorHelper
+      attr_reader :handler
+
+      def initialize(context)
+        @context = context
+        @handler = EXEL::Handlers::SidekiqHandler.new(context)
+
+        log_prefix_with '[AsyncProcessor]'
+      end
+
+      def process(block)
+        log_process do
+          @handler.do_async(block)
+          log_info 'call to async completed'
+        end
+      end
+    end
+  end
+end

data/lib/exel/processors/split_processor.rb

@@ -0,0 +1,85 @@
+require 'csv'
+require 'tempfile'
+require_relative '../processor_helper'
+
+module EXEL
+  module Processors
+    class SplitProcessor
+      include EXEL::ProcessorHelper
+
+      attr_accessor :chunk_size, :file_name, :block
+
+      DEFAULT_CHUNK_SIZE = 1000
+
+      def initialize(context)
+        @chunk_size = DEFAULT_CHUNK_SIZE
+        @buffer = []
+        @tempfile_count = 0
+        @context = context
+
+        @file = context[:resource]
+        @file_name = filename(@file)
+        @csv_options = context[:csv_options] || {col_sep: ','}
+
+        log_prefix_with '[SplitProcessor]'
+      end
+
+      def process(callback)
+        log_process do
+          begin
+            CSV.foreach(@file.path, @csv_options) do |line|
+              process_line(line, callback)
+            end
+          rescue CSV::MalformedCSVError => e
+            log_error "CSV::MalformedCSVError => #{e.message}"
+          end
+          process_line(:eof, callback)
+          File.delete(@file.path)
+        end
+      end
+
+      def process_line(line, callback)
+        if line == :eof
+          flush_buffer callback
+        else
+          @buffer << CSV.generate_line(line)
+
+          flush_buffer callback if buffer_full?
+        end
+      end
+
+      def generate_chunk(content)
+        @tempfile_count += 1
+        chunk = Tempfile.new([chunk_filename, '.csv'])
+        chunk.write(content)
+        chunk.rewind
+
+        log_info "Generated chunk # #{@tempfile_count} for file #{@file_name} in #{chunk.path}"
+        chunk
+      end
+
+      def chunk_filename
+        "#{@file_name}_#{@tempfile_count}_"
+      end
+
+      def filename(file)
+        file_name_with_extension = file.path.split('/').last
+        file_name_with_extension.split('.').first
+      end
+
+      private
+
+      def flush_buffer(callback)
+        unless @buffer.empty?
+          file = generate_chunk(@buffer.join(''))
+          callback.run(@context.merge!(resource: file))
+        end
+        @buffer = []
+      end
+
+      def buffer_full?
+        @buffer.size == @chunk_size
+      end
+    end
+  end
+end

data/lib/exel/resource.rb

@@ -0,0 +1,35 @@
+module EXEL
+  class Resource
+    def self.remotize(value)
+      file?(value) ? upload(value) : value
+    end
+
+    def self.localize(value)
+      serialized?(value) ? deserialize_file(value) : value
+    end
+
+    class << self
+      private
+
+      def file?(value)
+        value.is_a?(File) || value.is_a?(Tempfile)
+      end
+
+      def serialized?(value)
+        value =~ %r{^s3://}
+      end
+
+      def deserialize_file(uri)
+        download(uri)
+      end
+
+      def download(uri)
+        Handlers::S3Handler.new.download(uri)
+      end
+
+      def upload(file)
+        Handlers::S3Handler.new.upload(file)
+      end
+    end
+  end
+end

data/lib/exel/sequence_node.rb

@@ -0,0 +1,14 @@
+require_relative './ast_node'
+
+module EXEL
+  class SequenceNode < ASTNode
+    def initialize(*children)
+      @instruction = NullInstruction.new
+      @children = children
+    end
+
+    def run(context)
+      @children.each { |child| child.run(context) }
+    end
+  end
+end

data/lib/exel/version.rb

@@ -1,3 +1,3 @@
 module EXEL
-  VERSION = '0.0.1'
+  VERSION = '0.9.0'
 end

data/lib/exel.rb

@@ -1,5 +1,23 @@
 require 'exel/version'
+require 'exel/logging'
 
 module EXEL
-  # Your code goes here...
+  def self.logger
+    EXEL::Logging.logger
+  end
+
+  def self.logger=(logger)
+    EXEL::Logging.logger = logger
+  end
+
+  def self.configuration
+    @config ||= {}
+  end
+
+  def self.configure
+    yield configuration
+  end
+
+  root = File.expand_path('../..', __FILE__)
+  Dir[File.join(root, 'lib/exel/**/*.rb')].each { |file| require file }
 end