metacrunch 3.1.4 → 4.0.1

data/lib/metacrunch/job.rb

@@ -3,140 +3,121 @@ module Metacrunch
     require_relative "job/dsl"
     require_relative "job/buffer"
 
-    attr_reader :builder, :args
+    attr_reader :dsl
 
     class << self
-      def define(file_content = nil, filename: nil, args: nil, number_of_processes: 1, process_index: 0, &block)
-        self.new(file_content, filename: filename, args: args, number_of_processes: number_of_processes, process_index: process_index, &block)
+      def define(file_content = nil, &block)
+        self.new(file_content, &block)
       end
     end
 
-    def initialize(file_content = nil, filename: nil, args: nil, number_of_processes: 1, process_index: 0, &block)
-      @builder = Dsl.new(self)
-      @args = args
-      @number_of_processes = number_of_processes
-      @process_index = process_index
+    def initialize(file_content = nil, &block)
+      @dsl = Dsl.new(self)
 
       if file_content
-        @builder.instance_eval(file_content, filename || "")
+        @dsl.instance_eval(file_content, "Check your metacrunch Job at Line")
       elsif block_given?
-        @builder.instance_eval(&block)
+        @dsl.instance_eval(&block)
       end
     end
 
-    def sources
-      @sources ||= []
+    def source
+      @source
     end
 
-    def add_source(source)
+    def source=(source)
       ensure_source!(source)
-      sources << source
+      @source = source
     end
 
-    def destinations
-      @destinations ||= []
+    def destination
+      @destination
     end
 
-    def add_destination(destination)
+    def destination=(destination)
       ensure_destination!(destination)
-      destinations << destination
+      @destination = destination
     end
 
-    def pre_processes
-      @pre_processes ||= []
+    def pre_process
+      @pre_process
     end
 
-    def add_pre_process(callable = nil, &block)
-      add_callable_or_block(pre_processes, callable, &block)
+    def pre_process=(callable)
+      ensure_callable!(callable)
+      @pre_process = callable
     end
 
-    def post_processes
-      @post_processes ||= []
+    def post_process
+      @post_process
     end
 
-    def add_post_process(callable = nil, &block)
-      add_callable_or_block(post_processes, callable, &block)
+    def post_process=(callable)
+      ensure_callable!(callable)
+      @post_process = callable
     end
 
     def transformations
       @transformations ||= []
     end
 
-    def add_transformation(callable = nil, &block)
-      add_callable_or_block(transformations, callable, &block)
-    end
+    def add_transformation(callable, buffer_size: nil)
+      ensure_callable!(callable)
+
+      if buffer_size && buffer_size.to_i > 0
+        transformations << Metacrunch::Job::Buffer.new(buffer_size)
+      end
 
-    def add_transformation_buffer(size)
-      transformations << Metacrunch::Job::Buffer.new(size)
+      transformations << callable
     end
 
     def run
-      run_pre_processes
-      run_transformations
-      run_post_processes
-      self
-    end
+      run_pre_process
 
-  private
+      if source
+        # Run transformation for each data object available in source
+        source.each do |data|
+          data = run_transformations(data)
+          write_destination(data)
+        end
 
-    def add_callable_or_block(array, callable, &block)
-      if block_given?
-        array << block
-      elsif callable
-        ensure_callable!(callable)
-        array << callable
+        # Run all transformations a last time to flush existing buffers
+        data = run_transformations(data = nil, flush_buffers: true)
+        write_destination(data)
+
+        # Close destination
+        destination.close if destination
       end
-    end
 
-    def ensure_callable!(object)
-      raise ArgumentError, "#{object} does't respond to #call." unless object.respond_to?(:call)
+      run_post_process
+
+      self
     end
 
+  private
+
     def ensure_source!(object)
-      raise ArgumentError, "#{object} does't respond to #each." unless object.respond_to?(:each)
+      raise ArgumentError, "#{object} doesn't respond to #each." unless object.respond_to?(:each)
     end
 
     def ensure_destination!(object)
-      raise ArgumentError, "#{object} does't respond to #write." unless object.respond_to?(:write)
-      raise ArgumentError, "#{object} does't respond to #close." unless object.respond_to?(:close)
+      raise ArgumentError, "#{object} doesn't respond to #write." unless object.respond_to?(:write)
+      raise ArgumentError, "#{object} doesn't respond to #close." unless object.respond_to?(:close)
     end
 
-    def run_pre_processes
-      pre_processes.each(&:call)
+    def ensure_callable!(object)
+      raise ArgumentError, "#{object} doesn't respond to #call." unless object.respond_to?(:call)
     end
 
-    def run_post_processes
-      post_processes.each(&:call)
+    def run_pre_process
+      pre_process.call if pre_process
     end
 
-    def run_transformations
-      sources.each do |source|
-        # Setup parallel processing
-        if @number_of_processes > 1
-          if source.class.included_modules.include?(Metacrunch::ParallelProcessableReader)
-            source.set_parallel_process_options(
-              number_of_processes: @number_of_processes,
-              process_index: @process_index
-            )
-          else
-            raise RuntimeError, "source does't support parallel processing"
-          end
-        end
-
-        # sources are expected to respond to `each`
-        source.each do |data|
-          run_transformations_and_write_destinations(data)
-        end
-
-        # Run all transformations a last time to flush possible buffers
-        run_transformations_and_write_destinations(nil, flush_buffers: true)
-      end
-
-      # destination implementations are expected to respond to `close`
-      destinations.each(&:close)
+    def run_post_process
+      post_process.call if post_process
     end
 
-    def run_transformations_and_write_destinations(data, flush_buffers: false)
+    def run_transformations(data, flush_buffers: false)
       transformations.each do |transformation|
         if transformation.is_a?(Buffer)
           if data.present?
@@ -150,11 +131,11 @@ module Metacrunch
         end
       end
 
-      if data.present?
-        destinations.each do |destination|
-          destination.write(data) # destinations are expected to respond to `write(data)`
-        end
-      end
+      data
+    end
+
+    def write_destination(data)
+      destination.write(data) if destination
     end
 
   end

data/lib/metacrunch/job/dsl.rb

@@ -1,38 +1,34 @@
 module Metacrunch
   class Job::Dsl
-    require_relative "dsl/option_support"
+    require_relative "dsl/options"
 
     def initialize(job)
       @_job = job
     end
 
     def source(source)
-      @_job.add_source(source)
+      @_job.source = source
     end
 
     def destination(destination)
-      @_job.add_destination(destination)
+      @_job.destination = destination
     end
 
-    def pre_process(callable = nil, &block)
-      @_job.add_pre_process(callable, &block)
+    def pre_process(callable)
+      @_job.pre_process = callable
     end
 
-    def post_process(callable = nil, &block)
-      @_job.add_post_process(callable, &block)
+    def post_process(callable)
+      @_job.post_process = callable
     end
 
-    def transformation_buffer(size)
-      @_job.add_transformation_buffer(size)
-    end
-
-    def transformation(callable = nil, &block)
-      @_job.add_transformation(callable, &block)
+    def transformation(callable, buffer_size: nil)
+      @_job.add_transformation(callable, buffer_size: buffer_size)
     end
 
     def options(require_args: false, &block)
       if block_given?
-        @_options = OptionSupport.new.register_options(@_job.args, require_args: require_args, &block)
+        @_options = Options.new(require_args: require_args, &block).options
       else
         @_options ||= {}
       end

data/lib/metacrunch/job/dsl/options.rb

@@ -0,0 +1,80 @@
+module Metacrunch
+  class Job::Dsl::Options
+    require_relative "options/dsl"
+
+    attr_reader :options
+
+    def initialize(argv = ARGV, require_args: false, &block)
+      @options = {}
+      dsl.instance_eval(&block)
+
+      dsl.options.each do |key, opt_def|
+        # Set default value
+        @options[key] = opt_def[:default]
+
+        # Register with OptionParser
+        if opt_def[:args].present?
+          option = parser.define(*opt_def[:args]) { |value| @options[key] = value }
+
+          option.desc << "REQUIRED" if opt_def[:required]
+          option.desc << "DEFAULT: #{opt_def[:default]}" if opt_def[:default].present?
+
+          parser_options[key] = option
+        end
+      end
+
+      # Finally parse CLI options with OptionParser
+      parser.parse!(argv)
+
+      # Make sure required options are present
+      ensure_required_options!(@options)
+
+      # Make sure args are present if required
+      ensure_required_args!(argv) if require_args
+    rescue OptionParser::ParseError => e
+      error(e.message)
+    end
+
+  private
+
+    def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.banner = <<-BANNER.strip_heredoc
+          #{ColorizedString["Job options:"].bold}
+        BANNER
+      end
+    end
+
+    def dsl
+      @dsl ||= Dsl.new
+    end
+
+    def parser_options
+      @parser_options ||= {}
+    end
+
+    def error(message)
+      puts ColorizedString["Error: #{message}\n"].red.bold
+      puts parser.help
+      exit(1)
+    end
+
+    def ensure_required_options!(options)
+      dsl.options.each do |key, opt_def|
+        if opt_def[:required] && options[key].blank?
+          long_option = parser_options[key].long.try(:[], 0)
+          short_option = parser_options[key].short.try(:[], 0)
+
+          error("Required job option `#{long_option || short_option}` missing.")
+        end
+      end
+    end
+
+    def ensure_required_args!(argv)
+      if argv.blank?
+        error("Required ARGS are missing.")
+      end
+    end
+
+  end
+end

data/lib/metacrunch/job/dsl/options/dsl.rb

@@ -0,0 +1,21 @@
+module Metacrunch
+  class Job::Dsl::Options::Dsl
+
+    def add(name, *args, default: nil, required: false)
+      if default && required
+        raise ArgumentError, "You can't use `default` and `required` option at the same time."
+      end
+
+      options[name.to_sym] = {
+        args: args,
+        default: default,
+        required: required
+      }
+    end
+
+    def options
+      @options ||= {}
+    end
+
+  end
+end

data/lib/metacrunch/version.rb

@@ -1,3 +1,3 @@
 module Metacrunch
-  VERSION = "3.1.4"
+  VERSION = "4.0.1"
 end

data/metacrunch.gemspec

@@ -7,7 +7,6 @@ Gem::Specification.new do |spec|
   spec.name          = "metacrunch"
   spec.version       = Metacrunch::VERSION
   spec.authors       = ["René Sprotte", "Michael Sievers", "Marcel Otto"]
-  spec.email         = "r.sprotte@ub.uni-paderborn.de"
   spec.summary       = %q{Data processing and ETL toolkit for Ruby}
   spec.homepage      = "http://github.com/ubpb/metacrunch"
   spec.license       = "MIT"
@@ -17,9 +16,6 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activesupport", ">= 4.2", "< 5.1"
-  spec.add_dependency "colorize",      ">= 0.8"
-  spec.add_dependency "parallel",      "~> 1.9"
-  spec.add_dependency "sequel",        "~> 4.33"
-  spec.add_dependency "redis",         "~> 3.3"
+  spec.add_dependency "activesupport", ">= 5.1.0"
+  spec.add_dependency "colorize",      ">= 0.8.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch
 version: !ruby/object:Gem::Version
-  version: 3.1.4
+  version: 4.0.1
 platform: ruby
 authors:
 - René Sprotte
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-02-09 00:00:00.000000000 Z
+date: 2017-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -18,78 +18,30 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5.1'
+        version: 5.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5.1'
+        version: 5.1.0
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 0.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.8'
-- !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.9'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.9'
-- !ruby/object:Gem::Dependency
-  name: sequel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.33'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.33'
-- !ruby/object:Gem::Dependency
-  name: redis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.3'
+        version: 0.8.1
 description: 
-email: r.sprotte@ub.uni-paderborn.de
+email: 
 executables:
 - metacrunch
 extensions: []
@@ -106,21 +58,11 @@ files:
 - exe/metacrunch
 - lib/metacrunch.rb
 - lib/metacrunch/cli.rb
-- lib/metacrunch/db.rb
-- lib/metacrunch/db/reader.rb
-- lib/metacrunch/db/writer.rb
-- lib/metacrunch/fs.rb
-- lib/metacrunch/fs/entry.rb
-- lib/metacrunch/fs/reader.rb
 - lib/metacrunch/job.rb
 - lib/metacrunch/job/buffer.rb
 - lib/metacrunch/job/dsl.rb
-- lib/metacrunch/job/dsl/option_support.rb
-- lib/metacrunch/parallel_processable_reader.rb
-- lib/metacrunch/redis.rb
-- lib/metacrunch/redis/queue_reader.rb
-- lib/metacrunch/redis/queue_writer.rb
-- lib/metacrunch/redis/writer.rb
+- lib/metacrunch/job/dsl/options.rb
+- lib/metacrunch/job/dsl/options/dsl.rb
 - lib/metacrunch/test_utils.rb
 - lib/metacrunch/test_utils/dummy_callable.rb
 - lib/metacrunch/test_utils/dummy_destination.rb
@@ -147,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Data processing and ETL toolkit for Ruby