metacrunch 2.2.3 → 3.0.0

data/lib/metacrunch/job.rb

@@ -0,0 +1,149 @@
+module Metacrunch
+  class Job
+    require_relative "job/dsl"
+    require_relative "job/buffer"
+
+    attr_reader :builder, :args
+
+    class << self
+      def define(file_content = nil, filename: nil, args: nil, &block)
+        self.new(file_content, filename: filename, args: args, &block)
+      end
+    end
+
+    def initialize(file_content = nil, filename: nil, args: nil, &block)
+      @builder = Dsl.new(self)
+      @args = args
+
+      if file_content
+        @builder.instance_eval(file_content, filename || "")
+      elsif block_given?
+        @builder.instance_eval(&block)
+      end
+    end
+
+    def sources
+      @sources ||= []
+    end
+
+    def add_source(source)
+      ensure_source!(source)
+      sources << source
+    end
+
+    def destinations
+      @destinations ||= []
+    end
+
+    def add_destination(destination)
+      ensure_destination!(destination)
+      destinations << destination
+    end
+
+    def pre_processes
+      @pre_processes ||= []
+    end
+
+    def add_pre_process(callable = nil, &block)
+      add_callable_or_block(pre_processes, callable, &block)
+    end
+
+    def post_processes
+      @post_processes ||= []
+    end
+
+    def add_post_process(callable = nil, &block)
+      add_callable_or_block(post_processes, callable, &block)
+    end
+
+    def transformations
+      @transformations ||= []
+    end
+
+    def add_transformation(callable = nil, &block)
+      add_callable_or_block(transformations, callable, &block)
+    end
+
+    def add_transformation_buffer(size)
+      transformations << Metacrunch::Job::Buffer.new(size)
+    end
+
+    def run
+      run_pre_processes
+      run_transformations
+      run_post_processes
+      self
+    end
+
+  private
+
+    def add_callable_or_block(array, callable, &block)
+      if block_given?
+        array << block
+      elsif callable
+        ensure_callable!(callable)
+        array << callable
+      end
+    end
+
+    def ensure_callable!(object)
+      raise ArgumentError, "#{object} does't respond to #call." unless object.respond_to?(:call)
+    end
+
+    def ensure_source!(object)
+      raise ArgumentError, "#{object} does't respond to #each." unless object.respond_to?(:each)
+    end
+
+    def ensure_destination!(object)
+      raise ArgumentError, "#{object} does't respond to #write." unless object.respond_to?(:write)
+      raise ArgumentError, "#{object} does't respond to #close." unless object.respond_to?(:close)
+    end
+
+    def run_pre_processes
+      pre_processes.each(&:call)
+    end
+
+    def run_post_processes
+      post_processes.each(&:call)
+    end
+
+    def run_transformations
+      sources.each do |source|
+        # sources are expected to respond to `each`
+        source.each do |row|
+          _run_transformations(row)
+        end
+
+        # Run all transformations a last time to flush possible buffers
+        _run_transformations(nil, flush_buffers: true)
+      end
+
+      # destination implementations are expected to respond to `close`
+      destinations.each(&:close)
+    end
+
+    def _run_transformations(row, flush_buffers: false)
+      transformations.each do |transformation|
+        row = if transformation.is_a?(Buffer)
+          if flush_buffers
+            transformation.flush
+          else
+            transformation.buffer(row)
+          end
+        else
+          transformation.call(row) if row
+        end
+
+        break unless row
+      end
+
+      if row
+        destinations.each do |destination|
+          # destinations are expected to respond to `write(row)`
+          destination.write(row)
+        end
+      end
+    end
+
+  end
+end

data/lib/metacrunch/test_utils/dummy_callable.rb

@@ -0,0 +1,14 @@
+require_relative "../test_utils"
+
+module Metacrunch
+  class TestUtils::DummyCallable
+
+    def call(*args)
+      @call_called = true
+    end
+
+  end
+
+  class TestUtils::DummyNonCallable
+  end
+end

data/lib/metacrunch/test_utils/dummy_destination.rb

@@ -0,0 +1,21 @@
+require_relative "../test_utils"
+
+module Metacrunch
+  class TestUtils::DummyDestination
+
+    attr_reader :data
+
+    def write(row)
+      @write_called = true
+      (@data ||= []) << row
+    end
+
+    def close
+      @close_called = true
+    end
+
+  end
+
+  class TestUtils::InvalidDummyDestination
+  end
+end

data/lib/metacrunch/test_utils/dummy_source.rb

@@ -0,0 +1,22 @@
+require_relative "../test_utils"
+
+module Metacrunch
+  class TestUtils::DummySource
+
+    def initialize(count_until: 10)
+      @count_until = count_until
+    end
+
+    def each
+      @each_called = true
+
+      (1..@count_until).each do |number|
+        yield({number: number})
+      end
+    end
+
+  end
+
+  class TestUtils::InvalidDummySource
+  end
+end

data/lib/metacrunch/test_utils.rb

@@ -0,0 +1,7 @@
+module Metacrunch
+  module TestUtils
+    require_relative "test_utils/dummy_source"
+    require_relative "test_utils/dummy_destination"
+    require_relative "test_utils/dummy_callable"
+  end
+end

data/lib/metacrunch/version.rb

@@ -1,3 +1,3 @@
 module Metacrunch
-  VERSION = "2.2.3"
+  VERSION = "3.0.0"
 end

data/lib/metacrunch.rb

@@ -1,32 +1,19 @@
-require "active_support"
-require "active_support/core_ext"
-require "active_support/builder"
-require "parallel"
 require "rubygems"
-require "rubygems/package"
-require "thor"
-require "ox"
+require "bundler"
 
-module Metacrunch
-  require_relative "./metacrunch/version"
-  require_relative "./metacrunch/cli"
-  require_relative "./metacrunch/command"
-  require_relative "./metacrunch/file"
-  require_relative "./metacrunch/file_reader"
-  require_relative "./metacrunch/file_reader_entry"
-  require_relative "./metacrunch/file_writer"
-  require_relative "./metacrunch/hash"
-  require_relative "./metacrunch/parallel"
-  require_relative "./metacrunch/processor"
-  require_relative "./metacrunch/snr"
-  require_relative "./metacrunch/tar_writer"
-  require_relative "./metacrunch/transformator"
-  require_relative "./metacrunch/transformer"
+require "active_support"
+require "active_support/core_ext"
+require "commander"
+require "sequel"
 
-  def self.load_plugins
-    Gem.find_latest_files("metacrunch_plugin.rb").each do |path|
-      load(path)
-    end
-  end
+begin
+  require "pry"
+rescue LoadError ; end
 
+module Metacrunch
+  require_relative "metacrunch/version"
+  require_relative "metacrunch/cli"
+  require_relative "metacrunch/job"
+  require_relative "metacrunch/fs"
+  require_relative "metacrunch/db"
 end

data/metacrunch.gemspec

@@ -7,7 +7,8 @@ Gem::Specification.new do |spec|
   spec.name          = "metacrunch"
   spec.version       = Metacrunch::VERSION
   spec.authors       = ["René Sprotte", "Michael Sievers", "Marcel Otto"]
-  spec.summary       = %q{Data processing toolkit for Ruby}
+  spec.email         = "r.sprotte@ub.uni-paderborn.de"
+  spec.summary       = %q{Data processing and ETL toolkit for Ruby}
   spec.homepage      = "http://github.com/ubpb/metacrunch"
   spec.license       = "MIT"
 
@@ -16,13 +17,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activesupport", "~> 4.2", ">= 4.2.0"
-  spec.add_dependency "builder",       "~> 3.2", ">= 3.2.2"
-  spec.add_dependency "highline",      "~> 1.7"
-  spec.add_dependency "net-scp",       "~> 1.2"
-  spec.add_dependency "net-ssh",       "~> 2.9"
-  spec.add_dependency "parallel",      "~> 1.6", ">= 1.6.0"
-  spec.add_dependency "rubyzip",       ">= 1.0.0"
-  spec.add_dependency "thor",          "~> 0.19"
-  spec.add_dependency "ox",            "~> 2.1"
+  spec.add_dependency "activesupport", "~> 4.2"
+  spec.add_dependency "commander",     "~> 4.4"
+  spec.add_dependency "sequel",        "~> 4.33"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch
 version: !ruby/object:Gem::Version
-  version: 2.2.3
+  version: 3.0.0
 platform: ruby
 authors:
 - René Sprotte
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-11 00:00:00.000000000 Z
+date: 2016-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -19,9 +19,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,135 +26,36 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.2.0
-- !ruby/object:Gem::Dependency
-  name: builder
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.2.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.2.2
-- !ruby/object:Gem::Dependency
-  name: highline
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-- !ruby/object:Gem::Dependency
-  name: net-scp
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-- !ruby/object:Gem::Dependency
-  name: net-ssh
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.9'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.9'
-- !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.6'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.6.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.6'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.6.0
-- !ruby/object:Gem::Dependency
-  name: rubyzip
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.0.0
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: commander
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '4.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '4.4'
 - !ruby/object:Gem::Dependency
-  name: ox
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '4.33'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '4.33'
 description: 
-email: 
+email: r.sprotte@ub.uni-paderborn.de
 executables:
 - metacrunch
 extensions: []
@@ -174,38 +72,20 @@ files:
 - exe/metacrunch
 - lib/metacrunch.rb
 - lib/metacrunch/cli.rb
-- lib/metacrunch/cli/base.rb
-- lib/metacrunch/cli/command_definition.rb
-- lib/metacrunch/cli/command_registry.rb
-- lib/metacrunch/cli/main.rb
-- lib/metacrunch/command.rb
-- lib/metacrunch/file.rb
-- lib/metacrunch/file/reader.rb
-- lib/metacrunch/file/reader/file_system_fetcher.rb
-- lib/metacrunch/file/reader/plain_file_reader.rb
-- lib/metacrunch/file/reader/scp_fetcher.rb
-- lib/metacrunch/file/reader/tar_file_reader.rb
-- lib/metacrunch/file/reader/zip_file_reader.rb
-- lib/metacrunch/file/writer.rb
-- lib/metacrunch/file/writer/plain_file_writer.rb
-- lib/metacrunch/file/writer/tar_file_writer.rb
-- lib/metacrunch/file/writer/zip_file_writer.rb
-- lib/metacrunch/file_reader.rb
-- lib/metacrunch/file_reader_entry.rb
-- lib/metacrunch/file_writer.rb
-- lib/metacrunch/hash.rb
-- lib/metacrunch/parallel.rb
-- lib/metacrunch/processor.rb
-- lib/metacrunch/snr.rb
-- lib/metacrunch/snr/field.rb
-- lib/metacrunch/snr/section.rb
-- lib/metacrunch/tar_writer.rb
-- lib/metacrunch/transformator.rb
-- lib/metacrunch/transformator/transformation.rb
-- lib/metacrunch/transformator/transformation/step.rb
-- lib/metacrunch/transformer.rb
-- lib/metacrunch/transformer/helper.rb
-- lib/metacrunch/transformer/step.rb
+- lib/metacrunch/db.rb
+- lib/metacrunch/db/reader.rb
+- lib/metacrunch/db/writer.rb
+- lib/metacrunch/fs.rb
+- lib/metacrunch/fs/entry.rb
+- lib/metacrunch/fs/reader.rb
+- lib/metacrunch/job.rb
+- lib/metacrunch/job/buffer.rb
+- lib/metacrunch/job/dsl.rb
+- lib/metacrunch/job/dsl/option_support.rb
+- lib/metacrunch/test_utils.rb
+- lib/metacrunch/test_utils/dummy_callable.rb
+- lib/metacrunch/test_utils/dummy_destination.rb
+- lib/metacrunch/test_utils/dummy_source.rb
 - lib/metacrunch/version.rb
 - metacrunch.gemspec
 homepage: http://github.com/ubpb/metacrunch
@@ -231,5 +111,5 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Data processing toolkit for Ruby
+summary: Data processing and ETL toolkit for Ruby
 test_files: []

data/lib/metacrunch/cli/base.rb

@@ -1,29 +0,0 @@
-module Metacrunch
-  class Cli
-    class Base < Thor
-      no_commands do
-        def self.register_thor_command(command_definition)
-          desc(command_definition.usage, command_definition.desc)
-
-          command_definition.options.each do |key, value|
-            option(key, value)
-          end
-
-          define_method(command_definition.name) do |*params|
-            run_command(command_definition.command_class, params)
-          end
-        end
-
-        def run_command(command, params = [])
-          klass = command.class == Class ? command : command.to_s.constantize
-          raise ArgumentError, "command must be a Metacrunch::Command class" unless klass < Metacrunch::Command
-
-          command = klass.new(shell, options, params)
-          command.pre_perform
-          command.perform
-          command.post_perform
-        end
-      end
-    end
-  end
-end

data/lib/metacrunch/cli/command_definition.rb

@@ -1,41 +0,0 @@
-module Metacrunch
-  class Cli
-    class CommandDefinition
-      def initialize(command_class, description)
-        @klass       = command_class
-        @name        = nil
-        @description = description
-        @usage       = nil
-        @options     = {}
-      end
-
-      def command_class(value = nil)
-        @klass if value
-        @klass
-      end
-
-      def name(value = nil)
-        @name = value if value.present?
-        @name || @klass.to_s.demodulize.underscore.gsub!(/_command\Z/, "")
-      end
-
-      def usage(value = nil)
-        @usage = value if value.present?
-        @usage || name
-      end
-
-      def desc(value = nil)
-        @description = value if value.present?
-        @description
-      end
-
-      def option(name, options = {})
-        @options[name] = options
-      end
-
-      def options
-        @options
-      end
-    end
-  end
-end

data/lib/metacrunch/cli/command_registry.rb

@@ -1,17 +0,0 @@
-module Metacrunch
-  class Cli
-    class CommandRegistry
-      attr_reader :commands
-
-      def initialize
-        @commands = []
-      end
-
-      def register(klass, description = nil, &block)
-        command = CommandDefinition.new(klass, description)
-        yield(command) if block_given?
-        @commands << command
-      end
-    end
-  end
-end

data/lib/metacrunch/cli/main.rb

@@ -1,16 +0,0 @@
-module Metacrunch
-  class Cli
-    class Main < Thor
-      desc "console", "Start a console. It uses Pry if installed, IRB otherwise."
-      def console
-        begin
-          require "pry"
-          Pry.start
-        rescue LoadError
-          require "irb"
-          IRB.start
-        end
-      end
-    end
-  end
-end

data/lib/metacrunch/command.rb

@@ -1,27 +0,0 @@
-module Metacrunch
-  class Command
-
-    attr_reader :shell
-    attr_reader :options
-    attr_reader :params
-
-    def initialize(shell, options = {}, params = [])
-      @shell   = shell
-      @options = options
-      @params  = params
-    end
-
-    def pre_perform
-      # can be implemented in sub-class
-    end
-
-    def perform
-      raise NotImplementedError, "You must implement .perform() in your command sub-class"
-    end
-
-    def post_perform
-      # can be implemented in sub-class
-    end
-
-  end
-end

data/lib/metacrunch/file/reader/file_system_fetcher.rb

@@ -1,21 +0,0 @@
-require_relative "../reader"
-
-# This fetcher does not do anything besides returning the filename withouzt the
-# file:// protocol prefix. It only exists to keep the implementation consistent.
-class Metacrunch::File::Reader::FileSystemFetcher
-  include Enumerable
-
-  RECOGNIZED_PROTOCOL_REGEX = /\Afile:\/\//
-
-  def self.accepts?(url)
-    !!url[RECOGNIZED_PROTOCOL_REGEX]
-  end
-
-  def initialize(url, options = {})
-    @url = url
-  end
-
-  def each
-    yield @url.sub(RECOGNIZED_PROTOCOL_REGEX, "")
-  end
-end