metacrunch 2.1.1 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5bcfeeb60f1625e155e0af9a3b97fdc4d580f5d8
-  data.tar.gz: 9f157ad5acb41c7fe3fc8bad5585653a5a6ff41a
+  metadata.gz: ca4236a24f7c856c544dc3eef7f933eca1e0b4d7
+  data.tar.gz: 4ae2e318c548e41183759499d67211b0ed316b7c
 SHA512:
-  metadata.gz: d2a45e6ba7218f8b283c9b018b76129ce98187037c84986a11e27bffdad3cde4ce9f51c0112c5e34d17b40b5c46e101ddba74fe93f0fe48e00f51a4859da96a2
-  data.tar.gz: 39ff3a35ab79367db1af87ae9ba7abc92a957eb8b92a1c040eaca3cdc4140cde0e15501fe7204afb451fc971ecf9ecae934f74aac39f68f71f6e0cc0fe95e7a6
+  metadata.gz: fbc4b38921697648b09fa6f5b70665b3fceb3f9fa9fe1407da2d578515389646405b3decb7a0ce10dd056bd53a85b0a4cf9db909644f7fe658fc347b2a1938ad
+  data.tar.gz: 861bee6defe53134bd865742eac8aeb7731d4cf9c7252f12cc57521105057241637354278ec598f900a70bb52785e6f3f5dda351b7d8b9b38ae5ea732e64e063

data/Gemfile

@@ -1,17 +1,25 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
+# Specify your gem's dependencies in your gemspec
 gemspec
 
-gem "rake"
-gem "rspec", "~> 3.2.0"
+group :development do
+  gem "bundler"
+  gem "nokogiri"
+  gem "rake"
+  gem "rspec",           ">= 3.0.0",  "< 4.0.0"
+  gem "simplecov",       ">= 0.8.0"
 
-if !ENV["CI"]
-  group :development do
+  if !ENV["CI"]
     gem "hashdiff"
     gem "pry",                "~> 0.9.12.6"
     gem "pry-byebug",         "<= 1.3.2"
-    gem "pry-rescue",         "~> 1.4.1", github: "ConradIrwin/pry-rescue", branch: :master
+    gem "pry-rescue",         "~> 1.4.2"
     gem "pry-stack_explorer", "~> 0.4.9.1"
     gem "pry-syntax-hacks",   "~> 0.0.6"
   end
 end
+
+group :test do
+  gem "codeclimate-test-reporter", require: nil
+end

data/Rakefile

@@ -1,3 +1,4 @@
+require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "metacrunch"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require "pry"
+Pry.start

data/lib/metacrunch/file/reader/file_system_fetcher.rb

@@ -0,0 +1,21 @@
+require_relative "../reader"
+
+# This fetcher does not do anything besides returning the filename withouzt the
+# file:// protocol prefix. It only exists to keep the implementation consistent.
+class Metacrunch::File::Reader::FileSystemFetcher
+  include Enumerable
+
+  RECOGNIZED_PROTOCOL_REGEX = /\Afile:\/\//
+
+  def self.accepts?(url)
+    !!url[RECOGNIZED_PROTOCOL_REGEX]
+  end
+
+  def initialize(url, options = {})
+    @url = url
+  end
+
+  def each
+    yield @url.sub(RECOGNIZED_PROTOCOL_REGEX, "")
+  end
+end

data/lib/metacrunch/file/reader/plain_file_reader.rb

@@ -0,0 +1,33 @@
+require_relative "../reader"
+
+class Metacrunch::File::Reader::PlainFileReader
+  include Enumerable
+
+  def self.accepts?(filename)
+    true
+  end
+
+  def initialize(filename)
+    @filename = filename
+  end
+
+  def each
+    return enum_for(__method__) unless block_given?
+
+    io =
+    if @filename.end_with?("gz") # catches tgz and tar.gz
+      Zlib::GzipReader.open(@filename)
+    else
+      File.open(@filename)
+    end
+
+    yield Metacrunch::File.new({
+      content: io.read,
+      entry_name: File.basename(@filename),
+      file_name: @filename,
+      is_directory: false
+    })
+
+    io.close
+  end
+end

data/lib/metacrunch/file/reader/scp_fetcher.rb

@@ -0,0 +1,56 @@
+require "etc"
+require "net/scp"
+require "net/ssh"
+require "securerandom"
+require_relative "../reader"
+
+class Metacrunch::File::Reader::ScpFetcher
+  include Enumerable
+
+  RECOGNIZED_PROTOCOL_REGEX = /\Ascp:\/\//
+  TILDE_REPLACEMENT = "/__TILDE__"
+
+  def self.accepts?(url)
+    !!url[RECOGNIZED_PROTOCOL_REGEX]
+  end
+
+  def initialize(url, options = {})
+    URI(url.sub("~", TILDE_REPLACEMENT)).try do |_uri|
+      @host = _uri.host
+      @password = options[:password] || _uri.password
+      @path = _uri.path.sub(TILDE_REPLACEMENT, "~")
+      @username = options[:username] || _uri.user || Etc.getlogin
+    end
+  end
+
+  def each
+    return enum_for(__method__) unless block_given?
+
+    begin
+      begin
+        Dir.mkdir temporary_directory = File.join(Dir.tmpdir, SecureRandom.hex)
+      rescue Errno::EEXIST
+        retry
+      end
+
+      remote_filenames.each do |_remote_filename|
+        _local_filename = File.join(temporary_directory, File.basename(_remote_filename))
+        Net::SCP.download!(@host, @username, _remote_filename, _local_filename, ssh: { password: @password })
+        yield _local_filename
+        File.delete(_local_filename)
+      end
+    ensure
+      FileUtils.remove_dir(temporary_directory)
+    end
+  end
+
+  private
+
+  def remote_filenames
+    @remote_filenames ||= [].tap do |_remote_filenames|
+      Net::SSH.start(@host, @username, password: @password) do |_ssh|
+        _remote_filenames.concat _ssh.exec!("ruby -e \"puts Dir.glob(File.expand_path('#{@path}'))\"").try(:split, "\n") || []
+      end
+    end
+  end
+end

data/lib/metacrunch/file/reader/tar_file_reader.rb

@@ -0,0 +1,37 @@
+require "rubygems/package"
+require_relative "../reader"
+
+class Metacrunch::File::Reader::TarFileReader
+  include Enumerable
+
+  def self.accepts?(filename)
+    !!filename[/\.tar\Z|\.tar\.gz\Z|\.tgz\Z/]
+  end
+
+  def initialize(filename)
+    @filename = filename
+  end
+
+  def each
+    return enum_for(__method__) unless block_given?
+
+    io =
+    if @filename.end_with?("gz") # catches tgz and tar.gz
+      Zlib::GzipReader.open(@filename)
+    else
+      File.open(@filename)
+    end
+
+    Gem::Package::TarReader.new(io).each do |_tar_entry|
+      unless _tar_entry.directory?
+        yield Metacrunch::File.new({
+          content: _tar_entry.read,
+          entry_name: _tar_entry.full_name,
+          file_name: @filename
+        })
+      end
+    end
+
+    io.close
+  end
+end

data/lib/metacrunch/file/reader/zip_file_reader.rb

@@ -0,0 +1,30 @@
+require "zip"
+require_relative "../reader"
+
+class Metacrunch::File::Reader::ZipFileReader
+  include Enumerable
+
+  def self.accepts?(filename)
+    !!filename[/\.zip\Z/]
+  end
+
+  def initialize(filename)
+    @filename = filename
+  end
+
+  def each
+    return enum_for(__method__) unless block_given?
+
+    Zip::File.open(@filename) do |_zip_file|
+      _zip_file.each do |_zip_entry|
+        unless _zip_entry.directory?
+          yield Metacrunch::File.new({
+            content: _zip_entry.get_input_stream.read,
+            entry_name: _zip_entry.name,
+            file_name: @filename
+          })
+        end
+      end
+    end
+  end
+end

data/lib/metacrunch/file/reader.rb

@@ -0,0 +1,72 @@
+require_relative "../file"
+require_relative "../processor"
+
+class Metacrunch::File::Reader < Metacrunch::Processor
+  require_relative "./reader/file_system_fetcher"
+  require_relative "./reader/plain_file_reader"
+  require_relative "./reader/scp_fetcher"
+  require_relative "./reader/tar_file_reader"
+  require_relative "./reader/zip_file_reader"
+
+  include Enumerable
+
+  def initialize(options = {})
+    @bulk_size = options[:bulk_size].try(:to_i) || 1
+    @urls = [
+      options[:filename], options[:filenames],
+      options[:url], options[:urls]
+    ]
+    .flatten
+    .compact
+    .map do |_filename_or_url|
+      if (_url = _filename_or_url)[/\A\w+:\/\//]
+        _url
+      else
+        Dir.glob(File.expand_path(_filename_or_url)).map do |_filename|
+          "file://#{_filename}"
+        end
+      end
+    end
+    .flatten # because there might be arrays again because of Dir.glob
+
+    @force_content_encoding = options[:force_content_encoding]
+    @password = options[:password]
+    @username = options[:username]
+  end
+
+  def call(items = [], pipeline = nil)
+    @chunks_of_entries_enumerator ||= each_slice(@bulk_size) # instance method from Enumerable
+   
+    begin
+      items.concat(@chunks_of_entries_enumerator.next)
+    rescue StopIteration
+      pipeline.try(:terminate!)
+    end
+  end
+
+  def each
+    return enum_for(__method__) unless block_given?
+
+    @urls.each do |_url|
+      [FileSystemFetcher, ScpFetcher].find do |_fetcher|
+        _fetcher.accepts?(_url)
+      end
+      .try do |_appropriate_fetcher|
+        _appropriate_fetcher.new(_url, username: @username, password: @password).each do |_filename|
+          [TarFileReader, ZipFileReader, PlainFileReader].find do |_reader| # PlainFileReader as last will read any file
+            _reader.accepts?(_filename)
+          end
+          .try do |_appropriate_reader|
+            _appropriate_reader.new(_filename).each do |_file|
+              if @force_content_encoding.present?
+                _file.content.try(:force_encoding, @force_content_encoding)
+              end
+
+              yield _file
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metacrunch/file.rb

@@ -0,0 +1,23 @@
+require_relative "../metacrunch"
+
+class Metacrunch::File
+  require_relative "./file/reader"
+
+  attr_accessor :content
+  attr_accessor :entry_name # equals file_name for plain files
+  attr_accessor :file_name
+
+  def initialize(options = {})
+    @content = options[:content]
+    @entry_name = options[:entry_name]
+    @file_name = options[:file_name]
+  end
+
+  def to_h
+    {
+      content: @content,
+      entry_name: @entry_name,
+      file_name: @file_name
+    }
+  end
+end

data/lib/metacrunch/file_reader.rb

@@ -31,14 +31,14 @@ module Metacrunch
     end
 
     def read_regular_file(filename, &block)
-      if File.file?(filename)
-        io = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : File.open(filename, "r")
+      if ::File.file?(filename)
+        io = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
         yield Entry.new(filename: filename, archive_filename: nil, contents: io.read)
       end
     end
 
     def read_archive(filename, &block)
-      io        = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : File.open(filename, "r")
+      io        = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
       tarReader = Gem::Package::TarReader.new(io)
 
       tarReader.each do |_tar_entry|

data/lib/metacrunch/file_writer.rb

@@ -5,10 +5,10 @@ module Metacrunch
 
 
     def initialize(filename, override: false, compress: nil)
-      @path       = File.expand_path(filename)
+      @path       = ::File.expand_path(filename)
       @compressed = (compress ||= @path.ends_with?(".gz"))
 
-      if File.exist?(@path) && !override
+      if ::File.exist?(@path) && !override
         raise FileExistError, "File #{@path} already exists. Set override = true to override the existing file."
       end
     end
@@ -33,7 +33,7 @@ module Metacrunch
   private
 
     def io
-      @io ||= (@compressed == true) ? Zlib::GzipWriter.open(@path) : File.open(@path, "w")
+      @io ||= (@compressed == true) ? Zlib::GzipWriter.open(@path) : ::File.open(@path, "w")
     end
 
   end

data/lib/metacrunch/hash.rb

@@ -0,0 +1,51 @@
+require_relative "../metacrunch"
+
+module Metacrunch::Hash
+  def self.add(object, *args)
+    if args.length < 2
+      raise ArgumentError
+    else
+      return args.last if args.last.nil? || (args.last.respond_to?(:empty?) && args.last.empty?)
+
+      if args.length == 2
+        args_first_is_a_string = args.first.is_a?(String) # memoize
+
+        if args_first_is_a_string && args.first.include?("/")
+          add(object, *args.first.split("/"), args.last)
+        else
+          if args_first_is_a_string && args.first.start_with?(":")
+            _add(object, args.first[1..-1].to_sym, args.last)
+          else
+            _add(object, args.first, args.last)
+          end
+        end
+      else
+        nested_hash = args[0..-3].inject(object) do |_memo, _key|
+          if _key.is_a?(String) && _key.start_with?(":")
+            _key = _key[1..-1].to_sym
+          end
+
+          _memo[_key] ||= object.class.new
+        end
+
+        _add(nested_hash, args[-2], args[-1])
+      end
+    end
+  end
+
+  private
+
+  def self._add(hash, key, value)
+    #if value.is_a?(FalseClass) || (value.respond_to?(:empty?) ? !value.empty? : !!value) # like ActiveSupport implements blank?/present?
+    if hash[key].nil?
+      hash[key] = value.is_a?(Array) && value.length == 1 ? value.first : value
+    elsif hash[key].is_a?(Array)
+      (hash[key] << value).flatten!(1)
+    else
+      (hash[key] = [hash[key], value]).flatten!(1)
+    end
+    #end
+
+    hash[key]
+  end
+end

data/lib/metacrunch/processor.rb

@@ -0,0 +1,10 @@
+require_relative "../metacrunch"
+
+class Metacrunch::Processor
+  def initialize(options = {})
+    @options ||= options
+  end
+
+  def call(items = [], pipeline = nil)
+  end
+end

data/lib/metacrunch/snr/section.rb

@@ -19,7 +19,13 @@ module Metacrunch
       # Adds a field
       #
       def add(field_name, value)
-        add_field(Field.new(field_name, value))
+        if value.is_a?(Array)
+          value.each do |_value|
+            add_field(Field.new(field_name, _value))
+          end
+        else
+          add_field(Field.new(field_name, value))
+        end
       end
 
       # ------------------------------------------------------------------------------
@@ -27,12 +33,17 @@ module Metacrunch
       # ------------------------------------------------------------------------------
 
       #
-      # Return all fields.
+      # Return all fields. A name can be provided to filter fields by name.
       #
+      # @param [String, nil] name
       # @return [Array<Metacrunch::SNR::Section::Field>]
       #
-      def fields
-        @fields
+      def fields(name = nil)
+        if name
+          @fields.select{|field| field.name == name}
+        else
+          @fields
+        end
       end
 
       #

data/lib/metacrunch/snr.rb

@@ -33,6 +33,22 @@ module Metacrunch
       add_section(section)
     end
 
+    #
+    # Returns field values for a given path.
+    #
+    # @param [String] path A path to the fields seperated by /. E.g. section/field
+    # @return [Array<*>]
+    #
+    def values(path)
+      section_name, field_name = path.split("/")
+      section = self.section(section_name)
+      if section && field_name
+        section.fields(field_name).map{|field| field.value}
+      else
+        []
+      end
+    end
+
     # ------------------------------------------------------------------------------
     # Sections
     # ------------------------------------------------------------------------------

data/lib/metacrunch/transformator/transformation/step.rb

@@ -0,0 +1,45 @@
+require_relative "../transformation"
+
+class Metacrunch::Transformator::Transformation::Step
+  attr_accessor :transformation
+
+  def initialize(transformation = nil, options = {})
+    if transformation.is_a?(Hash)
+      options = transformation
+      transformation = nil
+    end
+
+    if transformation
+      @transformation = transformation
+    else
+      @transformation = Struct.new(:source, :target).new.tap do |_struct|
+        _struct.source = options[:source]
+        _struct.target = options[:target]
+      end
+    end
+  end
+
+  def call
+  end
+
+  #
+  # Each step has transparent access to all methods of it's transformation
+  #
+  def method_missing(method_name, *args, &block)
+    if @transformation.respond_to?(method_name)
+      @transformation.send(method_name, *args, &block)
+    else
+      super
+    end
+  end
+
+  def respond_to_missing?(method_name, include_private = false)
+    @transformation.respond_to?(method_name) || super
+  end
+
+  # avoid method_missing penalty for the most used transformation methods
+  def source;         @transformation.source;         end
+  def source=(value); @transformation.source=(value); end
+  def target;         @transformation.target;         end
+  def target=(value); @transformation.target=(value); end
+end

data/lib/metacrunch/transformator/transformation.rb

@@ -0,0 +1,48 @@
+require_relative "../transformator"
+
+class Metacrunch::Transformator::Transformation
+  require_relative "./transformation/step"
+
+  attr_accessor :source
+  attr_accessor :target
+
+  class << self
+    def steps(value = nil)
+      unless value
+        @steps
+      else
+        @steps = value
+      end
+    end
+    alias_method :sequence, :steps
+  end
+
+  def self.call(*args)
+    new.call(*args)
+  end
+
+  # since a transformation can have many steps, writing a "require" for each is tedious
+  def self.require_directory(directory)
+    Dir.glob("#{File.expand_path(directory)}/*.rb").each do |_filename|
+      require _filename
+    end
+  end
+
+  def initialize
+    # steps are instanced once, which means that instance variables retain
+    @steps = self.class.steps.flatten.map do |_step|
+      _step.is_a?(Class) ? _step.new(self) : _step
+    end
+  end
+
+  def call(source, options = {})
+    @source = source
+    @target = options[:target]
+
+    @steps.each do |_step|
+      _step.is_a?(Proc) ? instance_exec(&_step) : _step.call
+    end
+
+    return @target
+  end
+end

data/lib/metacrunch/transformator.rb

@@ -0,0 +1,5 @@
+require_relative "../metacrunch"
+
+module Metacrunch::Transformator
+  require_relative "./transformator/transformation"
+end

data/lib/metacrunch/transformer.rb

@@ -4,6 +4,7 @@ module Metacrunch
     require_relative "./transformer/helper"
 
     attr_accessor :source, :target, :options
+    attr_reader   :step
 
 
     def initialize(source:nil, target:nil, options: {})
@@ -14,10 +15,13 @@ module Metacrunch
 
     def transform(step_class = nil, &block)
       if block_given?
-        Step.new(self).instance_eval(&block) # TODO: Benchmark this
+        @step = Step.new(self)
+        @step.instance_eval(&block)
       else
         raise ArgumentError, "You need to provide a STEP or a block" if step_class.nil?
-        step_class.new(self).perform
+        clazz = step_class.is_a?(Class) ? step_class : step_class.to_s.constantize
+        @step = clazz.new(self)
+        @step.perform
       end
     end
 

data/lib/metacrunch/version.rb

@@ -1,3 +1,3 @@
 module Metacrunch
-  VERSION = "2.1.1"
+  VERSION = "2.2.0"
 end

data/lib/metacrunch.rb

@@ -7,21 +7,21 @@ require "rubygems/package"
 require "thor"
 require "ox"
 
-begin
-  require "pry"
-rescue LoadError ; end
-
 module Metacrunch
   require_relative "./metacrunch/version"
   require_relative "./metacrunch/cli"
   require_relative "./metacrunch/command"
+  require_relative "./metacrunch/file"
   require_relative "./metacrunch/file_reader"
   require_relative "./metacrunch/file_reader_entry"
   require_relative "./metacrunch/file_writer"
-  require_relative "./metacrunch/tar_writer"
+  require_relative "./metacrunch/hash"
+  require_relative "./metacrunch/parallel"
+  require_relative "./metacrunch/processor"
   require_relative "./metacrunch/snr"
+  require_relative "./metacrunch/tar_writer"
+  require_relative "./metacrunch/transformator"
   require_relative "./metacrunch/transformer"
-  require_relative "./metacrunch/parallel"
 
   def self.load_plugins
     Gem.find_latest_files("metacrunch_plugin.rb").each do |path|

data/metacrunch.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |s|
   s.licenses      = ["MIT"]
 
   s.files         = `git ls-files`.split($\)
-  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.executables   = s.files.grep(%r{^bin/metacrunch}) { |f| File.basename(f) }
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.name          = "metacrunch"
   s.require_paths = ["lib"]
@@ -21,7 +21,11 @@ Gem::Specification.new do |s|
 
   s.add_dependency "activesupport", "~> 4.2", ">= 4.2.0"
   s.add_dependency "builder",       "~> 3.2", ">= 3.2.2"
+  s.add_dependency "highline",      "~> 1.7"
+  s.add_dependency "net-scp",       "~> 1.2"
+  s.add_dependency "net-ssh",       "~> 2.9"
   s.add_dependency "parallel",      "~> 1.6", ">= 1.6.0"
+  s.add_dependency "rubyzip",       ">= 1.0.0"
   s.add_dependency "thor",          "~> 0.19"
   s.add_dependency "ox",            "~> 2.1"
 end

data/spec/assets/file/some_file

@@ -0,0 +1 @@
+Some text

data/spec/assets/sql_lite_config.rb

@@ -0,0 +1,17 @@
+pipeline do
+  reader "Metacrunch::Archive::Reader", {
+    bulk_size: 2,
+    filenames: "foo.tar.gz"
+  }
+
+  processor -> (items, pipeline) {
+    items.map! do |_item|
+
+    end
+  }
+  #processor "Metacrunch::Sqlite::Writer", {
+  #  database: "mab_xml.db"
+  #}
+
+  processor "Metacrunch::Debugger"
+end

data/spec/file_writer_spec.rb

@@ -4,8 +4,8 @@ describe Metacrunch::FileWriter do
   let(:compressed_file) { "/tmp/metacrunch_spec_regular_file.txt.gz" }
 
   before do
-    File.unlink(regular_file)    if File.exist?(regular_file)
-    File.unlink(compressed_file) if File.exist?(compressed_file)
+    ::File.unlink(regular_file)    if ::File.exist?(regular_file)
+    ::File.unlink(compressed_file) if ::File.exist?(compressed_file)
   end
 
   it "can write a file" do
@@ -32,7 +32,7 @@ describe Metacrunch::FileWriter do
   end
 
   it "can override existing file" do
-    File.write(regular_file, "FOO")
+    ::File.write(regular_file, "FOO")
 
     expect {
       Metacrunch::FileWriter.new(regular_file)

data/spec/metacrunch/file/reader/plain_file_reader_spec.rb

@@ -0,0 +1,11 @@
+describe Metacrunch::File::Reader::PlainFileReader do
+  describe ".accepts?" do
+    it "accepts all filenames" do
+      expect(described_class.accepts?("foo")).to equal(true)
+      expect(described_class.accepts?("foo.gz")).to equal(true)
+      expect(described_class.accepts?("foo.tar.gz")).to equal(true)
+      expect(described_class.accepts?("foo.xml")).to equal(true)
+      expect(described_class.accepts?("foo.zip")).to equal(true)
+    end
+  end
+end

data/spec/metacrunch/file/reader/tar_reader_spec.rb

@@ -0,0 +1,15 @@
+describe Metacrunch::File::Reader::TarFileReader do
+  describe ".accepts?" do
+    it "accepts filenames this reader can process" do
+      expect(described_class.accepts?("foo.tar")).to equal(true)
+      expect(described_class.accepts?("foo.tar.gz")).to equal(true)
+      expect(described_class.accepts?("foo.tgz")).to equal(true)
+    end
+
+    it "does not accept filenames this reader cannot process" do
+      expect(described_class.accepts?("plain_file")).to equal(false)
+      expect(described_class.accepts?("plain_file.gz")).to equal(false)
+      expect(described_class.accepts?("tatar.gz")).to equal(false)
+    end
+  end
+end

data/spec/metacrunch/file/reader/zip_reader_spec.rb

@@ -0,0 +1,13 @@
+describe Metacrunch::File::Reader::ZipFileReader do
+  describe ".accepts?" do
+    it "accepts filenames this reader can process" do
+      expect(described_class.accepts?("foo.zip")).to equal(true)
+    end
+
+    it "does not accept filenames this reader cannot process" do
+      expect(described_class.accepts?("plain_file")).to equal(false)
+      expect(described_class.accepts?("plain.zip.file")).to equal(false)
+      expect(described_class.accepts?("foozip")).to equal(false)
+    end
+  end
+end

data/spec/metacrunch/file/reader_spec.rb

@@ -0,0 +1,47 @@
+describe Metacrunch::File::Reader do
+  def pipeline_factory
+    Class.new do
+      def terminate!;  @terminated = true; end
+      def terminated?; !!@terminated;      end
+    end
+    .new
+  end
+
+  def reader_factory(filename)
+    described_class.new({
+      bulk_size: 10,
+      filenames: [filename],
+      force_content_encoding: "utf-8"
+    })
+  end
+
+  # filenames
+  let(:plain_file_name) { File.join(asset_dir, "file/some_file") }
+  let(:plain_gz_file_name) { File.join(asset_dir, "file/some_file.gz") }
+  let(:tar_file_name) { File.join(asset_dir, "file/tar_archive.tar") }
+  let(:tar_gz_file_name) { File.join(asset_dir, "file/tar_gz_archive.tar.gz") }
+  let(:tgz_file_name) { File.join(asset_dir, "file/tgz_archive.tgz") }
+  let(:zip_file_name) { File.join(asset_dir, "file/zip_archive.zip") }
+
+  # readers
+  let(:plain_file_reader) { reader_factory(plain_file_name) }
+  let(:plain_gz_file_reader) { reader_factory(plain_gz_file_name) }
+  let(:tar_reader) { reader_factory(tar_file_name) }
+  let(:tar_gz_reader) { reader_factory(tar_gz_file_name) }
+  let(:tgz_reader) { reader_factory(tgz_file_name) }
+  let(:zip_reader) { reader_factory(zip_file_name) }
+
+  describe "#call" do
+    it "fills items with #{_file_type = Metacrunch::File} objects" do
+      [plain_file_reader, plain_gz_file_reader, tar_reader, tar_gz_reader,tgz_reader, zip_reader].each do |_reader|
+        pipeline = pipeline_factory
+
+        _reader.call(items = [], pipeline)
+        expect(items.length).not_to eq(0)
+        expect(items).to all(be_a(_file_type))
+
+        _reader.call(items, pipeline) until pipeline.terminated?
+      end
+    end
+  end
+end

data/spec/metacrunch/file_spec.rb

@@ -0,0 +1,15 @@
+describe Metacrunch::File do
+  describe "#to_h" do
+    let(:file) do
+      described_class.new({
+        content: "content",
+        entry_name: "entry_name",
+        file_name: "file_name"
+      })
+    end
+
+    it "returns a hash representation of the file object" do
+      expect(file.to_h).to be_a(Hash)
+    end
+  end
+end

data/spec/metacrunch/hash_spec.rb

@@ -0,0 +1,93 @@
+describe Metacrunch::Hash do
+  describe ".add" do
+    it "adds the given key/value pair" do
+      described_class.add(hash = {}, :key, "value")
+      expect(hash[:key]).to eq("value")
+    end
+
+    it "respects the type of the key" do
+      hash = {}
+
+      described_class.add(hash, :symbol_key1, "value")
+      described_class.add(hash, ":symbol_key2", "value")
+      described_class.add(hash, "string_key", "value")
+
+      expect(hash[:symbol_key1]).to eq("value")
+      expect(hash[:symbol_key2]).to eq("value")
+      expect(hash["string_key"]).to eq("value")
+    end
+
+    context "if wrong arguments are given" do
+      it "raises an #{exception_type = ArgumentError}" do
+        expect { described_class.add(hash) }.to raise_error(exception_type)
+      end
+    end
+
+    context "if key includes forward dashes" do
+      it "splits the key by forward dashes" do
+        described_class.add(hash = {}, "key/subkey", "value")
+        expect(hash["key"]["subkey"]).to eq("value")
+      end
+
+      it "respects the type of the keys" do
+        described_class.add(hash = {}, "key/:subkey/sub_sub_key", "value")
+        expect(hash["key"][:subkey]["sub_sub_key"]).to eq("value")
+      end
+    end
+
+    context "if multiple keys are given" do
+      it "dives into the object and adds the key/value pair" do
+        hash = { "key" => { "subkey" => "former_value" } }
+        described_class.add(hash, "key", "subkey", "value")
+        expect(hash["key"]["subkey"]).to eq(["former_value", "value"])
+      end
+    end
+
+    context "if there already exists a value for the given key" do
+      context "if the value is an array" do
+        it "adds the value to the array" do
+          hash = { "key" => ["former_value1", "former_value2"] }
+          described_class.add(hash, "key", "value")
+          expect(hash["key"]).to eq(["former_value1", "former_value2", "value"])
+        end
+
+        it "returns aggregated value" do
+          hash = { "key" => ["former_value1", "former_value2"] }
+          expect(described_class.add(hash, "key", "value")).to eq(["former_value1", "former_value2", "value"])
+        end
+      end
+
+      it "makes the existing value an array and adds the given value" do
+        hash = { "key" => "former_value" }
+        described_class.add(hash, "key", "value")
+        expect(hash["key"]).to eq(["former_value", "value"])
+      end
+
+      it "returns aggregated value" do
+        hash = { "key" => "former_value" }
+        expect(described_class.add(hash, "key", "value")).to eq(["former_value", "value"])
+      end
+    end
+
+    context "if the given value is nil" do
+      it "does not add anything" do
+        described_class.add(hash = {}, "key/subkey", nil)
+        expect(hash).to eq({})
+      end
+    end
+
+    context "if the given value is empty" do
+      it "does not add anything" do
+        hash = {}
+        described_class.add(hash, "key/subkey1", "")
+        described_class.add(hash, "key/subkey2", [])
+        described_class.add(hash, "key/subkey3", Class.new { def empty?; true; end }.new)
+        expect(hash).to eq({})
+      end
+    end
+
+    it "returns the value" do
+      expect(described_class.add({}, "key", "value")).to eq("value")
+    end
+  end
+end

data/spec/metacrunch/transformator/transformation_spec.rb

@@ -0,0 +1,28 @@
+describe Metacrunch::Transformator::Transformation do
+  class SomeStep < described_class::Step
+    def call
+      self.target = { muff: 1 }
+    end
+  end
+
+  class SomeTransformation < described_class
+    sequence [
+      SomeStep
+    ]
+  end
+
+  it "foo" do
+    result = SomeTransformation.call({
+      my: {
+        awesome: {
+          source: {
+            title: "foo",
+            description: "bar"
+          }
+        }
+      }
+    })
+
+    result
+  end
+end

data/spec/snr_spec.rb

@@ -59,7 +59,7 @@ describe Metacrunch::SNR do
   describe ".to_xml" do
     before {
       snr.add("section1", "title", "Foo Bar")
-      snr.add("section1", "artists", ["Sievers, Michael", "Sprotte, René"])
+      snr.add("section1", "artist", ["Sievers, Michael", "Sprotte, René"])
       snr.add("section2", "link", {label: "Click here", url: "http://example.com"})
     }
 
@@ -67,7 +67,7 @@ describe Metacrunch::SNR do
 
     it { is_expected.not_to be_nil }
     it { is_expected.not_to be_empty }
-    it { is_expected.to eq("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<snr>\n  <section1>\n    <title>Foo Bar</title>\n    <artists type=\"array\">\n      <artist>Sievers, Michael</artist>\n      <artist>Sprotte, René</artist>\n    </artists>\n  </section1>\n  <section2>\n    <link>\n      <label>Click here</label>\n      <url>http://example.com</url>\n    </link>\n  </section2>\n</snr>\n") }
+    it { is_expected.to eq("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<snr>\n  <section1>\n    <title>Foo Bar</title>\n    <artist>Sievers, Michael</artist>\n    <artist>Sprotte, René</artist>\n  </section1>\n  <section2>\n    <link>\n      <label>Click here</label>\n      <url>http://example.com</url>\n    </link>\n  </section2>\n</snr>\n") }
   end
 
 end

data/spec/spec_helper.rb

@@ -1,6 +1,29 @@
+if ENV["CODECLIMATE_REPO_TOKEN"]
+  require "codeclimate-test-reporter"
+  CodeClimate::TestReporter.start
+else
+  require "simplecov"
+  SimpleCov.start
+end
+
 require "metacrunch"
 
+begin
+  require "pry"
+  require "yaml"
+rescue LoadError
+end
+
 RSpec.configure do |config|
+  # begin --- rspec 3.1 generator
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+  # end --- rspec 3.1 generator
 end
 
 # Helper to provide RSpec.root
@@ -10,3 +33,11 @@ module ::RSpec
     @spec_root ||= Pathname.new(__dir__)
   end
 end
+
+def asset_dir
+  File.expand_path(File.join(File.dirname(__FILE__), "assets"))
+end
+
+def read_asset(path_to_file)
+  File.read(File.expand_path(File.join(asset_dir, path_to_file)))
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0
 platform: ruby
 authors:
 - René Sprotte
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-15 00:00:00.000000000 Z
+date: 2015-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,6 +52,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.2.2
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: net-scp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -72,6 +114,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.6.0
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -114,6 +170,7 @@ files:
 - License.txt
 - Rakefile
 - Readme.md
+- bin/console
 - bin/metacrunch
 - lib/metacrunch.rb
 - lib/metacrunch/cli.rb
@@ -122,14 +179,26 @@ files:
 - lib/metacrunch/cli/command_registry.rb
 - lib/metacrunch/cli/main.rb
 - lib/metacrunch/command.rb
+- lib/metacrunch/file.rb
+- lib/metacrunch/file/reader.rb
+- lib/metacrunch/file/reader/file_system_fetcher.rb
+- lib/metacrunch/file/reader/plain_file_reader.rb
+- lib/metacrunch/file/reader/scp_fetcher.rb
+- lib/metacrunch/file/reader/tar_file_reader.rb
+- lib/metacrunch/file/reader/zip_file_reader.rb
 - lib/metacrunch/file_reader.rb
 - lib/metacrunch/file_reader_entry.rb
 - lib/metacrunch/file_writer.rb
+- lib/metacrunch/hash.rb
 - lib/metacrunch/parallel.rb
+- lib/metacrunch/processor.rb
 - lib/metacrunch/snr.rb
 - lib/metacrunch/snr/field.rb
 - lib/metacrunch/snr/section.rb
 - lib/metacrunch/tar_writer.rb
+- lib/metacrunch/transformator.rb
+- lib/metacrunch/transformator/transformation.rb
+- lib/metacrunch/transformator/transformation/step.rb
 - lib/metacrunch/transformer.rb
 - lib/metacrunch/transformer/helper.rb
 - lib/metacrunch/transformer/step.rb
@@ -137,10 +206,24 @@ files:
 - metacrunch.gemspec
 - spec/assets/archive.tar
 - spec/assets/archive.tar.gz
+- spec/assets/file/some_file
+- spec/assets/file/some_file.gz
+- spec/assets/file/tar_archive.tar
+- spec/assets/file/tar_gz_archive.tar.gz
+- spec/assets/file/tgz_archive.tgz
+- spec/assets/file/zip_archive.zip
 - spec/assets/regular_file.txt
 - spec/assets/regular_file.txt.gz
+- spec/assets/sql_lite_config.rb
 - spec/file_reader_spec.rb
 - spec/file_writer_spec.rb
+- spec/metacrunch/file/reader/plain_file_reader_spec.rb
+- spec/metacrunch/file/reader/tar_reader_spec.rb
+- spec/metacrunch/file/reader/zip_reader_spec.rb
+- spec/metacrunch/file/reader_spec.rb
+- spec/metacrunch/file_spec.rb
+- spec/metacrunch/hash_spec.rb
+- spec/metacrunch/transformator/transformation_spec.rb
 - spec/snr_spec.rb
 - spec/spec_helper.rb
 - spec/tar_writer_spec.rb
@@ -164,17 +247,31 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Data processing toolkit for Ruby
 test_files:
 - spec/assets/archive.tar
 - spec/assets/archive.tar.gz
+- spec/assets/file/some_file
+- spec/assets/file/some_file.gz
+- spec/assets/file/tar_archive.tar
+- spec/assets/file/tar_gz_archive.tar.gz
+- spec/assets/file/tgz_archive.tgz
+- spec/assets/file/zip_archive.zip
 - spec/assets/regular_file.txt
 - spec/assets/regular_file.txt.gz
+- spec/assets/sql_lite_config.rb
 - spec/file_reader_spec.rb
 - spec/file_writer_spec.rb
+- spec/metacrunch/file/reader/plain_file_reader_spec.rb
+- spec/metacrunch/file/reader/tar_reader_spec.rb
+- spec/metacrunch/file/reader/zip_reader_spec.rb
+- spec/metacrunch/file/reader_spec.rb
+- spec/metacrunch/file_spec.rb
+- spec/metacrunch/hash_spec.rb
+- spec/metacrunch/transformator/transformation_spec.rb
 - spec/snr_spec.rb
 - spec/spec_helper.rb
 - spec/tar_writer_spec.rb