metacrunch 2.2.3 → 3.0.0

data/lib/metacrunch/file/reader/plain_file_reader.rb

@@ -1,33 +0,0 @@
-require_relative "../reader"
-
-class Metacrunch::File::Reader::PlainFileReader
-  include Enumerable
-
-  def self.accepts?(filename)
-    true
-  end
-
-  def initialize(filename)
-    @filename = filename
-  end
-
-  def each
-    return enum_for(__method__) unless block_given?
-
-    io =
-    if @filename.end_with?("gz") # catches tgz and tar.gz
-      Zlib::GzipReader.open(@filename)
-    else
-      File.open(@filename)
-    end
-
-    yield Metacrunch::File.new({
-      content: io.read,
-      entry_name: File.basename(@filename),
-      file_name: @filename,
-      is_directory: false
-    })
-
-    io.close
-  end
-end

data/lib/metacrunch/file/reader/scp_fetcher.rb

@@ -1,56 +0,0 @@
-require "etc"
-require "net/scp"
-require "net/ssh"
-require "securerandom"
-require_relative "../reader"
-
-class Metacrunch::File::Reader::ScpFetcher
-  include Enumerable
-
-  RECOGNIZED_PROTOCOL_REGEX = /\Ascp:\/\//
-  TILDE_REPLACEMENT = "/__TILDE__"
-
-  def self.accepts?(url)
-    !!url[RECOGNIZED_PROTOCOL_REGEX]
-  end
-
-  def initialize(url, options = {})
-    URI(url.sub("~", TILDE_REPLACEMENT)).try do |_uri|
-      @host = _uri.host
-      @password = options[:password] || _uri.password
-      @path = _uri.path.sub(TILDE_REPLACEMENT, "~")
-      @username = options[:username] || _uri.user || Etc.getlogin
-    end
-  end
-
-  def each
-    return enum_for(__method__) unless block_given?
-
-    begin
-      begin
-        Dir.mkdir temporary_directory = File.join(Dir.tmpdir, SecureRandom.hex)
-      rescue Errno::EEXIST
-        retry
-      end
-
-      remote_filenames.each do |_remote_filename|
-        _local_filename = File.join(temporary_directory, File.basename(_remote_filename))
-        Net::SCP.download!(@host, @username, _remote_filename, _local_filename, ssh: { password: @password })
-        yield _local_filename
-        File.delete(_local_filename)
-      end
-    ensure
-      FileUtils.remove_dir(temporary_directory)
-    end
-  end
-
-  private
-
-  def remote_filenames
-    @remote_filenames ||= [].tap do |_remote_filenames|
-      Net::SSH.start(@host, @username, password: @password) do |_ssh|
-        _remote_filenames.concat _ssh.exec!("ruby -e \"puts Dir.glob(File.expand_path('#{@path}'))\"").try(:split, "\n") || []
-      end
-    end
-  end
-end

data/lib/metacrunch/file/reader/tar_file_reader.rb

@@ -1,37 +0,0 @@
-require "rubygems/package"
-require_relative "../reader"
-
-class Metacrunch::File::Reader::TarFileReader
-  include Enumerable
-
-  def self.accepts?(filename)
-    !!filename[/\.tar\Z|\.tar\.gz\Z|\.tgz\Z/]
-  end
-
-  def initialize(filename)
-    @filename = filename
-  end
-
-  def each
-    return enum_for(__method__) unless block_given?
-
-    io =
-    if @filename.end_with?("gz") # catches tgz and tar.gz
-      Zlib::GzipReader.open(@filename)
-    else
-      File.open(@filename)
-    end
-
-    Gem::Package::TarReader.new(io).each do |_tar_entry|
-      unless _tar_entry.directory?
-        yield Metacrunch::File.new({
-          content: _tar_entry.read,
-          entry_name: _tar_entry.full_name,
-          file_name: @filename
-        })
-      end
-    end
-
-    io.close
-  end
-end

data/lib/metacrunch/file/reader/zip_file_reader.rb

@@ -1,30 +0,0 @@
-require "zip"
-require_relative "../reader"
-
-class Metacrunch::File::Reader::ZipFileReader
-  include Enumerable
-
-  def self.accepts?(filename)
-    !!filename[/\.zip\Z/]
-  end
-
-  def initialize(filename)
-    @filename = filename
-  end
-
-  def each
-    return enum_for(__method__) unless block_given?
-
-    Zip::File.open(@filename) do |_zip_file|
-      _zip_file.each do |_zip_entry|
-        unless _zip_entry.directory?
-          yield Metacrunch::File.new({
-            content: _zip_entry.get_input_stream.read,
-            entry_name: _zip_entry.name,
-            file_name: @filename
-          })
-        end
-      end
-    end
-  end
-end

data/lib/metacrunch/file/reader.rb

@@ -1,72 +0,0 @@
-require_relative "../file"
-require_relative "../processor"
-
-class Metacrunch::File::Reader < Metacrunch::Processor
-  require_relative "./reader/file_system_fetcher"
-  require_relative "./reader/plain_file_reader"
-  require_relative "./reader/scp_fetcher"
-  require_relative "./reader/tar_file_reader"
-  require_relative "./reader/zip_file_reader"
-
-  include Enumerable
-
-  def initialize(options = {})
-    @bulk_size = options[:bulk_size].try(:to_i) || 1
-    @urls = [
-      options[:filename], options[:filenames],
-      options[:url], options[:urls]
-    ]
-    .flatten
-    .compact
-    .map do |_filename_or_url|
-      if (_url = _filename_or_url)[/\A\w+:\/\//]
-        _url
-      else
-        Dir.glob(File.expand_path(_filename_or_url)).map do |_filename|
-          "file://#{_filename}"
-        end
-      end
-    end
-    .flatten # because there might be arrays again because of Dir.glob
-
-    @force_content_encoding = options[:force_content_encoding]
-    @password = options[:password]
-    @username = options[:username]
-  end
-
-  def call(items = [], pipeline = nil)
-    @chunks_of_entries_enumerator ||= each_slice(@bulk_size) # instance method from Enumerable
-   
-    begin
-      items.concat(@chunks_of_entries_enumerator.next)
-    rescue StopIteration
-      pipeline.try(:terminate!)
-    end
-  end
-
-  def each
-    return enum_for(__method__) unless block_given?
-
-    @urls.each do |_url|
-      [FileSystemFetcher, ScpFetcher].find do |_fetcher|
-        _fetcher.accepts?(_url)
-      end
-      .try do |_appropriate_fetcher|
-        _appropriate_fetcher.new(_url, username: @username, password: @password).each do |_filename|
-          [TarFileReader, ZipFileReader, PlainFileReader].find do |_reader| # PlainFileReader as last will read any file
-            _reader.accepts?(_filename)
-          end
-          .try do |_appropriate_reader|
-            _appropriate_reader.new(_filename).each do |_file|
-              if @force_content_encoding.present?
-                _file.content.try(:force_encoding, @force_content_encoding)
-              end
-
-              yield _file
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/metacrunch/file/writer/plain_file_writer.rb

@@ -1,19 +0,0 @@
-require_relative "../writer"
-
-class Metacrunch::File::Writer::PlainFileWriter
-  def self.supports?(filename)
-    true
-  end
-
-  def initialize(filename)
-    @io = File.open(filename, "w")
-  end
-
-  def close
-    @io.close
-  end
-
-  def write(options = {})
-    @io.write(options[:content])
-  end
-end

data/lib/metacrunch/file/writer/tar_file_writer.rb

@@ -1,26 +0,0 @@
-require "rubygems/package"
-require "zlib"
-require_relative "../writer"
-
-class Metacrunch::File::Writer::TarFileWriter
-  def self.supports?(filename)
-    !!filename[/\.tar\Z|\.tar\.gz\Z|\.tgz\Z/]
-  end
-
-  def initialize(filename)
-    @io = File.open(filename, "w")
-    @io = Zlib::GzipWriter.new(@io) if filename.end_with?("gz")
-    @tar_writer = Gem::Package::TarWriter.new(@io)
-  end
-
-  def close
-    @tar_writer.close
-    @io.close
-  end
-
-  def write(options = {})
-    @tar_writer.add_file_simple(options[:entry_name], 0644, options[:content].bytesize) do |_tar_entry|
-      _tar_entry.write(options[:content])
-    end
-  end
-end

data/lib/metacrunch/file/writer/zip_file_writer.rb

@@ -1,29 +0,0 @@
-require "zip"
-require_relative "../writer"
-
-class Metacrunch::File::Writer::ZipFileWriter
-  def self.supports?(filename)
-    !!filename[/\.zip\Z/]
-  end
-
-  def self.write(*args)
-  end
-
-=begin
-  def each
-    return enum_for(__method__) unless block_given?
-
-    Zip::File.open(@filename) do |_zip_file|
-      _zip_file.each do |_zip_entry|
-        unless _zip_entry.directory?
-          yield Metacrunch::File.new({
-            content: _zip_entry.get_input_stream.read,
-            entry_name: _zip_entry.name,
-            file_name: @filename
-          })
-        end
-      end
-    end
-  end
-=end
-end

data/lib/metacrunch/file/writer.rb

@@ -1,26 +0,0 @@
-require_relative "../file"
-require_relative "../processor"
-
-class Metacrunch::File::Writer < Metacrunch::Processor
-  require_relative "./writer/plain_file_writer"
-  require_relative "./writer/tar_file_writer"
-  require_relative "./writer/zip_file_writer"
-
-  def initialize(filename)
-    @writer =
-    [TarFileWriter, ZipFileWriter, PlainFileWriter].find do |_writer|
-      _writer.supports?(filename)
-    end
-    .try do |_appropriate_writer_class|
-      _appropriate_writer_class.new(filename)
-    end
-  end
-
-  def close
-    @writer.close
-  end
-
-  def write(*args)
-    @writer.write(*args)
-  end
-end

data/lib/metacrunch/file.rb

@@ -1,24 +0,0 @@
-require_relative "../metacrunch"
-
-class Metacrunch::File
-  require_relative "./file/reader"
-  require_relative "./file/writer"
-
-  attr_accessor :content
-  attr_accessor :entry_name # equals file_name for plain files
-  attr_accessor :file_name
-
-  def initialize(options = {})
-    @content = options[:content]
-    @entry_name = options[:entry_name]
-    @file_name = options[:file_name]
-  end
-
-  def to_h
-    {
-      content: @content,
-      entry_name: @entry_name,
-      file_name: @file_name
-    }
-  end
-end

data/lib/metacrunch/file_reader_entry.rb

@@ -1,21 +0,0 @@
-require_relative "./file_reader"
-
-module Metacrunch
-  class FileReader
-    class Entry
-
-      attr_reader :filename, :archive_filename, :contents
-
-      def initialize(filename:, archive_filename:nil, contents:nil)
-        @filename         = filename
-        @archive_filename = archive_filename.presence
-        @contents         = contents
-      end
-
-      def from_archive?
-        @archive_filename != nil
-      end
-
-    end
-  end
-end

data/lib/metacrunch/file_writer.rb

@@ -1,40 +0,0 @@
-module Metacrunch
-  class FileWriter
-
-    class FileExistError < RuntimeError ; end
-
-
-    def initialize(filename, override: false, compress: nil)
-      @path       = ::File.expand_path(filename)
-      @compressed = (compress ||= @path.ends_with?(".gz"))
-
-      if ::File.exist?(@path) && !override
-        raise FileExistError, "File #{@path} already exists. Set override = true to override the existing file."
-      end
-    end
-
-    def write(data, options = {})
-      if block_given?
-        yield(io)
-      else
-        io.write(data)
-      end
-    end
-
-    def flush
-      @io.flush if @io
-    end
-
-    def close
-      flush
-      @io.close if @io
-    end
-
-  private
-
-    def io
-      @io ||= (@compressed == true) ? Zlib::GzipWriter.open(@path) : ::File.open(@path, "w")
-    end
-
-  end
-end

data/lib/metacrunch/hash.rb

@@ -1,51 +0,0 @@
-require_relative "../metacrunch"
-
-module Metacrunch::Hash
-  def self.add(object, *args)
-    if args.length < 2
-      raise ArgumentError
-    else
-      return args.last if args.last.nil? || (args.last.respond_to?(:empty?) && args.last.empty?)
-
-      if args.length == 2
-        args_first_is_a_string = args.first.is_a?(String) # memoize
-
-        if args_first_is_a_string && args.first.include?("/")
-          add(object, *args.first.split("/"), args.last)
-        else
-          if args_first_is_a_string && args.first.start_with?(":")
-            _add(object, args.first[1..-1].to_sym, args.last)
-          else
-            _add(object, args.first, args.last)
-          end
-        end
-      else
-        nested_hash = args[0..-3].inject(object) do |_memo, _key|
-          if _key.is_a?(String) && _key.start_with?(":")
-            _key = _key[1..-1].to_sym
-          end
-
-          _memo[_key] ||= object.class.new
-        end
-
-        _add(nested_hash, args[-2], args[-1])
-      end
-    end
-  end
-
-  private
-
-  def self._add(hash, key, value)
-    #if value.is_a?(FalseClass) || (value.respond_to?(:empty?) ? !value.empty? : !!value) # like ActiveSupport implements blank?/present?
-    if hash[key].nil?
-      hash[key] = value.is_a?(Array) && value.length == 1 ? value.first : value
-    elsif hash[key].is_a?(Array)
-      (hash[key] << value).flatten!(1)
-    else
-      (hash[key] = [hash[key], value]).flatten!(1)
-    end
-    #end
-
-    hash[key]
-  end
-end

data/lib/metacrunch/parallel.rb

@@ -1,69 +0,0 @@
-module Metacrunch
-  class Parallel
-
-    module DSL
-      def parallel(enumerable, options = {}, &block)
-        Parallel.each(enumerable, options, &block)
-      end
-    end
-
-    def self.each(enumerable, options = {}, &block)
-      self.new(enumerable, options, &block).call
-    end
-
-    def initialize(enumerable, options = {}, &block)
-      @enumerable          = enumerable
-      @callable            = block
-      @no_of_procs         = options[:in_processes] || 0
-      @on_process_finished = options[:on_process_finished] || -> {}
-
-      unless block_given?
-        raise ArgumentError, "you must provide a block"
-      end
-
-      unless @enumerable.respond_to?(:each)
-        raise ArgumentError, "enumerable must respond to each"
-      end
-
-      unless @on_process_finished.respond_to?(:call)
-        raise ArgumentError, "on_process_finished must respond to call"
-      end
-    end
-
-    def call
-      @enumerable.each do |_value|
-        if @no_of_procs == 0
-          @callable.call(_value)
-          @on_process_finished.call
-        else
-          fork_process do
-            @callable.call(_value)
-          end
-
-          if processes_limit_reached?
-            wait_for_some_process_to_terminate
-            @on_process_finished.call
-          end
-        end
-      end
-    ensure
-      Process.waitall.each { @on_process_finished.call }
-    end
-
-  private
-
-    def fork_process(&block)
-      (@pids ||= []).push(fork(&block))
-    end
-
-    def processes_limit_reached?
-      (@pids || []).length >= @no_of_procs
-    end
-
-    def wait_for_some_process_to_terminate
-      pid_of_finished_process = Process.wait
-      @pids.delete(pid_of_finished_process)
-    end
-
-  end
-end

data/lib/metacrunch/processor.rb

@@ -1,10 +0,0 @@
-require_relative "../metacrunch"
-
-class Metacrunch::Processor
-  def initialize(options = {})
-    @options ||= options
-  end
-
-  def call(items = [], pipeline = nil)
-  end
-end

data/lib/metacrunch/snr/field.rb

@@ -1,31 +0,0 @@
-module Metacrunch
-  class SNR
-    class Section
-      class Field
-
-        attr_reader   :name
-        attr_accessor :value
-
-        def initialize(name, value)
-          raise ArgumentError, "required Field#name not given" if name.nil?
-
-          @name  = name
-          @value = value
-        end
-
-        # ------------------------------------------------------------------------------
-        # Serialization
-        # ------------------------------------------------------------------------------
-
-        def to_xml(builder)
-          if value.respond_to?(:to_xml)
-            value.to_xml(root: self.name, builder: builder, skip_instruct: true)
-          else
-            builder.tag!(self.name, self.value)
-          end
-        end
-
-      end
-    end
-  end
-end

data/lib/metacrunch/snr/section.rb

@@ -1,74 +0,0 @@
-module Metacrunch
-  class SNR
-    class Section
-
-      attr_reader :name
-
-      def initialize(name)
-        raise ArgumentError, "required Section#name not given" if name.nil?
-
-        @name   = name
-        @fields = []
-      end
-
-      # ------------------------------------------------------------------------------
-      # Common API
-      # ------------------------------------------------------------------------------
-
-      #
-      # Adds a field
-      #
-      def add(field_name, value)
-        if value.is_a?(Array)
-          value.each do |_value|
-            add_field(Field.new(field_name, _value))
-          end
-        else
-          add_field(Field.new(field_name, value))
-        end
-      end
-
-      # ------------------------------------------------------------------------------
-      # Fields
-      # ------------------------------------------------------------------------------
-
-      #
-      # Return all fields. A name can be provided to filter fields by name.
-      #
-      # @param [String, nil] name
-      # @return [Array<Metacrunch::SNR::Section::Field>]
-      #
-      def fields(name = nil)
-        if name
-          @fields.select{|field| field.name == name}
-        else
-          @fields
-        end
-      end
-
-      #
-      # Adds a new field to this section.
-      #
-      # @param [Metacrunch::SNR::Section::Field] field
-      # @return [Metacrunch::SNR::Section::Field]
-      #
-      def add_field(field)
-        @fields << field
-        field
-      end
-
-      # ------------------------------------------------------------------------------
-      # Serialization
-      # ------------------------------------------------------------------------------
-
-      def to_xml(builder)
-        builder.tag!(self.name) do
-          @fields.each do |_field|
-            _field.to_xml(builder)
-          end
-        end
-      end
-
-    end
-  end
-end