transformator 0.1.4 → 1.0.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: de79125e94963cf8f64a509a0ad23c1f33b6404e
-  data.tar.gz: bd978c37ba87a8a64c31240d48096f0a477d5b4a
+  metadata.gz: 28e94ca52f9dd55df7e51dfed2433523c44e98d0
+  data.tar.gz: 904f122a44e0d63526d056cf027a4331348f32a3
 SHA512:
-  metadata.gz: 05aa679773f11e290dab9b3de31499dfd452940b6670c0e8894689e1605f9eb04b5e2a264440a1679d6935c05006b9d782d379b23ecfb212e41db643f0385dba
-  data.tar.gz: c352e245758bbc162a87ac6c8d7b9fded4bb9f060972b7379303c23eb0696303810eb64f05488706af58253b26f37b583a8b3f13731d15c54e262152bd0bd6d4
+  metadata.gz: b289c83d07d9c9a33c95e8612907479bed4b693a83b2873279411d4635d675dc64f2067acab65f7321474f4382ff43fc27ff3c63e45401ed271d9dffa35150f2
+  data.tar.gz: c769832a6ee38567c1e4b1a0189947c1a9df0467b91850073b10a915883beb970fb2aab75b5a567ab333e5ead0e0e15737f66c7d85778b2afb01a4402dbc9ac2

data/Gemfile

@@ -4,6 +4,11 @@ source "https://rubygems.org"
 gemspec
 
 gem "pry",                "~> 0.9.12.6"
-gem "pry-byebug",         "<= 1.3.2"
-gem "pry-stack_explorer", "~> 0.4.9.1"
 gem "pry-syntax-hacks",   "~> 0.0.6"
+
+if RUBY_ENGINE == "ruby"
+  gem "pry-byebug",         "<= 1.3.2"
+  gem "pry-stack_explorer", "~> 0.4.9.1"
+else
+  #gem "pry-nav",            "~> 0.2.4"
+end

data/Rakefile

@@ -3,47 +3,4 @@ require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :benchmark do
-  require "benchmark/ips"
-  require "pry"
-  require_relative "./benchmark/primo_search_response_transformation"
-  require_relative "./lib/transformator"
-
-  Benchmark.ips do |bm|
-    bm.report("Transformator::Benchmark::PrimoSearchResponseTransformation") do
-      Transformator::Benchmark::PrimoSearchResponseTransformation.new.apply(
-        to: File.read(File.expand_path(File.join(File.dirname(__FILE__), "benchmark/primo_search_response_transformation/primo_response_1.xml"))),
-        output: :hash
-      )
-    end
-  end
-
-=begin
-  #
-  # document_from_xml
-  #
-  xml_file_name = File.join(File.dirname(__FILE__), "benchmark/primo_search_response_transformation/primo_response.xml")
-  outer_document = Transformator.document_from_xml(File.read(xml_file_name))
-
-  Benchmark.ips do |bm|
-    bm.report("Transformator.document_from_xml") do
-      Transformator.document_from_xml(outer_document.locate("*/searchBriefReturn").first.text)
-    end
-  end
-
-  #
-  # hash_from_document
-  #
-  xml_file_name = File.join(File.dirname(__FILE__), "benchmark/primo_search_response_transformation/primo_response.xml")
-  outer_document = Transformator.document_from_xml(File.read(xml_file_name))
-  inner_document = Transformator.document_from_xml(outer_document.locate("*/searchBriefReturn").first.text)
-
-  Benchmark.ips do |bm|
-    bm.report("Transformator.hash_from_document") do
-      Transformator.hash_from_document(inner_document)
-    end
-  end
-=end
-end
-
 task :default => :spec

data/bin/transformator

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require "pry"
+require "transformator/cli"
+Transformator::Cli.new(ARGV)

data/lib/transformator.rb

@@ -1,33 +1,17 @@
-require "ox"
 require "transformator/version"
 
 module Transformator
-  require_relative "./transformator/dsl"
-  require_relative "./transformator/format_converter"
-  require_relative "./transformator/transformation"
-
-  extend Transformator::FormatConverter
-
-  def self.determine_format(obj)
-    if obj.is_a?(Hash)
-      :hash
-    elsif obj.is_a?(Ox::Document)
-      :ox_document
-    elsif obj.is_a?(String) && obj[/\A\s*{/]
-      :json
-    elsif obj.is_a?(String) && obj[/\A\s*</]
-      :xml
-    elsif obj.nil?
-      nil
-    else
-      raise "Unkown format!"
-    end
-  end
-
-  def self.oxify_path(path)
-    path
-    .gsub(/\A\/\/(\S+)/, "*/\\1")
-    .gsub(/(\S*)\/\/(\S*)/, "\\1/*/\\2")
-    .gsub(/\A\/(\w+)(\S*)/, "?/\\1\\2")  # replace "/foo" with "?/foo"
-  end
+  require_relative "./transformator/cli"
+  require_relative "./transformator/dispatcher"
+  require_relative "./transformator/filesystem_pattern_evaluator"
+  #require_relative "./transformator/gzip_reader"
+  #require_relative "./transformator/gzip_writer"
+  require_relative "./transformator/oga_xml_dumper"
+  require_relative "./transformator/oga_xml_parser"
+  #require_relative "./transformator/ox_xml_dumper"
+  #require_relative "./transformator/ox_xml_parser"
+  require_relative "./transformator/parallel_processor"
+  require_relative "./transformator/null_processor"
+  require_relative "./transformator/tar_reader"
+  #require_relative "./transformator/tar_writer"
 end

data/lib/transformator/cli.rb

@@ -0,0 +1,42 @@
+require "erb"
+require "optparse"
+require "transformator"
+require "yaml"
+
+require "pry"  # TODO: remove
+
+class Transformator::Cli
+  def initialize(argv = [])
+    if argv.empty?
+      puts options_parser.help
+    else
+      @options = parse_argv(argv) || {}
+    end
+
+    Transformator::Dispatcher.new(
+      YAML.load(
+        ERB.new(
+          File.read(
+            File.expand_path(@options[:config_file_name])
+          )
+        ).result
+      )
+    ).call
+  end
+
+  def parse_argv(argv)
+    presence({}.tap do |result|
+      OptionParser.new do |opts|
+        opts.banner = "Usage: transformator [options]"
+
+        opts.on( "-c", "--config-file FILE", "Configuration file in yaml format" ) do |config_file_name|
+          result[:config_file_name] = config_file_name
+        end
+      end.parse(argv)
+    end)
+  end
+
+  def presence(object)
+    (object.empty? rescue false) ? nil : object 
+  end
+end

data/lib/transformator/dispatcher.rb

@@ -0,0 +1,43 @@
+require "transformator"
+
+class Transformator::Dispatcher
+  class State
+    attr_accessor :finished
+    attr_accessor :result
+
+    def initialize(initial_result)
+      @initial_result = initial_result
+      reset_result!
+    end
+
+    def finished?
+      @finished == true
+    end
+
+    def reset_result!
+      @result = @initial_result.dup
+    end
+  end
+
+  def initialize(processors = [])
+    @processors = processors.map do |processor|
+      if processor["options"]
+        processor["class"].new(processor["options"])
+      else
+        processor["class"].new
+      end
+    end
+
+    @state = State.new([])
+  end
+
+  def call
+    until @state.finished?
+      @processors.each do |processor|
+        processor.call!(@state)
+      end
+
+      @state.reset_result!
+    end
+  end
+end

data/lib/transformator/filesystem_pattern_evaluator.rb

@@ -0,0 +1,14 @@
+require "transformator/processor"
+
+class Transformator::FilesystemPatternEvaluator < Transformator::Processor
+  def initialize(options = {})
+    @patterns = options["patterns"]
+  end
+
+  def call!(state)
+    state.result = @patterns.map do |pattern|
+      Dir.glob(File.expand_path(pattern))
+    end.flatten
+    binding.pry
+  end
+end

data/lib/transformator/null_processor.rb

@@ -0,0 +1,9 @@
+require "transformator/processor"
+
+class Transformator::NullProcessor < Transformator::Processor
+  def call!(state)
+    puts "null"
+    #binding.pry
+    #state.finished = true
+  end
+end

data/lib/transformator/oga_xml_dumper.rb

@@ -0,0 +1,10 @@
+require "oga"
+require "transformator/processor"
+
+class Transformator::OgaXmlDumper < Transformator::Processor
+  def call!(state)
+    state.result.map! do |oga_xml_document|
+      oga_xml_document.to_xml
+    end
+  end
+end

data/lib/transformator/oga_xml_parser.rb

@@ -0,0 +1,10 @@
+require "oga"
+require "transformator/processor"
+
+class Transformator::OgaXmlParser < Transformator::Processor
+  def call!(state)
+    state.result.map! do |xml|
+      Oga.parse_xml(xml)
+    end
+  end
+end

data/lib/transformator/ox_xml_dumper.rb

@@ -0,0 +1,10 @@
+require "ox"
+require "transformator/processor"
+
+class Transformator::OxXmlDumper < Transformator::Processor
+  def call!(state)
+    state.result.map! do |ox_xml_document|
+      Ox.dump(ox_xml_document)
+    end
+  end
+end

data/lib/transformator/ox_xml_parser.rb

@@ -0,0 +1,10 @@
+require "ox"
+require "transformator/processor"
+
+class Transformator::OxXmlParser < Transformator::Processor
+  def call!(state)
+    state.result.map! do |xml|
+      Ox.parse(xml)
+    end
+  end
+end

data/lib/transformator/parallel_processor.rb

@@ -0,0 +1,39 @@
+require "transformator/processor"
+require "thread/pool"
+
+class Transformator::ParallelProcessor < Transformator::Processor
+  def initialize(options = {})
+    @processors = options["processors"].map do |processor|
+      if processor["options"]
+        processor["class"].new(processor["options"])
+      else
+        processor["class"].new
+      end
+    end
+
+    @number_of_workers = options["workers"]
+    @thread_pool = Thread.pool(@number_of_workers)
+  end
+
+  def call!(state)
+    if state.result.length > 0
+      chunk_size_per_worker = state.result.length.fdiv(@number_of_workers).ceil
+      results_enumerator = state.result.each_slice(chunk_size_per_worker)
+
+      chunks = Array.new(@number_of_workers).map do |_|
+        results_enumerator.next
+      end
+
+      @number_of_workers.times do |index|
+        @thread_pool.process do
+          @processors.each do |processor|
+            processor.call!(Struct.new(:result).new(chunks[index]))
+          end
+        end
+      end
+
+      @thread_pool.wait_done
+      state.result = chunks.inject(&:concat)
+    end
+  end
+end

data/lib/transformator/processor.rb

@@ -0,0 +1,4 @@
+require "transformator"
+
+class Transformator::Processor
+end

data/lib/transformator/tar_reader.rb

@@ -0,0 +1,51 @@
+require "transformator/processor"
+
+class Transformator::TarReader < Transformator::Processor
+  require_relative "./tar_reader/patched_rubygems_tar_reader"
+
+  attr_accessor :files
+
+  def initialize(options = {})
+    @bulk_size = options["bulk_size"]
+    @current_archive_enumerator = nil
+    
+    if options["files"]
+      @files = ensure_array(options["files"]).map do |filename|
+        Dir.glob(File.expand_path(filename))
+      end.flatten
+    end
+  end
+
+  def call!(state)
+    @files = state.result.slice!(0..-1) unless @files
+
+    if @current_archive_enumerator.nil?
+      if !@files.empty?
+        @current_archive_enumerator = PatchedRubygemsTarReader.new(
+          Zlib::GzipReader.open(
+            File.expand_path(@files.pop)
+          )
+        ).each
+      else
+        state.finished = true
+      end
+    end
+
+    @bulk_size.times do
+      begin
+        if @current_archive_enumerator
+          state.result.push @current_archive_enumerator.next
+        end
+      rescue StopIteration
+        @current_archive_enumerator = nil
+      end
+    end
+  end
+
+  #
+  private
+  #
+  def ensure_array(object)
+    object.is_a?(Array) ? object : [object]
+  end
+end

data/lib/transformator/tar_reader/patched_rubygems_tar_reader.rb

@@ -0,0 +1,41 @@
+#--
+# Copyright (C) 2004 Mauricio Julio Fernández Pradier
+# See LICENSE.txt for additional licensing information.
+#++
+require "rubygems/package"
+
+class Transformator::TarReader::PatchedRubygemsTarReader < Gem::Package::TarReader
+  def each
+    return enum_for __method__ unless block_given?
+
+    until @io.eof? do
+      header = Gem::Package::TarHeader.from @io
+      return if header.empty?
+
+      entry = Gem::Package::TarReader::Entry.new header, @io
+      size = entry.header.size
+
+      yield entry.read
+
+      skip = (512 - (size % 512)) % 512
+      pending = size - entry.bytes_read
+
+      begin
+        # avoid reading...
+        @io.seek pending, IO::SEEK_CUR
+        pending = 0
+      rescue Errno::EINVAL, NameError
+        while pending > 0 do
+          bytes_read = @io.read([pending, 4096].min).size
+          raise UnexpectedEOF if @io.eof?
+          pending -= bytes_read
+        end
+      end
+
+      @io.read skip # discard trailing zeros
+
+      # make sure nobody can use #read, #getc or #rewind anymore
+      entry.close
+    end
+  end
+end

data/lib/transformator/version.rb

@@ -1,3 +1,3 @@
 module Transformator
-  VERSION = "0.1.4"
+  VERSION = "1.0.0.pre1"
 end