transformator 1.0.0.pre1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 28e94ca52f9dd55df7e51dfed2433523c44e98d0
-  data.tar.gz: 904f122a44e0d63526d056cf027a4331348f32a3
+  metadata.gz: 4833b5ce9ba4c2b367a97b9bc745050098603911
+  data.tar.gz: 212cd88f4bfb6e2cc8b43ec8b81b20e46a6ee191
 SHA512:
-  metadata.gz: b289c83d07d9c9a33c95e8612907479bed4b693a83b2873279411d4635d675dc64f2067acab65f7321474f4382ff43fc27ff3c63e45401ed271d9dffa35150f2
-  data.tar.gz: c769832a6ee38567c1e4b1a0189947c1a9df0467b91850073b10a915883beb970fb2aab75b5a567ab333e5ead0e0e15737f66c7d85778b2afb01a4402dbc9ac2
+  metadata.gz: 65f9078502c79144bb44b3216a05b71a567e93297d8fe35091cdd1e53ab08dafff637777a7a6a1adb96d12e0ef51881ae8bb47ab53847f9b2a53c629967ccb08
+  data.tar.gz: 96090ba140428d845d58cd822a443f25cb5d6fcce4b80c1166da4ff0f3b17203719cf2e79482934e851ae9645926f04d7099f9d4c33cdbe4273983814fa1c773

data/Gemfile

@@ -1,14 +1,19 @@
 source "https://rubygems.org"
 
-# Specify your gem's dependencies in xml_transformer.gemspec
+# Specify your gem's dependencies in your gemspec
 gemspec
 
-gem "pry",                "~> 0.9.12.6"
-gem "pry-syntax-hacks",   "~> 0.0.6"
+if !ENV["CI"]
+  group :development do
+    gem "hashdiff"
+    gem "pry",                "~> 0.9.12.6"
+    gem "pry-byebug",         "<= 1.3.2"
+    gem "pry-rescue",         "~> 1.4.1"
+    gem "pry-stack_explorer", "~> 0.4.9.1"
+    gem "pry-syntax-hacks",   "~> 0.0.6"
+  end
+end
 
-if RUBY_ENGINE == "ruby"
-  gem "pry-byebug",         "<= 1.3.2"
-  gem "pry-stack_explorer", "~> 0.4.9.1"
-else
-  #gem "pry-nav",            "~> 0.2.4"
+group :test do
+  gem "codeclimate-test-reporter", require: nil
 end

data/bin/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "transformator"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/braindump.md

@@ -0,0 +1,32 @@
+- Mehrstufiger Lademechanismus (konkret 2 stufig)
+  - implizierter cli Parameter --bundle-transformations
+  - um das original Gemfile nicht zu ändern, könnte man es nach Gemfile_transformations kopieren
+    - Gemfile_transformations(.lock) müssen in .gitignore
+  - in Gemfile_transformations werden nun die Transformations Gems/Pfade/gits angehängt
+  - darauf folgende wird gebundelt
+  - anschließend wird der aktuelle Prozess durch einen Aufruf folgender Form *ersetzt*
+    BUNDLE_GEMFILE=Gemfile.transformations bundle exec bin/transformator --no-bundle-transformations ... (oder so ähnlich)
+    
+- Möglichkeit mehrere Transformationen in Reihe zu schalten
+  - dazu müsste lediglich innerhalb des ForkingEnumProzessors mehrere Transformationsscritte nacheinander durchgeführt werden
+  
+- da jede Transformation auf die source/target URI zugreifen kann, könnten hierrüber auch Reader/writer realisiert werden
+- die Frage ist, inwieweit man die einzelnen Reader/writer in einzelne Gems aufsplittet
+  - ein pragmatischer Kompromiss wäre, dass die Reader/Writer die oft benutzt werden (Elasticsearch, Postgres, Tar) Bestandteil
+    des Transformator Gems sind/bleiben
+  - man hat darüber hinaus aber die Möglichkeit,  Reader/Writer über "Transformationen" hinzuzufügen
+
+- cool wäre, wenn man Files (Tar oder plain Files) über ssh URIs spezifizieren könnte
+  - transformator -s ssh://msievers@node1/~/mab/archive1.tgz
+- die frage ist, wie man dies intern realisiert
+  - die einfache möglichkeit wäre zunächst einmal, den TarReader aufzubohren, damit dieser diesen Fall kennt/untersützt
+    - das hätte jedoch den nachteil, dass jeder reader den ssh Kram erneut für sich (implementieren) bzw. behandeln müsste
+  - die zweite Möglichkeit wäre, einen SSH Reader zu haben, welcher vor den TarReader geschaltet wird
+    - dieser könnte bsp. das File nach /tmp Laden und anschließend den TarReader damit starten
+    - Schwierig hierbei wäre, die Kontinuität des TarReaders (Enumerators) sicherzustellen, wenn diesem ein anderer Reader vorgeschaltet wäre
+    - dies könnte wieder über Enumeratoren gelingen
+      - ssh_reader.each -> gibt dateinamen im lokalen fs zurück
+      - tar_reader call ss_reader.each, liest jeweils ein file und enumeriert über die darin befindlichen einträge
+      - sobald keine einträge mehr vorhanden sind, bekommt er den nächsten dateinamen vom ssh reader usw.
+
+- neben den CLI Parametern sollte es möglich sein, ein YAML Config File anzugeben, welches vorallem bei koomplexeren Settings das Handling erleichtert

data/lib/transformator/transformation/step.rb

@@ -0,0 +1,45 @@
+require_relative "../transformation"
+
+class Transformator::Transformation::Step
+  attr_accessor :transformation
+
+  def initialize(transformation = nil, options = {})
+    if transformation.is_a?(Hash)
+      options = transformation
+      transformation = nil
+    end
+
+    if transformation
+      @transformation = transformation
+    else
+      @transformation = Struct.new(:source, :target).new.tap do |_struct|
+        _struct.source = options[:source]
+        _struct.target = options[:target]
+      end
+    end
+  end
+
+  def call
+  end
+
+  #
+  # Each step has transparent access to all methods of it's transformation
+  #
+  def method_missing(method_name, *args, &block)
+    if @transformation.respond_to?(method_name)
+      @transformation.send(method_name, *args, &block)
+    else
+      super
+    end
+  end
+
+  def respond_to_missing?(method_name, include_private = false)
+    @transformation.respond_to?(method_name) || super
+  end
+
+  # avoid method_missing penalty for the most used transformation methods
+  def source;         @transformation.source;         end
+  def source=(value); @transformation.source=(value); end
+  def target;         @transformation.target;         end
+  def target=(value); @transformation.target=(value); end
+end

data/lib/transformator/transformation.rb

@@ -0,0 +1,48 @@
+require_relative "../transformator"
+
+class Transformator::Transformation
+  require_relative "./transformation/step"
+
+  attr_accessor :source
+  attr_accessor :target
+
+  class << self
+    def steps(value = nil)
+      unless value
+        @steps
+      else
+        @steps = value
+      end
+    end
+    alias_method :sequence, :steps
+  end
+
+  def self.call(*args)
+    new.call(*args)
+  end
+
+  # since a transformation can have many steps, writing a "require" for each is tedious
+  def self.require_directory(directory)
+    Dir.glob("#{File.expand_path(directory)}/*.rb").each do |_filename|
+      require _filename
+    end
+  end
+
+  def initialize
+    # steps are instanced once, which means that instance variables retain
+    @steps = self.class.steps.flatten.map do |_step|
+      _step.is_a?(Class) ? _step.new(self) : _step
+    end
+  end
+
+  def call(source, options = {})
+    @source = source
+    @target = options[:target]
+
+    @steps.each do |_step|
+      _step.is_a?(Proc) ? instance_exec(&_step) : _step.call
+    end
+
+    return @target
+  end
+end

data/lib/transformator/version.rb

@@ -1,3 +1,3 @@
 module Transformator
-  VERSION = "1.0.0.pre1"
+  VERSION = "1.0.0"
 end

data/lib/transformator.rb

@@ -1,17 +1,7 @@
+require "active_support"
+require "active_support/core_ext"
 require "transformator/version"
 
 module Transformator
-  require_relative "./transformator/cli"
-  require_relative "./transformator/dispatcher"
-  require_relative "./transformator/filesystem_pattern_evaluator"
-  #require_relative "./transformator/gzip_reader"
-  #require_relative "./transformator/gzip_writer"
-  require_relative "./transformator/oga_xml_dumper"
-  require_relative "./transformator/oga_xml_parser"
-  #require_relative "./transformator/ox_xml_dumper"
-  #require_relative "./transformator/ox_xml_parser"
-  require_relative "./transformator/parallel_processor"
-  require_relative "./transformator/null_processor"
-  require_relative "./transformator/tar_reader"
-  #require_relative "./transformator/tar_writer"
+  require_relative "./transformator/transformation"
 end

data/spec/spec_helper.rb

@@ -1,6 +1,14 @@
+if ENV["CODECLIMATE_REPO_TOKEN"]
+  require "codeclimate-test-reporter"
+  CodeClimate::TestReporter.start
+else
+  require "simplecov"
+  SimpleCov.start
+end
+
 require "transformator"
+require "yaml"
 
-# require pry if possible
 begin
   require "pry"
 rescue LoadError
@@ -17,3 +25,7 @@ RSpec.configure do |config|
   end
   # end --- rspec 3.1 generator
 end
+
+def read_asset(path_to_file)
+  File.read(File.expand_path(File.join(File.dirname(__FILE__), "assets", path_to_file)))
+end

data/spec/transformator/transformation_spec.rb

@@ -0,0 +1,32 @@
+require "transformator/transformation"
+require "transformator/transformation/step"
+
+describe Transformator::Transformation do
+  class SomeStep < Transformator::Transformation::Step
+    def call
+      self.target = { muff: 1 }
+    end
+  end
+
+  class SomeTransformation < Transformator::Transformation
+    sequence [
+      SomeStep
+    ]
+  end
+
+  it "foo" do
+    result = SomeTransformation.call({
+      my: {
+        awesome: {
+          source: {
+            title: "foo",
+            description: "bar"
+          }
+        }
+      }
+    })
+
+    binding.pry
+    result
+  end
+end

data/transformator.gemspec

@@ -16,10 +16,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "oga",    ">= 0.2.0"
-  #spec.add_dependency "ox"
-  spec.add_dependency "rubysl", "~> 2.1.0" if RUBY_ENGINE == "rbx"
-  spec.add_dependency "thread", "~> 0.1.5"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "rubysl", ">= 2.1.0" if RUBY_ENGINE == "rbx"
 
   spec.add_development_dependency "benchmark-ips", ">= 2.0.0"
   spec.add_development_dependency "bundler",       "~> 1.7"

metadata

@@ -1,43 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: transformator
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre1
+  version: 1.0.0
 platform: ruby
 authors:
 - Michael Sievers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-01 00:00:00.000000000 Z
+date: 2015-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: oga
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
-- !ruby/object:Gem::Dependency
-  name: thread
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.1.5
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.1.5
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: benchmark-ips
   requirement: !ruby/object:Gem::Requirement
@@ -117,7 +103,8 @@ dependencies:
 description: 
 email: 
 executables:
-- transformator
+- console
+- setup
 extensions: []
 extra_rdoc_files: []
 files:
@@ -127,28 +114,16 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/transformator
+- bin/console
+- bin/setup
+- braindump.md
 - lib/transformator.rb
-- lib/transformator/cli.rb
-- lib/transformator/dispatcher.rb
-- lib/transformator/filesystem_pattern_evaluator.rb
-- lib/transformator/gzip_reader.rb
-- lib/transformator/gzip_writer.rb
-- lib/transformator/null_processor.rb
-- lib/transformator/oga_xml_dumper.rb
-- lib/transformator/oga_xml_parser.rb
-- lib/transformator/ox_xml_dumper.rb
-- lib/transformator/ox_xml_parser.rb
-- lib/transformator/parallel_processor.rb
-- lib/transformator/processor.rb
-- lib/transformator/tar_reader.rb
-- lib/transformator/tar_reader/patched_rubygems_tar_reader.rb
-- lib/transformator/tar_writer.rb
+- lib/transformator/transformation.rb
+- lib/transformator/transformation/step.rb
 - lib/transformator/version.rb
 - spec/spec_helper.rb
-- spec/transformator_spec.rb
+- spec/transformator/transformation_spec.rb
 - transformator.gemspec
-- ubpb.yml.erb
 homepage: https://github.com/ubpb/transformator
 licenses:
 - MIT
@@ -164,15 +139,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: A data transformation framework
 test_files:
 - spec/spec_helper.rb
-- spec/transformator_spec.rb
+- spec/transformator/transformation_spec.rb

data/bin/transformator

@@ -1,4 +0,0 @@
-#!/usr/bin/env ruby
-require "pry"
-require "transformator/cli"
-Transformator::Cli.new(ARGV)

data/lib/transformator/cli.rb

@@ -1,42 +0,0 @@
-require "erb"
-require "optparse"
-require "transformator"
-require "yaml"
-
-require "pry"  # TODO: remove
-
-class Transformator::Cli
-  def initialize(argv = [])
-    if argv.empty?
-      puts options_parser.help
-    else
-      @options = parse_argv(argv) || {}
-    end
-
-    Transformator::Dispatcher.new(
-      YAML.load(
-        ERB.new(
-          File.read(
-            File.expand_path(@options[:config_file_name])
-          )
-        ).result
-      )
-    ).call
-  end
-
-  def parse_argv(argv)
-    presence({}.tap do |result|
-      OptionParser.new do |opts|
-        opts.banner = "Usage: transformator [options]"
-
-        opts.on( "-c", "--config-file FILE", "Configuration file in yaml format" ) do |config_file_name|
-          result[:config_file_name] = config_file_name
-        end
-      end.parse(argv)
-    end)
-  end
-
-  def presence(object)
-    (object.empty? rescue false) ? nil : object 
-  end
-end

data/lib/transformator/dispatcher.rb

@@ -1,43 +0,0 @@
-require "transformator"
-
-class Transformator::Dispatcher
-  class State
-    attr_accessor :finished
-    attr_accessor :result
-
-    def initialize(initial_result)
-      @initial_result = initial_result
-      reset_result!
-    end
-
-    def finished?
-      @finished == true
-    end
-
-    def reset_result!
-      @result = @initial_result.dup
-    end
-  end
-
-  def initialize(processors = [])
-    @processors = processors.map do |processor|
-      if processor["options"]
-        processor["class"].new(processor["options"])
-      else
-        processor["class"].new
-      end
-    end
-
-    @state = State.new([])
-  end
-
-  def call
-    until @state.finished?
-      @processors.each do |processor|
-        processor.call!(@state)
-      end
-
-      @state.reset_result!
-    end
-  end
-end

data/lib/transformator/filesystem_pattern_evaluator.rb

@@ -1,14 +0,0 @@
-require "transformator/processor"
-
-class Transformator::FilesystemPatternEvaluator < Transformator::Processor
-  def initialize(options = {})
-    @patterns = options["patterns"]
-  end
-
-  def call!(state)
-    state.result = @patterns.map do |pattern|
-      Dir.glob(File.expand_path(pattern))
-    end.flatten
-    binding.pry
-  end
-end

data/lib/transformator/null_processor.rb

@@ -1,9 +0,0 @@
-require "transformator/processor"
-
-class Transformator::NullProcessor < Transformator::Processor
-  def call!(state)
-    puts "null"
-    #binding.pry
-    #state.finished = true
-  end
-end

data/lib/transformator/oga_xml_dumper.rb

@@ -1,10 +0,0 @@
-require "oga"
-require "transformator/processor"
-
-class Transformator::OgaXmlDumper < Transformator::Processor
-  def call!(state)
-    state.result.map! do |oga_xml_document|
-      oga_xml_document.to_xml
-    end
-  end
-end

data/lib/transformator/oga_xml_parser.rb

@@ -1,10 +0,0 @@
-require "oga"
-require "transformator/processor"
-
-class Transformator::OgaXmlParser < Transformator::Processor
-  def call!(state)
-    state.result.map! do |xml|
-      Oga.parse_xml(xml)
-    end
-  end
-end

data/lib/transformator/ox_xml_dumper.rb

@@ -1,10 +0,0 @@
-require "ox"
-require "transformator/processor"
-
-class Transformator::OxXmlDumper < Transformator::Processor
-  def call!(state)
-    state.result.map! do |ox_xml_document|
-      Ox.dump(ox_xml_document)
-    end
-  end
-end

data/lib/transformator/ox_xml_parser.rb

@@ -1,10 +0,0 @@
-require "ox"
-require "transformator/processor"
-
-class Transformator::OxXmlParser < Transformator::Processor
-  def call!(state)
-    state.result.map! do |xml|
-      Ox.parse(xml)
-    end
-  end
-end

data/lib/transformator/parallel_processor.rb

@@ -1,39 +0,0 @@
-require "transformator/processor"
-require "thread/pool"
-
-class Transformator::ParallelProcessor < Transformator::Processor
-  def initialize(options = {})
-    @processors = options["processors"].map do |processor|
-      if processor["options"]
-        processor["class"].new(processor["options"])
-      else
-        processor["class"].new
-      end
-    end
-
-    @number_of_workers = options["workers"]
-    @thread_pool = Thread.pool(@number_of_workers)
-  end
-
-  def call!(state)
-    if state.result.length > 0
-      chunk_size_per_worker = state.result.length.fdiv(@number_of_workers).ceil
-      results_enumerator = state.result.each_slice(chunk_size_per_worker)
-
-      chunks = Array.new(@number_of_workers).map do |_|
-        results_enumerator.next
-      end
-
-      @number_of_workers.times do |index|
-        @thread_pool.process do
-          @processors.each do |processor|
-            processor.call!(Struct.new(:result).new(chunks[index]))
-          end
-        end
-      end
-
-      @thread_pool.wait_done
-      state.result = chunks.inject(&:concat)
-    end
-  end
-end

data/lib/transformator/processor.rb

@@ -1,4 +0,0 @@
-require "transformator"
-
-class Transformator::Processor
-end

data/lib/transformator/tar_reader/patched_rubygems_tar_reader.rb

@@ -1,41 +0,0 @@
-#--
-# Copyright (C) 2004 Mauricio Julio Fernández Pradier
-# See LICENSE.txt for additional licensing information.
-#++
-require "rubygems/package"
-
-class Transformator::TarReader::PatchedRubygemsTarReader < Gem::Package::TarReader
-  def each
-    return enum_for __method__ unless block_given?
-
-    until @io.eof? do
-      header = Gem::Package::TarHeader.from @io
-      return if header.empty?
-
-      entry = Gem::Package::TarReader::Entry.new header, @io
-      size = entry.header.size
-
-      yield entry.read
-
-      skip = (512 - (size % 512)) % 512
-      pending = size - entry.bytes_read
-
-      begin
-        # avoid reading...
-        @io.seek pending, IO::SEEK_CUR
-        pending = 0
-      rescue Errno::EINVAL, NameError
-        while pending > 0 do
-          bytes_read = @io.read([pending, 4096].min).size
-          raise UnexpectedEOF if @io.eof?
-          pending -= bytes_read
-        end
-      end
-
-      @io.read skip # discard trailing zeros
-
-      # make sure nobody can use #read, #getc or #rewind anymore
-      entry.close
-    end
-  end
-end

data/lib/transformator/tar_reader.rb

@@ -1,51 +0,0 @@
-require "transformator/processor"
-
-class Transformator::TarReader < Transformator::Processor
-  require_relative "./tar_reader/patched_rubygems_tar_reader"
-
-  attr_accessor :files
-
-  def initialize(options = {})
-    @bulk_size = options["bulk_size"]
-    @current_archive_enumerator = nil
-    
-    if options["files"]
-      @files = ensure_array(options["files"]).map do |filename|
-        Dir.glob(File.expand_path(filename))
-      end.flatten
-    end
-  end
-
-  def call!(state)
-    @files = state.result.slice!(0..-1) unless @files
-
-    if @current_archive_enumerator.nil?
-      if !@files.empty?
-        @current_archive_enumerator = PatchedRubygemsTarReader.new(
-          Zlib::GzipReader.open(
-            File.expand_path(@files.pop)
-          )
-        ).each
-      else
-        state.finished = true
-      end
-    end
-
-    @bulk_size.times do
-      begin
-        if @current_archive_enumerator
-          state.result.push @current_archive_enumerator.next
-        end
-      rescue StopIteration
-        @current_archive_enumerator = nil
-      end
-    end
-  end
-
-  #
-  private
-  #
-  def ensure_array(object)
-    object.is_a?(Array) ? object : [object]
-  end
-end

data/spec/transformator_spec.rb

@@ -1,2 +0,0 @@
-describe Transformator do
-end

data/ubpb.yml.erb

@@ -1,25 +0,0 @@
----
-#- class: !ruby/class "Transformator::FilesystemPatternEvaluator"
-#  options:
-#    patterns:
-#    - ~/github/mab*.tar.gz
-
-- class: !ruby/class "Transformator::TarReader"
-  options:
-    bulk_size: 320
-    files:
-    - ~/github/mab*.tar.gz
-
-- class: !ruby/class "Transformator::ParallelProcessor"
-  options:
-    workers: 2
-    processors:
-    - class: !ruby/class "Transformator::OgaXmlParser"
-    - class: !ruby/class "Transformator::OgaXmlDumper"
-
-- class: !ruby/class "Transformator::NullProcessor"
-#- class: !ruby/class "Transformator::TarWriter"
-#  options:
-#    archive_size: 5000
-#    gzip_archive: true
-#    filename: /tmp/out.tar.gz