data_collector 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8410e7e260bf1989dfee0d4279b4c7d57dcd6276a5219c3bfb48663659044436
+  data.tar.gz: 0d0e78c365f927287ea7875392e83ac946da4e9ca8a764e86dc34d0e03e195b5
+SHA512:
+  metadata.gz: 72ed7c28c8f9513be5ff136827dcf07336b1fee931c17049448c8c915530e919a278dd6e478119dac3c8fb1c20e98cce1314cdbd33dd71110cb07e0fc7ab7286
+  data.tar.gz: 20ea6369d094f13ce763305369037591a5c3412562fe350fccd58b9f4de8bfd5d37b830e49214d835c2ef039ca2252bac001ea41f571bde2ab51302e3d4c3146

data/.gitignore

@@ -0,0 +1,11 @@
+/.idea/
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.1
+before_install: gem install bundler -v 2.0.1

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in data_collector.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Mehmet Celik
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,174 @@
+# DataCollector
+Convinience module to Extract, Transform and Load your data.
+
+#### input    
+Read input from an URI
+example:
+```ruby  
+    input.from_uri("http://www.libis.be")
+    input.from_uri("file://hello.txt")
+```
+
+Inputs can be JSON, XML or CSV
+
+ ### output 
+Output is an object you can store data that needs to be written to an output stream.  
+```ruby  
+    output[:name] = 'John'
+    output[:last_name] = 'Doe'
+```    
+
+Write output to a file, string use an ERB file as a template
+example:
+___test.erb___
+```ruby
+<names>
+    <combined><%= data[:name] %> <%= data[:last_name] %></combined>
+    <%= print data, :name, :first_name %>
+    <%= print data, :last_name %>
+</names>
+```
+will produce
+```ruby
+   <names>
+     <combined>John Doe</combined>
+     <first_name>John</first_name>
+     <last_name>Doe</last_name>
+   </names>
+```
+
+Into a variable
+```ruby
+    result = output.to_s("test.erb")
+```  
+
+Into a file stored in records dir
+```ruby
+    output.to_file("test.erb")
+``` 
+
+Into a tar file stored in data
+```ruby
+    output.to_file("test.erb", "my_data.tar.gz")
+```    
+
+Other output methods
+```ruby
+output.raw
+output.clear
+output.to_tmp_file("test.erb", "tmp_data")
+output.to_jsonfile(data, "test")
+output.flatten
+```
+
+Into a temp directory
+```ruby
+    output.to_tmp_file("test.erb","directory")
+```    
+   
+#### filter
+filter data from a hash using [JsonPath](http://goessner.net/articles/JsonPath/index.html)
+
+```ruby
+    filtered_data = filter(data, "$..metadata.record")
+```
+
+#### config
+config is an object that points to "config.yml" you can read and/or store data to this object.
+
+___read___    
+```ruby
+    config[:active]
+```    
+___write___
+```ruby
+    config[:active] = false
+```    
+#### log
+Log to stdout
+```ruby
+    log("hello world")
+```
+
+
+## Example
+Input data ___test.csv___
+```csv
+sequence, data
+1, apple
+2, banana
+3, peach
+```
+
+Output template ___test.erb___
+```ruby
+ <data>
+ <% data[:record].each do |d| %>
+    <record sequence="<%= d[:sequence] %>">
+      <%= print d, :data %>
+    </record>
+ <% end %>
+</data>
+```
+
+```ruby
+require 'data_collector'
+include DataCollector::Core
+
+data = input.from_uri('file://test.csv')
+data.map{ |m| m[:sequence] *=2; m }
+
+output[:record]=data
+
+puts output.to_s('test.erb')
+```
+
+Should give as output
+```xml
+<data>
+    <record sequence="11">
+        <data> apple</data>    
+    </record>
+    <record sequence="22">
+        <data> banana</data>    
+    </record>
+    <record sequence="33">
+        <data> peach</data>    
+    </record>
+</data>
+```
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'data_collector'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install data_collector
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/data_collector.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "data_collector"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/data_collector.gemspec

@@ -0,0 +1,52 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "data_collector/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "data_collector"
+  spec.version       = DataCollector::VERSION
+  spec.authors       = ["Mehmet Celik"]
+  spec.email         = ["mehmet@celik.be"]
+
+  spec.summary       = %q{ETL library}
+  spec.description   = %q{INPUT, FILTER, OUTPUT data}
+  spec.homepage      = "https://github.com/mehmetc/data_collector"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    #spec.metadata["allowed_push_host"] = "https://github.com/mehmetc/data_collect'"
+
+    spec.metadata["homepage_uri"] = spec.homepage
+    spec.metadata["source_code_uri"] = "https://github.com/mehmetc/data_collect"
+    spec.metadata["changelog_uri"] = "https://github.com/mehmetc/data_collect"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "nokogiri", "~> 1.10"
+  spec.add_runtime_dependency "json", "~> 2.2"
+  spec.add_runtime_dependency "jsonpath", "~> 1.0"
+  spec.add_runtime_dependency "nori", "~> 2.6"
+  spec.add_runtime_dependency "http", "~> 4.1"
+  spec.add_runtime_dependency "mime-types", "~> 3.2"
+  spec.add_runtime_dependency "minitar", "= 0.8"
+  spec.add_runtime_dependency "activesupport", "~> 5.2"
+  spec.add_runtime_dependency "redis", "~> 4.1"
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

data/lib/data_collector.rb

@@ -0,0 +1,11 @@
+#encoding: UTF-8
+require 'active_support/core_ext/hash'
+require 'logger'
+
+require 'data_collector/version'
+require 'data_collector/runner'
+require 'data_collector/ext/xml_utility_node'
+
+module DataCollector
+  class Error < StandardError; end
+end

data/lib/data_collector/config_file.rb

@@ -0,0 +1,72 @@
+#encoding: UTF-8
+
+require 'yaml'
+
+module DataCollector
+  class ConfigFile
+    @config = {}
+    @config_file_path = ''
+
+    def self.version
+      '0.0.1'
+    end
+
+    def self.path
+      @config_file_path
+    end
+
+    def self.path=(config_file_path)
+      @config_file_path = config_file_path
+    end
+
+    def self.[](key)
+      init
+      @config[key]
+    end
+
+    def self.[]=(key, value)
+      init
+      @config[key] = value
+      File.open("#{path}/config.yml", 'w') do |f|
+        f.puts @config.to_yaml
+      end
+    end
+
+    def self.include?(key)
+      init
+      @config.include?(key)
+    end
+
+
+    private_class_method def self.init
+      discover_config_file_path
+      if @config.empty?
+        config = YAML::load_file("#{path}/config.yml")
+        @config = process(config)
+      end
+    end
+
+
+    private_class_method def self.discover_config_file_path
+      if @config_file_path.nil? || @config_file_path.empty?
+        if File.exist?('config.yml')
+          @config_file_path = '.'
+        elsif File.exist?("config/config.yml")
+          @config_file_path = 'config'
+        end
+      end
+    end
+
+    private_class_method def self.process(config)
+      new_config = {}
+      config.each do |k, v|
+        if config[k].is_a?(Hash)
+          v = process(v)
+        end
+        new_config.store(k.to_sym, v)
+      end
+
+      new_config
+    end
+  end
+end

data/lib/data_collector/core.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+require 'jsonpath'
+
+require_relative 'input'
+require_relative 'output'
+require_relative 'config_file'
+
+module DataCollector
+  module Core
+    # Read input from an URI
+    # example:  input.from_uri("http://www.libis.be")
+    #           input.from_uri("file://hello.txt")
+    def input
+      @input ||= DataCollector::Input.new
+    end
+
+    # Output is an object you can store data that needs to be written to an output stream
+    # output[:name] = 'John'
+    # output[:last_name] = 'Doe'
+    #
+    # Write output to a file, string use an ERB file as a template
+    # example:
+    # test.erb
+    #   <names>
+    #     <combined><%= data[:name] %> <%= data[:last_name] %></combined>
+    #     <%= print data, :name, :first_name %>
+    #     <%= print data, :last_name %>
+    #   </names>
+    #
+    # will produce
+    #   <names>
+    #     <combined>John Doe</combined>
+    #     <first_name>John</first_name>
+    #     <last_name>Doe</last_name>
+    #   </names>
+    #
+    # Into a variable
+    # result = output.to_s("test.erb")
+    # Into a file stored in records dir
+    # output.to_file("test.erb")
+    # Into a tar file stored in data
+    # output.to_file("test.erb", "my_data.tar.gz")
+    # Into a temp directory
+    # output.to_tmp_file("test.erb","directory")
+    def output
+      @output ||= Output.new
+    end
+
+    # evaluator http://jsonpath.com/
+    # uitleg http://goessner.net/articles/JsonPath/index.html
+    def filter(data, filter_path)
+      filtered = []
+      if filter_path.is_a?(Array) && data.is_a?(Array)
+        filtered = data.map {|m| m.select {|k, v| filter_path.include?(k.to_sym)}}
+      elsif filter_path.is_a?(String)
+        filtered = JsonPath.on(data, filter_path)
+      end
+
+      filtered = [filtered] unless filtered.is_a?(Array)
+      filtered = filtered.first if filtered.length == 1 && filtered.first.is_a?(Array)
+
+      filtered
+    rescue StandardError => e
+      @logger.error("#{filter_path} failed: #{e.message}")
+      []
+    end
+
+    def config
+      @config ||= ConfigFile
+    end
+
+    def log(message)
+      @logger.info(message)
+    end
+  end
+end

data/lib/data_collector/ext/xml_utility_node.rb

@@ -0,0 +1,72 @@
+require "date"
+require "time"
+require "yaml"
+require "bigdecimal"
+
+require "nori/string_with_attributes"
+require "nori/string_io_file"
+
+class Nori
+  class XMLUtilityNode
+    alias_method :old_to_hash, :to_hash
+    def to_hash
+      if @type == "file"
+        f = StringIOFile.new((@children.first || '').unpack('m').first)
+        f.original_filename = attributes['name'] || 'untitled'
+        f.content_type = attributes['content_type'] || 'application/octet-stream'
+        return { name => f }
+      end
+
+      if @text
+        t = typecast_value(inner_html)
+        t = advanced_typecasting(t) if t.is_a?(String) && @options[:advanced_typecasting]
+
+        if t.is_a?(String)
+
+          # if converter = @options[:convert_attributes_to]
+          #   intermediate = attributes.map {|k, v| converter.call(k, v) }.flatten
+          #   attributes = Hash[*intermediate]
+          # end
+
+          t = {"$text" => t}.merge(prefixed_attributes) unless attributes.empty?
+        end
+
+        return { name => t }
+      else
+        #change repeating groups into an array
+        groups = @children.inject({}) { |s,e| (s[e.name] ||= []) << e; s }
+
+        out = nil
+        if @type == "array"
+          out = []
+          groups.each do |k, v|
+            if v.size == 1
+              out << v.first.to_hash.entries.first.last
+            else
+              out << v.map{|e| e.to_hash[k]}
+            end
+          end
+          out = out.flatten
+
+        else # If Hash
+          out = {}
+          groups.each do |k,v|
+            if v.size == 1
+              out.merge!(v.first)
+            else
+              out.merge!( k => v.map{|e| e.to_hash[k]})
+            end
+          end
+          out.merge! prefixed_attributes unless attributes.empty?
+          out = out.empty? ? @options[:empty_tag_value] : out
+        end
+
+        if @type && out.nil?
+          { name => typecast_value(out) }
+        else
+          { name => out }
+        end
+      end
+    end
+  end
+end

data/lib/data_collector/input.rb

@@ -0,0 +1,172 @@
+#encoding: UTF-8
+require 'http'
+require 'open-uri'
+require 'nokogiri'
+require 'json'
+require 'nori'
+require 'uri'
+require 'logger'
+require 'cgi'
+require 'mime/types'
+require 'active_support/core_ext/hash'
+require 'zlib'
+require 'minitar'
+require 'csv'
+
+#require_relative 'ext/xml_utility_node'
+module DataCollector
+class Input
+  attr_reader :raw
+
+  def initialize
+    @logger = Logger.new(STDOUT)
+  end
+
+  def from_uri(source, options = {})
+    source = CGI.unescapeHTML(source)
+    @logger.info("Loading #{source}")
+    uri = URI(source)
+    begin
+      data = nil
+      case uri.scheme
+        when 'http'
+          data = from_http(uri, options)
+        when 'https'
+          data = from_https(uri, options)
+        when 'file'
+          data = from_file(uri, options)
+        else
+          raise "Do not know how to process #{source}"
+      end
+
+      data = data.nil? ? 'no data found' : data
+
+      if block_given?
+        yield data
+      else
+        data
+      end
+    rescue => e
+      @logger.info(e.message)
+      puts e.backtrace.join("\n")
+      nil
+    end
+  end
+
+  private
+  def from_http(uri, options = {})
+    from_https(uri, options)
+  end
+
+  def from_https(uri, options = {})
+    data = nil
+    http = HTTP
+
+    if options.keys.include?(:user) && options.keys.include?(:password)
+      user = options[:user]
+      password = options[:password]
+      http = HTTP.basic_auth(user: user, pass: password)
+    else
+      @logger.warn ("User or Password parameter not found")
+    end
+
+    http_response = http.get(escape_uri(uri))
+
+    case http_response.code
+      when 200
+        @raw = data = http_response.body.to_s
+
+        # File.open("#{rand(1000)}.xml", 'wb') do |f|
+        #   f.puts data
+        # end
+
+        file_type = file_type_from(http_response.headers)
+
+        unless options.with_indifferent_access.has_key?(:raw) && options.with_indifferent_access[:raw] == true
+          case file_type
+            when 'application/json'
+              data = JSON.parse(data)
+            when 'application/atom+xml'
+              data = xml_to_hash(data)
+            when 'text/csv'
+              data = csv_to_hash(data)
+            when 'application/xml'
+            when 'text/xml'
+              data = xml_to_hash(data)
+            else
+              data = xml_to_hash(data)
+          end
+        end
+      when 401
+        raise 'Unauthorized'
+      when 404
+        raise 'Not found'
+      else
+        raise "Unable to process received status code = #{http_response.code}"
+    end
+
+    data
+  end
+
+  def from_file(uri, options = {})
+    data = nil
+    absolute_path = File.absolute_path("#{uri.host}#{uri.path}")
+    unless options.has_key?('raw') && options['raw'] == true
+      @raw = data = File.read("#{absolute_path}")
+      case File.extname(absolute_path)
+        when '.json'
+          data = JSON.parse(data)
+        when '.xml'
+          data = xml_to_hash(data)
+        when '.gz'
+          Minitar.open(Zlib::GzipReader.new(File.open("#{absolute_path}", 'rb'))) do |i|
+            i.each do |entry|
+              data = entry.read
+            end
+          end
+          data = xml_to_hash(data)
+        when '.csv'
+          data = csv_to_hash(data)
+        else
+          raise "Do not know how to process #{uri.to_s}"
+      end
+    end
+
+    data
+  end
+
+  private
+  def xml_to_hash(data)
+    #gsub('<\/', '< /') outherwise wrong XML-parsing (see records lirias1729192 )
+    data = data.gsub /</, '< /'
+    nori = Nori.new(parser: :nokogiri, strip_namespaces: true, convert_tags_to: lambda {|tag| tag.gsub(/^@/, '_')})
+    nori.parse(data)
+    #JSON.parse(nori.parse(data).to_json)
+  end
+
+  def csv_to_hash(data)
+    csv = CSV.parse(data, headers: true, header_converters: [:downcase, :symbol])
+
+    csv.collect do |record|
+      record.to_hash
+    end
+  end
+
+  def escape_uri(uri)
+    #"#{uri.to_s.gsub(uri.query, '')}#{CGI.escape(CGI.unescape(uri.query))}"
+    uri.to_s
+  end
+
+  def file_type_from(headers)
+    file_type = 'application/octet-stream'
+    file_type = if headers.include?('Content-Type')
+                  headers['Content-Type'].split(';').first
+                else
+                  MIME::Types.of(filename_from(headers)).first.content_type
+                end
+
+    return file_type
+  end
+
+end
+end

data/lib/data_collector/output.rb

@@ -0,0 +1,203 @@
+#encoding: UTF-8
+require 'nokogiri'
+require 'erb'
+require 'date'
+require 'minitar'
+require 'zlib'
+require 'cgi'
+require 'active_support/core_ext/hash'
+require 'fileutils'
+
+module DataCollector
+  class Output
+    include Enumerable
+    attr_reader :data, :tar_file
+
+    def initialize(data = {})
+      @data = data
+      @logger = Logger.new(STDOUT)
+    end
+
+    def each
+      @data.each do |d|
+        yield d
+      end
+    end
+
+    def [](k, v = nil)
+      data[k]
+    end
+
+    def []=(k, v = nil)
+      unless v.nil?
+        if data.has_key?(k)
+          if data[k].is_a?(Array) then
+            data[k] << v
+          else
+            t = data[k]
+            data[k] = Array.new([t, v])
+          end
+        else
+          data[k] = v
+        end
+      end
+
+      data
+    end
+
+    def <<(input_data)
+      if input_data.is_a?(Hash)
+        input_data.each do |k, v|
+          self[k] = input_data[k]
+        end
+      elsif input_data.is_a?(Array)
+        data["datap"] = [] unless @data.has_key?("datap")
+        d = @data["datap"].flatten.compact
+        d += input_data
+        @data["datap"] = d.compact.flatten
+      end
+    end
+
+    def raw
+      @data
+    end
+
+    def clear
+      @data = {}
+      GC.start(full_mark: true, immediate_sweep: true)
+    end
+
+
+    def to_s(erb_file)
+      data = @data
+
+      def print(data, symbol, to_symbol = nil)
+        tag = to_symbol ? to_symbol.to_s : symbol.to_s
+
+        if data.with_indifferent_access[symbol]
+          if data.with_indifferent_access[symbol].is_a?(Array)
+            r = []
+            data.with_indifferent_access[symbol].each do |d|
+              r << "<#{tag}>#{CGI.escapeHTML(d.to_s)}</#{tag}>"
+            end
+            r.join("\n")
+          elsif data.with_indifferent_access[symbol].is_a?(Hash)
+            r = []
+            r << "<#{tag}>"
+            data.with_indifferent_access[symbol].keys.each do |k|
+              r << print(data.with_indifferent_access[symbol], k)
+            end
+            r << "</#{tag}>"
+            r.join("\n")
+          else
+            "<#{tag}>#{CGI.escapeHTML(data.with_indifferent_access[symbol].to_s)}</#{tag}>"
+          end
+        else
+          nil
+        end
+      rescue Exception => e
+        @logger.error("unable to print data '#{symbol}'")
+      end
+
+      def no_tag_print(data, symbol)
+        if data.with_indifferent_access[symbol]
+          if data.with_indifferent_access[symbol].is_a?(Array)
+            r = []
+            data.with_indifferent_access[symbol].each do |d|
+              r << "#{CGI.escapeHTML(d.to_s)}"
+            end
+            r.join(",\n")
+          else
+            "#{CGI.escapeHTML(data.with_indifferent_access[symbol].to_s)}"
+          end
+        else
+          nil
+        end
+      rescue Exception => e
+        @logger.error("unable to print (without tag) data '#{symbol}'")
+      end
+
+      data[:response_date] = DateTime.now.xmlschema
+
+      result = ERB.new(File.read(erb_file), 0, '>').result(binding)
+
+      result
+    rescue Exception => e
+      raise "unable to transform to text: #{e.message}"
+      ""
+    end
+
+    def to_tmp_file(erb_file, records_dir)
+      id = data[:id].first rescue 'unknown'
+      result = to_s(erb_file)
+      xml_result = Nokogiri::XML(result, nil, 'UTF-8') do |config|
+        config.noblanks
+      end
+
+      unless File.directory?(records_dir)
+        FileUtils.mkdir_p(records_dir)
+      end
+
+      file_name = "#{records_dir}/#{id}_#{rand(1000)}.xml"
+
+      File.open(file_name, 'wb:UTF-8') do |f|
+        f.puts xml_result.to_xml
+      end
+      return file_name
+    end
+
+    def to_file(erb_file, tar_file_name = nil)
+      id = data[:id].first rescue 'unknown'
+      result = to_s(erb_file)
+
+      xml_result = Nokogiri::XML(result, nil, 'UTF-8') do |config|
+        config.noblanks
+      end
+
+      if tar_file_name.nil?
+        file_name = "records/#{id}_#{rand(1000)}.xml"
+        File.open(file_name, 'wb:UTF-8') do |f|
+          f.puts xml_result.to_xml
+        end
+
+        return file_name
+      else
+
+        Minitar::Output.open(Zlib::GzipWriter.new(File.open("records/#{tar_file_name}", 'wb:UTF-8'))) do |f|
+          xml_data = xml_result.to_xml
+          f.tar.add_file_simple("#{id}_#{rand(1000)}.xml", data: xml_data, size: xml_data.size, mtime: Time.now.to_i)
+        end
+
+        return tar_file_name
+      end
+
+    rescue Exception => e
+      raise "unable to save to file: #{e.message}"
+    end
+
+
+    def to_jsonfile (jsondata, jsonfile)
+      file_name = "records/#{jsonfile}_#{Time.now.to_i}_#{rand(1000)}.json"
+      File.open(file_name, 'wb') do |f|
+        f.puts jsondata.to_json
+      end
+    rescue Exception => e
+      raise "unable to save to jsonfile: #{e.message}"
+    end
+
+    def flatten()
+      out = Hash.new
+      @data.each do |m|
+        out[m[0]] = m[1]
+      end
+      out
+    end
+
+
+    private
+
+    def tar_file(tar_file_name)
+      @tar_file ||= Minitar::Output.open(File.open("records/#{tar_file_name}", "a+b"))
+    end
+  end
+end

data/lib/data_collector/runner.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+require 'logger'
+require_relative 'core'
+
+module DataCollector
+  class Runner
+    def initialize(logger = Logger.new(STDOUT))
+      Encoding.default_external = 'UTF-8'
+      @logger = logger
+    end
+
+    def run(rule_file_name = nil)
+      @time_start = Time.now
+      prg = self
+      if block_given?
+        a = Class.new do
+          include DataCollector::Core
+        end.new
+
+        yield a
+      elsif !rule_file_name.nil?
+        prg.instance_eval(File.read(rule_file_name))
+      else
+        @logger.error('Please supply a block or file')
+      end
+
+      prg
+    rescue Error => e
+      puts e.message
+      puts e.backtrace.join("\n")
+    ensure
+#    output.tar_file.close unless output.tar_file.closed?
+      @logger.info("Finished in #{((Time.now - @time_start)*1000).to_i} ms")
+    end
+
+    private
+    include DataCollector::Core
+  end
+end

data/lib/data_collector/version.rb

@@ -0,0 +1,4 @@
+# encoding: utf-8
+module DataCollector
+  VERSION = "0.1.1"
+end

metadata

@@ -0,0 +1,231 @@
+--- !ruby/object:Gem::Specification
+name: data_collector
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Mehmet Celik
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-07-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: jsonpath
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: nori
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: minitar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: INPUT, FILTER, OUTPUT data
+email:
+- mehmet@celik.be
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- data_collector.gemspec
+- lib/data_collector.rb
+- lib/data_collector/config_file.rb
+- lib/data_collector/core.rb
+- lib/data_collector/ext/xml_utility_node.rb
+- lib/data_collector/input.rb
+- lib/data_collector/output.rb
+- lib/data_collector/runner.rb
+- lib/data_collector/version.rb
+homepage: https://github.com/mehmetc/data_collector
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/mehmetc/data_collector
+  source_code_uri: https://github.com/mehmetc/data_collect
+  changelog_uri: https://github.com/mehmetc/data_collect
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.2
+signing_key: 
+specification_version: 4
+summary: ETL library
+test_files: []