xml_data_extractor 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2ea176d26d7d1e43ca91ab838b75eca58044da1af8a98675dc3c1c1ba453f313
+  data.tar.gz: b4a69acdf185a5b830ef267cbb4cd86a08562795f178d5c0f79c4b893501a3b7
+SHA512:
+  metadata.gz: 9bd4f5a9ea20e5ce63d92e9d213267eb06097f4a43e851bd65f622404a5f180e877409164ba5716d9e49dfa29ef1bb9b4cf8ff17ae269867ea74d2b040a7b85d
+  data.tar.gz: 21e37f2de2a5bc185fc2f6a30ea0367e67f9130f44a2645d53e10e1e7015e7569e47ccc7bb3e98203a12bae853bcff3d5d8ad60ff5ced36b702934a74a54bef2

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.6
+before_install: gem install bundler -v 2.1.4

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in xml_data_extractor.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "rspec", "~> 3.0"

data/Gemfile.lock

@@ -0,0 +1,53 @@
+PATH
+  remote: .
+  specs:
+    xml_data_extractor (0.1.0)
+      activesupport
+      nokogiri
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.0.3.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    concurrent-ruby (1.1.7)
+    diff-lcs (1.3)
+    i18n (1.8.5)
+      concurrent-ruby (~> 1.0)
+    mini_portile2 (2.4.0)
+    minitest (5.14.1)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    thread_safe (0.3.6)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
+    zeitwerk (2.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  xml_data_extractor!
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Fernando Almeida
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,381 @@
+# XmlDataExtractor
+
+This gem provides a DSL for extracting formatted data from any XML structure.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'xml_data_extractor'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install xml_data_extractor
+
+## Usage
+
+The general ideia is to declare a ruby Hash that represents the fields structure, containing instructions of how every piece of data should be retrieved from the XML document.
+
+```ruby
+structure = { schemas: { character: { path: "xml/FirstName" } } }
+xml = "<xml><FirstName>Gandalf</FirstName></xml>"
+
+result = XmlDataExtractor.new(structure).parse(xml)
+
+# result -> { character: "Gandalf" }
+```
+
+For convenience, you can write the structure in yaml, which can be easily converted to a ruby hash using `YAML.load(yml).deep_symbolize_keys`.
+
+Considering the following yaml and xml:
+
+```yml
+schemas:
+  description:
+    path: xml/desc
+    modifier: downcase
+  amount:  
+    path: xml/info/price
+    modifier: to_f
+```
+```xml
+<xml>
+  <desc>HELLO WORLD</desc>
+  <info>
+    <price>123</price>
+  </info>  
+</xml>
+```
+
+The output is:
+```ruby
+{
+  description: "hello world",
+  amount: 123.0
+}
+```
+
+### Defining the structure
+
+The structure should be defined as a hash inside the `schemas` key. See the [complete example](https://github.com/monde-sistemas/xml_data_extractor/blob/master/spec/complete_example_spec.rb#L5).
+
+When defining the structure you can combine any available command in order to extract and format the data as needed.
+
+The available commands are separated in two general pusposes:
+
+- [Navigation & Extraction](#navigation--extraction)
+- [Formatting](#formatting)
+
+### Navigation & Extraction:
+
+The data extraction process is based on `Xpath` using Nokogiri.
+* [Xpath introduction](https://blog.scrapinghub.com/2016/10/27/an-introduction-to-xpath-with-examples)  
+* [Xpath cheatsheet](https://devhints.io/xpath)
+
+#### path
+
+Defines the `xpath` of the element.
+The `path` is the default command of a field definition, so this:
+```yml
+  schemas:
+    description: 
+      path: xml/desc
+```    
+Is equivalent to this:
+```yml
+  schemas:
+    description: xml/desc
+```    
+
+It can be defined as a string:
+```yml
+schemas:
+  description:
+    path: xml/some_field
+```
+```xml
+<xml>
+  <some_field>ABC</some_field>
+</xml>
+```
+```ruby
+{ description: "ABC" }
+```
+
+Or as a string array:
+```yml
+schemas:
+  address:
+    path: [street, info/city]     
+```
+```xml
+<xml>
+  <street>Diagon Alley</street>
+  <info>
+    <city>London</city>
+  </info>  
+</xml>
+```
+```ruby
+{ address: ["Diagon Alley", "London"] }
+```
+
+And even as a hash array, for complex operations:
+```yml
+schemas:
+  address:
+    path:
+      - path: street
+        modifier: downcase
+      - path: info/city
+        modifier: upcase   
+```
+```ruby
+{ address: ["diagon alley", "LONDON"] }
+```
+
+#### attr
+
+Defines a tag attribute which the value should be extracted from, instead of the tag value itself:
+```yml
+schemas:
+  description:
+    path: xml/info 
+    attr: desc
+```
+```xml
+<xml>
+  <info desc="ABC">some stuff<info>
+</xml> 
+```
+```ruby
+{ description: "ABC" }
+```
+
+Like the path, it can also be defined as a string array.
+
+#### within
+
+To define a root path for the fields:
+```yml
+schemas:  
+  movie:
+    within: info/movie_data
+    title: original_title
+    actor: main_actor
+
+```
+```xml
+<xml>
+  <info>
+    <movie_data>
+      <original_title>The Irishman</original_title>
+      <main_actor>Robert De Niro</main_actor>
+    </movie_data>
+  </info>
+</xml>
+```
+```ruby
+{ movie: { title: "The Irishman", actor: "Robert De Niro" } }
+```
+
+#### array_of
+
+Defines the path to a XML collection, which will be looped generating an array of hashes:
+```yml
+schemas:
+  people:
+    array_of: characters/character
+    name: firstname
+    age: age
+```
+```xml
+<xml>
+  <characters>
+    <character>
+      <firstname>Geralt</firstname>
+      <age>97</age>
+    </character>
+    <character>
+      <firstname>Yennefer</firstname>
+      <age>102</age>
+    </character>
+  </characters>
+</xml>
+```
+```ruby
+{
+  people: [
+    { name: "Geralt", age: "97" },
+    { name: "Yennefer", age: "102" }
+  ]
+}
+```
+
+If you need to loop trough nested collections, you can define an array of paths:
+```yml
+schemas:
+  show:    
+    within: show_data
+    title: description
+    people:
+      array_of: [characters/character, info]
+      name: name
+```
+```xml
+<xml>
+  <show_data>
+    <description>Peaky Blinders</description>
+    <characters>
+      <character>
+        <info>
+          <name>Tommy Shelby</name>          
+        </info>
+      </character>
+      <character>
+        <info>
+          <name>Arthur Shelby</name>          
+        </info>
+        <info>
+          <name>Alfie Solomons</name>
+        </info>
+      </character>
+    </characters>
+  </show_data>
+</xml>
+```
+```ruby
+{
+  show: {
+    title: "Peaky Blinders",
+    people: [
+      { name: "Tommy Shelby" },
+      { name: "Arthur Shelby" },
+      { name: "Alfie Solomons" }      
+    ]
+  }  
+}
+```
+
+### Formatting:
+
+#### fixed
+
+Defines a fixed value for the field:
+```yml
+  currency:
+    fixed: BRL
+```
+```ruby
+  { currency: "BRL" }
+```
+
+#### mapper
+
+Uses a hash of predefined values to replace the extracted value with its respective option.
+If the extracted value is not found in any of the mapper options, it will be replaced by the `default` value, but if the default value is not defined, the returned value is not replaced.
+```yml
+mappers:
+  currencies:
+    default: unknown
+    options:      
+      BRL: R$
+      USD: [US$, $]
+schemas:
+  money:    
+    array_of: curr_types/type
+    path: symbol
+    mapper: currencies
+```
+```xml
+  <xml>
+    <curr_type>
+      <type>
+        <symbol>US$</symbol>
+      </type>
+      <type>
+        <symbol>R$</symbol>
+      </type>
+      <type>
+        <symbol>RB</symbol>
+      </type>      
+      <type>
+        <symbol>$</symbol>
+      </type>      
+    </curr_type>  
+  </xml>
+```
+```ruby
+  {
+    money: ["USD", "BRL", "unknown", "USD"]
+  }
+```
+
+#### modifier
+
+Defines a method to be called on the returned value.
+```yml
+schemas:
+  name:
+    path: some_field
+    modifier: upcase
+```
+```xml
+<xml>
+  <some_field>Lewandovski</some_field>
+</xml>
+```
+```ruby
+{ name: "LEWANDOVSKI" }
+```
+
+You can also pass parameters to the method. In this case you will have to declare the modifier as an array of hashes, with the `name` and `params` keys:
+```yml
+schemas:
+  name:
+    path: [firstname, lastname]
+    modifier: 
+      - name: join
+        params: [" "]    
+      - downcase      
+```
+```xml
+<xml>
+  <firstname>Robert</firstname>
+  <lastname>Martin</lastname>
+</xml>
+```
+```ruby
+{ name: "robert martin" }
+```
+
+If you need to use custom methods, you can pass an object containing the methods in the initialization. The custom method will receive the value as parameter:
+```yml
+schemas:
+  name:
+    path: final_price
+    modifier: format_as_float      
+```
+```xml
+<xml>
+  <final_price>R$ 12.99</final_price>  
+</xml>
+```
+```ruby
+class MyMethods 
+  def format_as_float(value)
+    value.gsub(/[^\d.]/, "").to_f    
+  end
+end
+
+XmlDataExtractor.new(yml, MyMethods.new).parse(xml)
+```
+```ruby
+{ price: 12.99 }
+```

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "xml_data_extractor"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/src/extract/array_of.rb

@@ -0,0 +1,58 @@
+module Extract
+  class ArrayOf < Base
+    def initialize(node, extractor, index = 0)
+      super(node, extractor)
+      @index = index
+    end
+
+    def value
+      process_paths.flatten.compact
+    end
+
+    private
+
+    attr_reader :index
+
+    def array_items
+      arr_path, link_path, uniq_by = node.array_of_paths
+
+      paths = extractor.paths_of(node.path, arr_path, link_path)
+      paths = uniq_paths(paths, uniq_by) if uniq_by
+
+      paths.each_with_index.map do |path, idx|
+        HashBuilder.new(Node.new(node.props, path), extractor).value(index + idx)
+      end.compact
+    end
+
+    def process_paths
+      paths = paths_from_props
+
+      if paths.size > 1
+        process_path(paths.shift, paths)
+      else
+        node.props[:array_of] = paths.first
+        array_items
+      end
+    end
+
+    def process_path(path, inner_paths)
+      path = build_path(path) if path.is_a?(Hash)
+
+      extractor.paths_of(node.path, path).each_with_index.map do |some, idx|
+        ArrayOf.new(Node.new(node.props.merge(array_of: inner_paths), some), extractor, index + idx).value
+      end
+    end
+
+    def uniq_paths(paths, uniq_by)
+      extractor.uniq_paths(paths, uniq_by)
+    end
+
+    def build_path(hash)
+      extractor.replace_link(hash[:path], [node.path, hash[:link]].join("/"))
+    end
+
+    def paths_from_props
+      [node.props[:array_of]].flatten
+    end
+  end
+end

data/lib/src/extract/array_value.rb

@@ -0,0 +1,10 @@
+module Extract
+  class ArrayValue < Base
+    def value
+      props, path = node.to_h.values_at(:props, :path)
+      props.map do |prop|
+        ValueBuilder.new(Node.new(prop, path), extractor).value
+      end.flatten
+    end
+  end
+end

data/lib/src/extract/base.rb

@@ -0,0 +1,12 @@
+module Extract
+  class Base
+    def initialize(node, extractor)
+      @node = node
+      @extractor = extractor
+    end
+
+    private
+
+    attr_reader :node, :extractor
+  end
+end

data/lib/src/extract/expression.rb

@@ -0,0 +1,20 @@
+module Extract
+  class Expression
+    def initialize(expression, hash)
+      @expression = expression
+      @hash = hash
+    end
+
+    def evaluate
+      field_name = expression.split.first.parameterize
+      field_value = hash[field_name.to_sym]
+      condition = expression.gsub(field_name, field_value.to_s)
+
+      eval(condition)
+    end
+
+    private
+
+    attr_reader :expression, :hash
+  end
+end

data/lib/src/extract/hash_builder.rb

@@ -0,0 +1,33 @@
+module Extract
+  class HashBuilder < Base
+    INTERNAL_FIELDS = %i[array_of keep_if within].freeze
+
+    def value(index = 0)
+      path, props = node.to_h.values_at(:path, :props)
+
+      hash = {}
+      props.each do |field_name, nested_props|
+        next unless valuable_field? field_name, nested_props, index
+
+        value = ValueBuilder.new(Node.new(nested_props, path), extractor).value
+        hash[field_name.to_sym] = value if value.present?
+      end
+
+      keep_hash?(hash, props) ? hash : nil
+    end
+
+    private
+
+    def keep_hash?(hash, props)
+      expression = props[:keep_if]
+      expression.present? ? Expression.new(expression, hash).evaluate : true
+    end
+
+    def valuable_field?(field_name, props, index)
+      return false if INTERNAL_FIELDS.include? field_name
+      return false if index.positive? && Node.new(props, "").first_only?
+
+      true
+    end
+  end
+end

data/lib/src/extract/string_value.rb

@@ -0,0 +1,32 @@
+module Extract
+  class StringValue < Base
+    def value
+      path = node[:props][:path]
+      return formatted_array_values(path) if path.is_a?(Array)
+
+      extract_value(node)
+    end
+
+    private
+
+    def extract_value(node_to_extract)
+      extractor.extract(node_to_extract)
+    end
+
+    def formatted_array_values(paths)
+      extractor.format_value(values_from_array(paths), node[:props])
+    end
+
+    def values_from_array(paths)
+      node_path = node.path
+
+      paths.map do |inner|
+        if inner.is_a?(String)
+          extract_value(Node.new({ path: inner }, node_path))
+        else
+          StringValue.new(Node.new(inner, node_path), extractor).value
+        end
+      end
+    end
+  end
+end

data/lib/src/extract/value_builder.rb

@@ -0,0 +1,44 @@
+require_relative "base"
+require_relative "array_value"
+require_relative "array_of"
+require_relative "hash_builder"
+require_relative "string_value"
+require_relative "value_builder"
+require_relative "within"
+require_relative "expression"
+
+module Extract
+  class ValueBuilder < Base
+    def value
+      props = node.props
+      case props
+      when String then value_for_string
+      when Array then value_for_array
+      when Hash then value_for_hash
+      else
+        raise "Invalid kind #{props.class} (#{props})"
+      end
+    end
+
+    private
+
+    def value_for_hash
+      props = node.props
+      fixed_value = props[:fixed]
+      return fixed_value if fixed_value
+      return ArrayOf.new(node, extractor).value if props[:array_of]
+      return Within.new(node, extractor).value if props[:within]
+      return StringValue.new(node, extractor).value if (props.keys & %i[path attr]).any?
+
+      HashBuilder.new(node, extractor).value
+    end
+
+    def value_for_string
+      StringValue.new(Node.new({ path: node.props }, node.path), extractor).value
+    end
+
+    def value_for_array
+      ArrayValue.new(node, extractor).value
+    end
+  end
+end

data/lib/src/extract/within.rb

@@ -0,0 +1,11 @@
+module Extract
+  class Within < Base
+    def value
+      props = node.props
+      paths = extractor.paths_of(node.path, props[:within])
+      return "" if paths.empty?
+
+      HashBuilder.new(Node.new(props, paths.first), extractor).value
+    end
+  end
+end

data/lib/src/extractor.rb

@@ -0,0 +1,236 @@
+require "cgi"
+require "active_support/core_ext/string"
+require_relative "format/formatter"
+
+class PathBuilder < Struct.new(:base, :parent, :tag, keyword_init: true)
+  def build
+    paths = relative_path.split("/").then do |paths|
+      if parent.present?
+        navigate_to_parent(parent, paths)
+      else
+        paths
+      end
+    end
+
+    paths << tag unless tag.is_a? Array
+    full_path = paths.flatten.compact.join("/")
+    "//#{full_path}"
+  end
+
+  private
+
+  def relative_path
+    base.start_with?("//") ? base[2..-1] : base
+  end
+
+  def navigate_to_parent(parent_tag, paths)
+    index = path_index(parent_tag, paths)
+
+    paths[0, index + 1]
+  end
+
+  def path_index(tag, paths)
+    paths.each_with_index do |path, index|
+      return index if matching_tags?(path, tag)
+    end
+    0
+  end
+
+  def matching_tags?(item, tag)
+    item.gsub(/\[\d\]/, "") == tag
+  end
+end
+
+class NodeParamsExtractor < Struct.new(:node)    
+  def extract
+    [node.path, *node.props.values_at(:in_parent, :path, :link, :attr)]
+  end
+end
+
+class NodeExtractor
+  def initialize(xml)
+    @xml = Nokogiri::XML(remove_special_elements(xml), nil, Encoding::UTF_8.to_s)
+    @xml.remove_namespaces!
+  end
+
+  def extract(path)
+    xml.xpath(path)
+  rescue StandardError
+    nil
+  end
+
+  private
+
+  def remove_special_elements(xml)
+    CGI.unescapeHTML(xml).gsub(/<br>|&nbsp;/, { "<br>" => "", "&nbsp;" => " " })
+  end
+
+  attr_reader :xml
+end
+
+class NodeValueExtractor
+  def initialize(node_extractor)
+    @node_extractor = node_extractor
+  end
+
+  def attr_values(path, attributes)
+    return attributes.map { |atr| attr_value(path, atr) } if attributes.is_a? Array
+    return tag_count(path) if attributes == :tag_count
+
+    attr_value(path, attributes)
+  end
+
+  def tag_count(path)
+    node_extractor.extract(path).size
+  end
+
+  def tag_values(base_path, paths)
+    return tag_value(base_path) unless paths.is_a? Array
+
+    paths.map { |path| tag_value([base_path, path].flatten.compact.join("/")) }
+  end
+
+  private
+
+  attr_reader :node_extractor
+
+  def tag_value(path)
+    node_raw_value node_extractor.extract(path)
+  end
+
+  def attr_value(path, att)
+    node_raw_value node_extractor.extract(path).attribute(att)
+  end
+
+  def node_raw_value(node)
+    NodeValue.new(node).raw_value
+  end
+end
+
+class NodeValue
+  def initialize(node)
+    @node = node
+  end
+
+  def raw_value
+    return "" unless node
+
+    node_size = node.try(:size).to_i
+    return node.map(&:text) if node_size > 1
+    return node.first if node_size == 1 && contains_children?
+
+    node.text
+  end
+
+  private
+
+  attr_reader :node
+
+  def contains_children?
+    node.first.try(:children).any? { |child| child.is_a? Nokogiri::XML::Element }
+  end
+end
+
+class PathManipulator
+  def initialize(node_value_extractor)
+    @node_value_extractor = node_value_extractor
+  end
+
+  def replace_link(original_path, link_path)
+    return original_path if link_path.blank?
+
+    link_value = node_value_extractor.tag_values(link_path, nil)
+
+    original_path.gsub "<link>", link_value
+  end
+
+  def uniq_paths(paths, uniq_by_path)
+    paths
+      .map { |path| { path: path, value: tag_value(path, uniq_by_path) } }
+      .then { |paths_values| remove_duplicated_paths(paths_values) }
+      .map { |path_value| path_value[:path] }
+  end
+
+  private
+
+  attr_reader :node_value_extractor
+
+  def tag_value(path, uniq_by_path)
+    node_value_extractor.tag_values([path, uniq_by_path].join("/"), "")
+  end
+
+  def remove_duplicated_paths(paths_values)
+    paths_values.delete_if.with_index do |path_value, index|
+      index != first_path_value_index(paths_values, path_value)
+    end
+  end
+
+  def first_path_value_index(paths_values, current_path)
+    paths_values.find_index { |path_value| path_value[:value] == current_path[:value] }
+  end
+end
+
+class Extractor
+  def initialize(xml, yml, modifiers)
+    @node_extractor = NodeExtractor.new(xml)
+    @node_value_extractor = NodeValueExtractor.new(node_extractor)
+    @path_manipulator = PathManipulator.new(node_value_extractor)
+    @formatter = Format::Formatter.new(yml, modifiers)
+  end
+
+  def extract(node)
+    base, parent, tag, link, attribute = NodeParamsExtractor.new(node).extract
+    path = PathBuilder.new(base: base, parent: parent, tag: tag).build
+
+    if link.present?
+      link_path = PathBuilder.new(base: base, parent: parent, tag: link).build
+
+      if tag.is_a? Array
+        tag = tag.map { |tag_path| replace_link(tag_path, link_path) }
+      else
+        path = replace_link(path, link_path)
+      end
+    end
+
+    value = path_value(path, tag, attribute)
+    format_value(value, node.props) 
+  end
+
+  def format_value(value, props)
+    formatter.format_value(value, props)
+  end
+
+  def replace_link(original_path, link_path)
+    path_manipulator.replace_link(original_path, link_path)
+  end
+
+  def paths_of(base_path, tag_path, link_path = nil)
+    path = PathBuilder.new(base: base_path, tag: tag_path).build
+
+    if link_path.present?
+      link_path = PathBuilder.new(base: base_path, tag: link_path).build
+      path = replace_link(path, link_path)
+    end
+
+    node = node_extractor.extract(path)
+    (node || []).size.times.map do |index|
+      "#{path}[#{index + 1}]"
+    end
+  end
+
+  def uniq_paths(paths, uniq_by_path)
+    return paths if uniq_by_path.blank?
+
+    path_manipulator.uniq_paths(paths, uniq_by_path)
+  end
+
+  private
+
+  attr_reader :node_extractor, :node_value_extractor, :path_manipulator, :formatter
+
+  def path_value(path, tag, attribute)
+    return node_value_extractor.attr_values(path, attribute) if attribute.present?
+
+    node_value_extractor.tag_values(path, tag)
+  end
+end

data/lib/src/format/formatter.rb

@@ -0,0 +1,28 @@
+require_relative "mapper"
+require_relative "modifier"
+
+module Format
+  class Formatter
+    def initialize(yml, modifiers)
+      @mapper = Format::Mapper.new(yml)
+      @modifier = Format::Modifier.new(yml, modifiers)
+    end
+
+    def format_value(value, props)
+      modifier_prop, mapper_prop = props.values_at(:modifier, :mapper)
+
+      value
+        .then { |it| modifier.apply(it, modifier_prop) }
+        .then { |it| nullify_empty_value(it) }
+        .then { |it| mapper.apply(it, mapper_prop) }
+    end
+
+    private
+
+    attr_reader :modifier, :mapper
+
+    def nullify_empty_value(value)
+      value.blank? || value.try(:zero?) ? nil : value
+    end
+  end
+end

data/lib/src/format/mapper.rb

@@ -0,0 +1,28 @@
+module Format
+  class Mapper
+    def initialize(yml)
+      @mappers = yml.fetch(:mappers, {})
+    end
+
+    def apply(raw_value, mapper_name)
+      return raw_value unless mapper_name
+
+      mappers.each do |name, fields|
+        return mapper_value(fields, raw_value) if mapper_name.to_sym == name
+      end
+
+      raise "Mapper not found #{mapper_name}"
+    end
+
+    private
+
+    attr_reader :mappers
+
+    def mapper_value(fields, value)
+      (fields[:options] || []).each do |option, values|
+        return option.to_s if [values].flatten.include?(value.to_s)
+      end
+      fields[:default] || value
+    end
+  end
+end

data/lib/src/format/modifier.rb

@@ -0,0 +1,37 @@
+module Format
+  class Modifier
+    def initialize(yml, helper)
+      @debug = yml.fetch(:debug, false)
+      @helper = helper
+    end
+
+    def apply(raw_value, modifiers)
+      [modifiers].flatten.compact.reduce(raw_value) do |value, modifier|
+        method_name, params = modifier_props(modifier).values_at(:name, :params)
+
+        modify_value(value, method_name, params)
+      end
+    end
+
+    private
+
+    attr_reader :helper, :debug
+
+    def modifier_props(modifier)
+      modifier.is_a?(String) ? { name: modifier } : modifier
+    end
+
+    def modify_value(value, method, params)
+      args = [value]
+      if params.present?
+        args = params.is_a?(Array) ? [value, *params] : [value, **params]
+      end
+
+      value.try(method, *params) || helper.send(method, *args)
+    rescue StandardError => error
+      raise error unless debug
+
+      "Error invoking '#{method}' with (#{args.join(',')}): #{error}"
+    end
+  end
+end

data/lib/src/node.rb

@@ -0,0 +1,26 @@
+class Node < Struct.new(:props, :path)
+  def initialize(*)
+    super
+    self.path ||= ""
+  end
+
+  def first_only?
+    return unless props.is_a? Hash
+
+    props[:array_presence] == "first_only"
+  end
+
+  def array_of_paths
+    array_paths(props[:array_of])
+  end
+
+  private
+
+  def array_paths(array_props)
+    if array_props.is_a?(Hash)
+      array_props.values_at(:path, :link, :uniq_by)
+    else
+      [array_props].flatten
+    end
+  end
+end

data/lib/xml_data_extractor.rb

@@ -0,0 +1,27 @@
+require "nokogiri"
+require_relative "src/extractor"
+require_relative "src/node"
+require_relative "src/extract/value_builder"
+
+class XmlDataExtractor   
+  def initialize(config, modifiers = nil)
+    @config = config      
+    @modifiers = modifiers
+  end
+
+  def parse(xml)
+    extractor = Extractor.new(xml, config, modifiers)
+    schemas = config.fetch(:schemas, {})
+
+    {}.tap do |hash|
+      schemas.map do |key, val|
+        value = Extract::ValueBuilder.new(Node.new(val), extractor).value
+        hash[key] = value if value.present?
+      end
+    end      
+  end
+
+  private
+
+  attr_reader :config, :modifiers
+end

data/xml_data_extractor.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |spec|
+  spec.name          = "xml_data_extractor"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Fernando Almeida"]
+  spec.email         = ["fernandoprsbr@gmail.com"]
+
+  spec.summary       = "Provides a simples DSL for extracting data from XML documents"
+  spec.homepage      = "https://github.com/monde-sistemas/xml_data_extractor"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "nokogiri"
+  spec.add_dependency "activesupport"
+  spec.add_development_dependency "rspec"
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: xml_data_extractor
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Fernando Almeida
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-08-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- fernandoprsbr@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/src/extract/array_of.rb
+- lib/src/extract/array_value.rb
+- lib/src/extract/base.rb
+- lib/src/extract/expression.rb
+- lib/src/extract/hash_builder.rb
+- lib/src/extract/string_value.rb
+- lib/src/extract/value_builder.rb
+- lib/src/extract/within.rb
+- lib/src/extractor.rb
+- lib/src/format/formatter.rb
+- lib/src/format/mapper.rb
+- lib/src/format/modifier.rb
+- lib/src/node.rb
+- lib/xml_data_extractor.rb
+- xml_data_extractor.gemspec
+homepage: https://github.com/monde-sistemas/xml_data_extractor
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/monde-sistemas/xml_data_extractor
+  source_code_uri: https://github.com/monde-sistemas/xml_data_extractor
+  changelog_uri: https://github.com/monde-sistemas/xml_data_extractor
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Provides a simples DSL for extracting data from XML documents
+test_files: []