eatr 0.1.1

data/lib/eatr/version.rb

@@ -0,0 +1,3 @@
+module Eatr
+  VERSION = "0.1.1"
+end

data/lib/eatr/xml/document.rb

@@ -0,0 +1,87 @@
+require "yaml"
+require "nokogiri"
+
+module Eatr
+  module Xml
+    NodeNotFound = Class.new(StandardError)
+
+    class Document
+      include ParseValue
+      extend Forwardable
+
+      attr_reader :schema
+
+      def_delegator :schema,
+        :transformation_pipeline
+
+      def initialize(schema_path)
+        @schema = Schema.new(YAML.load(File.read(schema_path)))
+      end
+
+      def parse(xml_document_path)
+        @namespaces = {}
+
+        doc = Nokogiri::XML(File.open(xml_document_path)) do |config|
+          config.strict.nonet
+        end
+
+        if @schema.remove_namespaces?
+          doc.remove_namespaces!
+          @namespaces = {}
+        else
+          @namespaces = doc.collect_namespaces
+        end
+
+        cardinality = @schema.fields.inject(1) do |memo, field|
+          if field.node?
+            memo * [doc.xpath(field.xpath, @namespaces).count, 1].max
+          else
+            memo
+          end
+        end
+
+        objects = []
+
+        cardinality.times do |n|
+          objects << @schema.to_struct.new
+        end
+
+        @schema.fields.each do |field|
+          objects = set_field(objects, doc, field)
+        end
+
+        objects
+      end
+
+      private
+
+      def set_field(objects, doc, field)
+        if field.node?
+          doc.xpath(field.xpath, @namespaces).each_with_index do |child_xml, idx|
+            field.children.flat_map do |child|
+              set_field([objects[idx]], child_xml, child)
+            end
+          end
+        elsif field.name
+          objects.each do |o|
+            o.public_send("#{field.name}=", value_at(doc, field))
+          end
+        end
+
+        objects
+      end
+
+      def value_at(doc, field)
+        if field.value
+          field.value
+        elsif field.xpath
+          if node = doc.at_xpath(field.xpath, @namespaces)
+            parse_value(field, node.content)
+          elsif field.required?
+            raise NodeNotFound, "Unable to find '#{field.name}' using xpath '#{field.xpath}'"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eatr/xml/schema_generator.rb

@@ -0,0 +1,69 @@
+module Eatr
+  module Xml
+    class SchemaGenerator
+      def initialize(xml_path)
+        @xml_path = xml_path
+      end
+
+      def schema(starting_point)
+        doc = Nokogiri::XML(File.open(@xml_path)) do |config|
+          config.strict.nonet
+        end
+
+        doc.remove_namespaces!
+
+        fields = doc.at_xpath(starting_point).element_children.flat_map do |child|
+          field_def(child)
+        end
+
+        schema = {
+          'name' => '',
+          'remove_namespaces' => true,
+          'fields' => fields
+        }
+
+        YAML.dump(schema)
+      end
+
+      private
+
+      def field_def(child, name_prefix: '', xpath_relative_to: nil)
+        if unique_children_count(child) == 1 && child.element_children.map(&:name).count > 1
+          relative_path = Regexp.new(child.element_children.first.path.gsub(/\[\d+\]/, "\\[\\d+\\]"))
+          node_path = child.element_children.first.path.gsub(/\[\d+\]/, "")
+
+          {
+            'node' => name_prefix + underscore(child.name),
+            'xpath' => xpath_relative_to ? child.path.gsub(xpath_relative_to, ".") : node_path,
+            'children' => child.element_children.first.element_children.flat_map do |c|
+              field_def(c, name_prefix: "#{underscore(child.name)}_", xpath_relative_to: relative_path)
+            end
+          }
+        elsif unique_children_count(child) >= 1
+          child.element_children.flat_map do |c|
+            field_def(c, name_prefix: "#{underscore(child.name)}_", xpath_relative_to: xpath_relative_to)
+          end
+        else
+          {
+            'name' => name_prefix + underscore(child.name),
+            'xpath' => xpath_relative_to ? child.path.gsub(xpath_relative_to, ".") : child.path,
+            'type' => 'string',
+            'required' => false
+          }
+        end
+      end
+
+      def unique_children_count(element)
+        element.element_children.map(&:name).uniq.count
+      end
+
+      def underscore(str)
+        str.gsub(/::/, '/').
+          gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+          gsub(/([a-z\d])([A-Z])/,'\1_\2').
+          tr("-", "_").
+          downcase
+      end
+    end
+  end
+end

data/sample.dot

@@ -0,0 +1,42 @@
+strict digraph g {
+  ranksep="1.6"
+  graph [
+  rankdir = "LR"
+  ];
+  node [
+  fontsize = "16"
+  ];
+  edge [
+  arrowhead = "none"
+  ];
+"books" [shape=none, margin=0, label=<
+  <table border="0" cellborder="1" cellspacing="0" cellpadding="4">
+    <tr><td bgcolor="lightblue">books</td></tr>
+    <tr><td port="id" align="left">id</td></tr>
+    <tr><td port="author" align="left">author</td></tr>
+    <tr><td port="library_id" align="left">library_id</td></tr>
+    <tr><td port="pages" align="left">pages</td></tr>
+    <tr><td port="for_sale" align="left">for_sale</td></tr>
+    <tr><td port="published_at" align="left">published_at</td></tr>
+    <tr><td port="rating" align="left">rating</td></tr>
+    <tr><td port="icbn" align="left">icbn</td></tr>
+    <tr><td port="summary" align="left">summary</td></tr>
+    <tr><td port="age" align="left">age</td></tr>
+  </table>>];
+"chapters" [shape=none, margin=0, label=<
+  <table border="0" cellborder="1" cellspacing="0" cellpadding="4">
+    <tr><td bgcolor="lightblue">chapters</td></tr>
+    <tr><td port="book_id" align="left">book_id</td></tr>
+    <tr><td port="title" align="left">title</td></tr>
+  </table>>];
+"libraries" [shape=none, margin=0, label=<
+  <table border="0" cellborder="1" cellspacing="0" cellpadding="4">
+    <tr><td bgcolor="lightblue">libraries</td></tr>
+    <tr><td port="id" align="left">id</td></tr>
+    <tr><td port="book_title" align="left">book_title</td></tr>
+    <tr><td port="desk_number" align="left">desk_number</td></tr>
+  </table>>];
+"books":"id" -> "chapters":"book_id" [arrowhead="crow"];
+"books":"library_id" -> "libraries":"id" [arrowhead="teeodot"];
+"chapters":"book_id" -> "books":"id" [arrowhead="tee"];
+}

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: eatr
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Greggory Rothmeier
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-01-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+description: Configuration-based document parsing and transformation framework.
+email:
+- greggroth@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/rspec
+- bin/setup
+- eatr.gemspec
+- lib/eatr.rb
+- lib/eatr/csv/document.rb
+- lib/eatr/dot_generator.rb
+- lib/eatr/dot_template.dot
+- lib/eatr/parse_value.rb
+- lib/eatr/pipeline.rb
+- lib/eatr/pipeline_spec.rb
+- lib/eatr/schema.rb
+- lib/eatr/sql/table_generator.rb
+- lib/eatr/transformation/add_date_id.rb
+- lib/eatr/transformation_set.rb
+- lib/eatr/version.rb
+- lib/eatr/xml/document.rb
+- lib/eatr/xml/schema_generator.rb
+- sample.dot
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Configuration-based document parsing and transformation framework.
+test_files: []
+has_rdoc: