eatr 0.1.1

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "eatl"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/rspec

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/eatr.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'eatr/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "eatr"
+  spec.version       = Eatr::VERSION
+  spec.authors       = ["Greggory Rothmeier"]
+  spec.email         = ["greggroth@gmail.com"]
+
+  spec.summary       = %q{Configuration-based document parsing and transformation framework.}
+  spec.description   = %q{Configuration-based document parsing and transformation framework.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "nokogiri", "~> 1.6"
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "pry", "~>0.10"
+end

data/lib/eatr.rb

@@ -0,0 +1,11 @@
+require "eatr/version"
+require "eatr/parse_value"
+require "eatr/schema"
+require "eatr/xml/document"
+require "eatr/csv/document"
+require "eatr/pipeline"
+require "eatr/transformation_set"
+require "eatr/transformation/add_date_id"
+
+module Eatr
+end

data/lib/eatr/csv/document.rb

@@ -0,0 +1,49 @@
+require 'csv'
+
+module Eatr
+  module Csv
+    ValueNotFound = Class.new(StandardError)
+
+    class Document
+      include ParseValue
+      extend Forwardable
+
+      attr_reader :schema
+
+      def_delegator :schema,
+        :transformation_pipeline
+
+      def initialize(schema_path)
+        @schema = Schema.new(YAML.load(File.read(schema_path)))
+      end
+
+      def parse(csv_document_path)
+        objects = []
+
+        CSV.foreach(csv_document_path, headers: true) do |row|
+          obj = @schema.to_struct.new
+
+          @schema.fields.each do |field|
+            obj.public_send("#{field.name}=", value_at(row, field))
+          end
+
+          objects << obj
+        end
+
+        objects
+      end
+
+      private
+
+      def value_at(row, field)
+        if field.value
+          field.value
+        elsif text = row[field.csv_header]
+          parse_value(field, text)
+        elsif field.required?
+          raise ValueNotFound, "Unable to find '#{field.name}' with header '#{field.csv_header}'"
+        end
+      end
+    end
+  end
+end

data/lib/eatr/dot_generator.rb

@@ -0,0 +1,28 @@
+module Eatr
+  class DotGenerator
+    attr_reader :tables
+
+    DEFAULT_TEMPLATE_PATH = "#{File.dirname(__FILE__)}/dot_template.dot"
+
+    def initialize(schema_paths, template_path: DEFAULT_TEMPLATE_PATH)
+      @tables = Array[schema_paths].flatten.map { |s| Schema.new(YAML.load(File.read(s))) }
+      @template_path = template_path
+    end
+
+    def to_dot
+      ERB.new(File.read(@template_path), nil, '-').result(binding)
+    end
+
+    private
+
+    def table_included?(belongs_to_str)
+      table_name, _ = belongs_to_str.split('.')
+      @tables.any? { |t| t.table_name == table_name }
+    end
+
+    def arrow_target(belongs_to_str)
+      table_name, column = belongs_to_str.split('.')
+      "\"#{table_name}\":\"#{column}\""
+    end
+  end
+end

data/lib/eatr/dot_template.dot

@@ -0,0 +1,35 @@
+strict digraph g {
+  ranksep="1.6"
+  graph [
+  rankdir = "LR"
+  ];
+  node [
+  fontsize = "16"
+  ];
+  edge [
+  arrowhead = "none"
+  ];
+<%- tables.each do |table| -%>
+"<%= table.table_name %>" [shape=none, margin=0, label=<
+  <table border="0" cellborder="1" cellspacing="0" cellpadding="4">
+    <tr><td bgcolor="lightblue"><%= table.table_name %></td></tr>
+  <%- table.flat_fields.each do |field| -%>
+    <tr><td port="<%= field.name %>" align="left"><%= field.name %></td></tr>
+  <%- end -%>
+  </table>>];
+<%- end -%>
+<%- tables.each do |table| -%>
+  <%- table.flat_fields.each do |field| -%>
+    <%- if field.belongs_to_one && table_included?(field.belongs_to_one)-%>
+"<%= table.table_name %>":"<%= field.name %>" -> <%= arrow_target(field.belongs_to_one) %> [arrowhead="tee<%= 'odot' if !field.required? %>"];
+    <%- end -%>
+    <%- if field.has_many -%>
+      <%- field.has_many.each do |r| -%>
+        <%- if table_included?(r)-%>
+"<%= table.table_name %>":"<%= field.name %>" -> <%= arrow_target(r) %> [arrowhead="crow<%= 'odot' if !field.required? %>"];
+        <%- end -%>
+      <%- end -%>
+    <%- end -%>
+  <%- end -%>
+<%- end -%>
+}

data/lib/eatr/parse_value.rb

@@ -0,0 +1,25 @@
+module Eatr
+  module ParseValue
+    def parse_value(field, text)
+      case field.type
+      when 'integer' then text.to_i
+      when 'float' then text.to_f
+      when 'timestamp'
+        if field.strptime
+          DateTime.strptime(text, field.strptime)
+        else
+          DateTime.parse(text)
+        end
+      when 'boolean' then YAML.load(text)
+      else
+        if field.max_length
+          text[0...field.max_length]
+        elsif field.length
+          text[0...field.length]
+        else
+          text
+        end
+      end
+    end
+  end
+end

data/lib/eatr/pipeline.rb

@@ -0,0 +1,11 @@
+module Eatr
+  class Pipeline
+    def initialize(steps)
+      @steps = steps
+    end
+
+    def call(row)
+      @steps.reduce(row) { |memo, step| step.call(memo) }
+    end
+  end
+end

data/lib/eatr/pipeline_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+require './lib/eatr/pipeline'
+
+describe Eatr::Pipeline do
+  it 'performs each action on an object' do
+    pipeline = described_class.new([
+      ->(str) { str.upcase },
+      ->(str) { str[1..3] },
+    ])
+
+    expect(pipeline.call("hello")).to eq("ELL")
+  end
+end

data/lib/eatr/schema.rb

@@ -0,0 +1,94 @@
+module Eatr
+  class Schema
+    class Field
+      def initialize(field_attributes)
+        @field_attributes = field_attributes
+      end
+
+      %w[
+        name
+        type
+        xpath
+        csv_header
+        strptime
+        value
+        max_length
+        length
+        belongs_to_one
+        has_many
+      ].each do |f|
+        define_method(f) do
+          @field_attributes[f]
+        end
+      end
+
+      def type
+        @field_attributes['type'].to_s.downcase
+      end
+
+      def node?
+        @field_attributes.has_key?('node')
+      end
+
+      def required?
+        @field_attributes.fetch('required', true)
+      end
+
+      def children
+        Array[*@field_attributes['children']].map { |f| Field.new(f) }
+      end
+    end
+
+    def initialize(schema_hash)
+      @schema = schema_hash
+    end
+
+    def fields
+      @fields ||= @schema.fetch('fields').map { |f| Field.new(f) }
+    end
+
+    def flat_fields
+      @flat_fields ||= fields.select(&:name).
+        concat(fields.flat_map(&:children))
+    end
+
+    def name
+      @schema.fetch('name', 'schema')
+    end
+
+    def table_name
+      @schema.fetch('table_name', name)
+    end
+
+    def remove_namespaces?
+      @schema.fetch('remove_namespaces', false)
+    end
+
+    def to_struct
+      @struct_klass ||= begin
+                          Object.const_get("Struct::#{constant_name}")
+                        rescue NameError
+                          Struct.new(constant_name, *field_names)
+                        end
+    end
+
+    def transformation_pipeline
+      Pipeline.new(TransformationSet.new(@schema.fetch('transformations', [])))
+    end
+
+    private
+
+    def constant_name
+      constantize(name)
+    end
+
+    def field_names
+      flat_fields.map { |f| f.name.to_sym }
+    end
+
+    def constantize(underscore_name)
+      underscore_name.split('_').map(&:capitalize).join
+    end
+
+  end
+end

data/lib/eatr/sql/table_generator.rb

@@ -0,0 +1,52 @@
+module Eatr
+  module Sql
+    class TableGenerator
+      def initialize(schema_path)
+        @schema = Schema.new(YAML.load(File.read(schema_path)))
+      end
+
+      def statement
+        <<-STATEMENT
+CREATE TABLE #{@schema.table_name} (
+  #{column_defs.join(",\n  ")}
+);
+        STATEMENT
+      end
+
+      private
+
+      def column_defs
+        @schema.flat_fields.map do |f|
+          "#{f.name} #{type(f)}#{nullness(f)}"
+        end
+      end
+
+      def type(f)
+        case f.type
+        when nil,'string',''
+          if f.length
+            "CHAR(#{f.length})"
+          elsif f.max_length
+            "VARCHAR(#{f.max_length})"
+          else
+            'TEXT'
+          end
+        when 'integer'
+          'INT'
+        when 'float'
+          'REAL'
+        when 'timestamp'
+          'TIMESTAMP'
+        when 'boolean'
+          'BOOLEAN'
+        end
+      end
+
+      def nullness(f)
+        if f.required?
+          " NOT NULL"
+        end
+      end
+    end
+  end
+end

data/lib/eatr/transformation/add_date_id.rb

@@ -0,0 +1,20 @@
+module Eatr
+  module Transformation
+    class AddDateId
+      def initialize(args)
+        @source = args.fetch('source')
+        @destination = args.fetch('destination')
+      end
+
+      def call(*objs)
+        Array(objs.flatten).map do |o|
+          if !o[@source].nil?
+            o[@destination] = o[@source].strftime('%Y%m%d').to_i
+          end
+
+          o
+        end
+      end
+    end
+  end
+end

data/lib/eatr/transformation_set.rb

@@ -0,0 +1,27 @@
+module Eatr
+  class TransformationSet
+    include Enumerable
+
+    def initialize(transformations)
+      @transformations = transformations
+    end
+
+    def each
+      to_a.each do |t|
+        yield t
+      end
+    end
+
+    def to_a
+      @transformations.map do |t|
+        const = Object.const_get(t.fetch('class'))
+
+        if t['args']
+          const.new(t['args'])
+        else
+          const.new
+        end
+      end
+    end
+  end
+end