davidrichards-etl 0.0.4

data/lib/etl.rb

@@ -0,0 +1,36 @@
+# TODO:
+# Test this (a bit of a bugger, because I need to fail at every callback and make sure that I can recover.)
+# Get the logging done and demonstrated, because an ETL process without good logging really is useless.
+# Include a logging example for syslog-ng and syslog
+# Work through some bucket thoughts that I was having this morning: how to take random percepts and create consolidated snapshots of an environment at a point in time.  This is driven from the belief maintenance systems, but certainly needs to be worked out.
+# Figure out if TeguGears really should be doing this.  Come back to how I'll parallelize this process.  Demonstrate running this in parallel.
+
+require 'rubygems'
+require 'activesupport'
+require 'ostruct'
+require 'log4r'
+require 'fileutils'
+
+def load_gem_casually(name)
+  begin
+    gem name
+    require name
+  rescue Gem::LoadError
+    # Do nothing if this is not available.  It's a convenience, not a requirement.
+  end
+end
+
+load_gem_casually('tegu_gears')
+load_gem_casually('data_frame')
+load_gem_casually('babel_icious')
+
+Dir.glob("#{File.dirname(__FILE__)}/helpers/*.rb").each { |file| require file }
+
+$:.unshift(File.dirname(__FILE__))
+
+class ExtractError < StandardError; end
+class TransformError < StandardError; end
+# Note, LoadError is already used.
+class LoadingError < StandardError; end
+
+require 'etl/etl'

data/lib/helpers/array.rb

@@ -0,0 +1,11 @@
+class Array # :nodoc:
+  # From Carl Youngblood's excellent SBN gem: http://sbn.rubyforge.org/
+  def symbolize_values
+    self.map {|e| e.to_underscore_sym }
+  end
+  
+  # From Carl Youngblood's excellent SBN gem: http://sbn.rubyforge.org/
+  def symbolize_values!
+    self.map! {|e| e.to_underscore_sym }
+  end
+end

data/lib/helpers/observation.rb

@@ -0,0 +1,10 @@
+# Generic OpenStruct, with occured_at automatically set to the initialization time.
+class Observation < OpenStruct
+  def initialize(*args)
+    @occured_at = Time.now
+    super
+  end
+  
+  # Need to know when the observation was recorded to batch observations
+  attr_accessor :occured_at
+end

data/lib/helpers/open_struct.rb

@@ -0,0 +1,18 @@
+class OpenStruct
+    
+  def table
+    @table
+  end
+  
+  def keys
+    self.table.keys
+  end
+  
+  def values
+    self.table.values
+  end
+  
+  def include?(key)
+    self.keys.include?(key)
+  end
+end

data/lib/helpers/string.rb

@@ -0,0 +1,6 @@
+class String # :nodoc:
+  # From Carl Youngblood's excellent SBN gem: http://sbn.rubyforge.org/
+  def to_underscore_sym
+    self.titleize.gsub(/\s+/, '').underscore.to_sym
+  end
+end

data/lib/helpers/symbol.rb

@@ -0,0 +1,6 @@
+class Symbol # :nodoc:
+  # From Carl Youngblood's excellent SBN gem: http://sbn.rubyforge.org/
+  def to_underscore_sym
+    self.to_s.titleize.gsub(/\s+/, '').underscore.to_sym
+  end
+end

data/spec/etl/bucket_spec.rb

@@ -0,0 +1,112 @@
+require File.join(File.dirname(__FILE__), "/../spec_helper")
+require 'etl/bucket'
+
+describe Bucket do
+  
+  before(:all) do
+    class A
+      def initialize(*args)
+        @value = args
+      end
+      attr_reader :value
+    end
+
+    S = Struct.new(:this)
+  end
+  
+  before do
+    @b = Bucket.new
+    @h = {:this => 1}
+    @o = OpenStruct.new(:this => 1)
+    @s = S.new(1)
+    @b1 = Bucket.new(@h)
+  end
+  
+  it "should create a hash for storing raw, unordered data" do
+    @b.raw_data.should be_is_a(Hash)
+  end
+  
+  it "should be able to add a record with a hash" do
+    @b.add(@h)
+    @b.filtered_data.should == @h
+  end
+  
+  it "should be able to add a record with an OpenStruct" do
+    @b.add(@o)
+    @b.filtered_data.should == @h
+  end
+  
+  it "should be able to add a record with a Struct" do
+    @b.add(@s)
+    @b.filtered_data.should == @h
+  end
+  
+  it "should be able to override values" do
+    @b.add(@h)
+    @b.add(:this => 2)
+    @b.filtered_data.should == {:this => 2}
+  end
+  
+  it "should create a way to setup labels" do
+    a = [:three, :two, :one]
+    @b.labels = a
+    @b.labels.all? {|l| a.should be_include(l)}
+  end
+  
+  it "should be constructable with a hash" do
+    b = Bucket.new(@h)
+    b.filtered_data.should == @h
+  end
+  
+  it "should be constructable with an OpenStruct" do
+    b = Bucket.new(@o)
+    b.filtered_data.should == @h
+  end
+  
+  it "should be constructable with a Struct" do
+    b = Bucket.new(@s)
+    b.filtered_data.should == @h
+  end
+  
+  it "should be able to dump the contents of the bucket" do
+    @b1.dump.should == @h
+    @b1.raw_data.should == {}
+  end
+  
+  it "should be able to take an arbitrary filter" do
+    b = Bucket.new(@h) {|h| :not_the_data}
+    b.raw_data.should == @h
+    b.filtered_data.should eql(:not_the_data)
+  end
+
+  it "should be able to return an array" do
+    @b1.to_a.should eql([1])
+  end
+  
+  it "should be able to return a hash" do
+    @b1.to_hash.should == @h
+  end
+  
+  it "should be able to return any object that initializes with the bucket values" do
+    a = @b1.to_obj(A)
+    a.value.should eql(@b1.to_a)
+  end
+  
+  it "should be able to return a Struct" do
+    s = @b1.to_struct(S)
+    s.this.should eql(1)
+  end
+  
+  it "should be able to return an OpenStruct" do
+    o = @b1.to_open_struct
+    o.table.should == @h
+  end
+  
+  it "should be able to constrain and order keys, silently ignoring data that isn't white listed" do
+    h = {:ones => 1, :twos => 2, :threes => 3}
+    @b.white_list = [:ones, :twos, :threes]
+    @b.add :ones => 1, :twos => 2, :threes => 3, :fours => 4
+    @b.filtered_data.should == h
+    @b.to_a.should eql([1,2,3])
+  end
+end

data/spec/etl/csv_et_spec.rb

@@ -0,0 +1,43 @@
+require File.join(File.dirname(__FILE__), "/../spec_helper")
+require 'etl/csv_et'
+
+describe CSV::ET do
+  
+  before do
+    @csv_file = File.expand_path("#{File.dirname(__FILE__)}/../fixtures/test_file.csv")
+  end
+  
+  it "should be able to transform a csv file into an array of arrays" do
+    @etl = CSV::ET.process(:source => @csv_file)
+    @etl.data.should be_is_a(Array)
+    @etl.data.size.should eql(3)
+    @etl.data.first.should eql(["some", "data", "here"])
+    @etl.data.last.should eql([4,5,6])
+  end
+  
+  it "should be able to transforrm csv data into an array of arrays" do
+    content = File.read(@csv_file)
+    @etl = CSV::ET.process(:source => content)
+    @etl.data.should be_is_a(Array)
+    @etl.data.size.should eql(3)
+    @etl.data.first.should eql(["some", "data", "here"])
+    @etl.data.last.should eql([4,5,6])
+  end
+  
+  it "should be able to pull the header out of the extracted data" do
+    @etl = CSV::ET.process(:source => @csv_file, :extract_header => true)
+    @etl.header.should eql(["some", "data", "here"])
+    @etl.data.first.should eql([1,2,3])
+  end
+  
+  it "should be able to use the FasterCSV options" do
+    FasterCSV::Converters[:foo] = lambda{|f| :foo }
+    @etl = CSV::ET.process(
+      :source => @csv_file, 
+      :extract_header => true, 
+      :parse_with => {:converters => :foo}
+    )
+    @etl.header.should eql([:foo, :foo, :foo])
+    @etl.data.first.should eql([:foo, :foo, :foo])
+  end
+end

data/spec/etl/etl_spec.rb

@@ -0,0 +1,237 @@
+require File.join(File.dirname(__FILE__), "/../spec_helper")
+
+describe ETL do
+  
+  after(:all) do
+    FileUtils.rm_f(ETL.logger_filename)
+  end
+  
+  it "should have a series of valid states" do
+    ETL::VALID_STATES.should eql([
+      :before_extract, 
+      :extract,
+      :after_extract, 
+      :before_transform, 
+      :transform,
+      :after_transform, 
+      :before_load, 
+      :load,
+      :after_load, 
+      :complete
+    ])
+  end
+  
+  context "Class Methods" do
+    it "should be able to process the ETL class" do
+      val = ETL.process
+      val.should be_is_a(ETL)
+      val.state.should eql(:complete)
+    end
+    
+    it "should be able to run call as an alias to process" do
+      val = ETL.call
+      val.should be_is_a(ETL)
+      val.state.should eql(:complete)
+    end
+    
+    it "should have a logger" do
+      ETL.logger.should be_is_a(Log4r::Logger)
+      ETL.logger.name.should eql('ETL')
+    end
+    
+    it "should have a console logger" do
+      cl = ETL.logger.outputters.find {|l| l.is_a?(Log4r::StderrOutputter)}
+      cl.name.should eql('console')
+      cl.level.should eql(Log4r::WARN)
+      cl.formatter.should be_is_a(Log4r::PatternFormatter)
+      cl.formatter.pattern.should eql("[%l] %d :: %m")
+    end
+
+    it "should have a file logger" do
+      fl = ETL.logger.outputters.find {|l| l.is_a?(Log4r::FileOutputter)}
+      fl.name.should eql('logfile')
+      fl.filename.should match(/ETL.log$/)
+      fl.formatter.pattern.should eql("[%l] %d :: %m")
+    end
+    
+    it "should log a script to duplicate the ETL" do
+      ETL.process(:funny => :options)
+      r = Regexp.new(Regexp.escape("ETL.process(:funny => :options)"))
+      logger_contents.should match(r)
+    end
+    
+  end
+  
+  it "should have a beginning state of :before_extract" do
+    ETL.new.state.should eql(:before_extract)
+  end
+  
+  it "should have data and raw readers" do
+    e = ETL.new
+    e.should be_respond_to(:data)
+    e.should be_respond_to(:raw)
+  end
+  
+  context "Process" do
+    it "should call each transition" do
+      PostBoard.reset
+      CheckTransitions.process
+      PostBoard.board.should eql([:before_extract, :extract, :after_extract, :before_transform, :transform, :after_transform, :before_load, :load, :after_load])
+    end
+    
+    it "should use raw as a data holding bucket, useful for using post-transactional validations" do
+      PostBoard.reset
+      ShowRaw.process
+      PostBoard.board.should eql([nil, :extract, :extract, nil, :transform, :transform, nil, :load, :load])
+    end
+
+    it "should convert raw to data after each step" do
+      PostBoard.reset
+      ShowData.process
+      PostBoard.board.should eql([nil, nil, nil, :extract, :extract, :extract, :transform, :transform, :transform])
+    end
+    
+    it "should be able to reverse back to a prior state and restart" do
+      PostBoard.reset
+      counter = ShowCounter.new
+      counter.process
+      PostBoard.board.last.should eql(9)
+      counter.reverse_to(:transform)
+      counter.process
+      PostBoard.board.last.should eql(14)
+    end
+    
+    it "should move data in @raw to @data at every stage" do
+      etl = ExplicitRawToDataShow.new
+      etl.process
+      etl.data.should eql(2)
+    end
+
+  end
+end
+
+class PostBoard
+  class << self
+    def post(value)
+      self.board << value
+    end
+    
+    def board
+      @@board ||= []
+    end
+    
+    def reset
+      @@board = []
+    end
+  end
+end
+
+# Setting up for various ETL tests.  Must implement post_state with an optional paramater
+class Demo < ETL
+  before_extract :post_state
+  after_extract :post_state
+  before_transform :post_state
+  after_transform :post_state
+  before_load :post_state
+  after_load :post_state
+  
+  def extract
+    post_state(:extract)
+  end
+  
+  def transform
+    post_state(:transform)
+  end
+  
+  def load
+    post_state(:load)
+  end
+  
+end
+
+# Doesn't do much but mark that the states were passed.
+class CheckTransitions < Demo
+  def post_state(s=nil)
+    s ||= self.state
+    PostBoard.post s
+  end
+end
+
+# Marks the value of raw at every transition
+class ShowRaw < Demo
+  
+  def extract
+    @raw = :extract
+    post_state(self.raw)
+  end
+  
+  def transform
+    @raw = :transform
+    post_state(self.raw)
+  end
+  
+  def load
+    @raw = :load
+    post_state(self.raw)
+  end
+  
+  def post_state(s=nil)
+    s ||= self.raw
+    PostBoard.post s
+  end
+end
+
+class ShowData < Demo
+  
+  def extract
+    @raw = :extract
+    post_state(self.data)
+  end
+  
+  def transform
+    @raw = :transform
+    post_state(self.data)
+  end
+  
+  def load
+    @raw = :load
+    post_state(self.data)
+  end
+  
+  def post_state(s=nil)
+    s ||= self.data
+    PostBoard.post s
+  end
+end
+
+class ShowCounter < Demo
+  
+  def advance_count
+    @count = self.count + 1
+  end
+  
+  def count
+    @count ||= 0
+  end
+  
+  def extract
+    post_state
+  end
+  alias :transform :extract
+  alias :load :extract
+    
+  def post_state
+    advance_count
+    PostBoard.post self.count
+  end
+end
+
+class ExplicitRawToDataShow < ETL
+  def extract
+    @raw = 1
+  end
+  
+  def transform
+    @raw = @data + 1
+  end
+end