chicago-etl 0.0.9

data/lib/chicago/etl/load_dataset_builder.rb

@@ -0,0 +1,75 @@
+require 'set'
+
+module Chicago
+  module ETL
+    class LoadDatasetBuilder
+      def initialize(&block)
+        @constructed_columns = {}
+        @joins = []
+        configure(&block) if block_given?
+      end
+
+      def configure(&block)
+        instance_eval(&block)
+        self
+      end
+
+      def table(table_name)
+        @table_name = table_name
+        self
+      end
+
+      def denormalize(table, keys)
+        @joins << [:left_outer, table, keys]
+        self
+      end
+
+      def provide(target_name, source_column)
+        @constructed_columns[target_name] = source_column
+        self
+      end
+
+      def build(db, columns)
+        dataset = @joins.inject(db[@table_name]) {|ds, join|
+          ds.join_table(*join)
+        }
+
+        available_columns = available_columns_index(db, dataset)
+
+        select_columns = columns.map {|name|
+          if @constructed_columns[name].kind_of?(Symbol)
+            qualify_column(available_columns, @constructed_columns[name]).as(name)
+          elsif @constructed_columns[name]
+            @constructed_columns[name].as(name)
+          else
+            qualify_column(available_columns, name)
+          end
+        }
+
+        dataset.select(*select_columns)
+      end
+      
+      private
+
+      def available_columns_index(db, dataset)
+        dataset.dependant_tables.inject({}) do |hsh, table|
+          db[table].columns.each do |column|
+            (hsh[column] ||= Set.new) << table
+          end
+          hsh
+        end
+      end
+
+      def qualify_column(available_columns, name)
+        if available_columns[name] && available_columns[name].size == 1
+          name.qualify(available_columns[name].first)
+        elsif name == :id
+          name.qualify(@table_name)
+        else
+          raise "Column #{name} was either ambiguous or non-existant"
+        end
+      end
+        
+    end
+  end
+end

data/lib/chicago/etl/mysql_dumpfile.rb

@@ -0,0 +1,32 @@
+require 'chicago/etl/sink'
+
+module Chicago
+  module ETL
+    # Wrapper around FasterCSV's output object, to convert values to a
+    # format required by MySQL's LOAD DATA INFILE command.
+    #
+    # @api public
+    class MysqlDumpfile < Sink
+      # Creates a new writer.
+      #
+      # @param csv a FasterCSV output object
+      # @param [Symbol] column_names columns to be output
+      # @param key an optional key to ensure rows are written only once.
+      def initialize(csv, column_names, key=nil)
+        super(csv, column_names, key)
+        @transformer = MysqlLoadFileValueTransformer.new
+      end
+
+      protected
+
+      # Writes a row to the output.
+      #
+      # @param Hash row Only keys in column_names will be output.
+      def write(row)
+        output << @column_names.map {|name| 
+          @transformer.transform(row[name]) 
+        }
+      end
+    end
+  end
+end

data/lib/chicago/etl/mysql_load_file_value_transformer.rb

@@ -0,0 +1,24 @@
+module Chicago
+  module ETL
+    class MysqlLoadFileValueTransformer
+      # Transforms a value to be suitable for use in file in a LOAD
+      # DATA INFILE mysql statement.
+      def transform(value)
+        case value
+        when nil
+          "\\N"
+        when true
+          "1"
+        when false
+          "0"
+        when Time, DateTime
+          value.strftime("%Y-%m-%d %H:%M:%S")
+        when Date
+          value.strftime("%Y-%m-%d")
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/chicago/etl/screens/column_screen.rb

@@ -0,0 +1,59 @@
+module Chicago
+  module ETL
+    module Screens
+      class ColumnScreen
+        attr_reader :column, :table_name
+        
+        def initialize(table_name, column)
+          @table_name = table_name
+          @column = column
+          @error_name = self.class.name.split('::').last.sub(/Screen$/,'').titlecase
+        end
+
+        def self.for_columns(table_name, columns)
+          screens = columns.map {|column| new(table_name, column) }
+          CompositeScreen.new(screens)
+        end
+
+        def call(row, errors=[])
+          value = row[column.database_name]
+
+          if applies?(value)
+            overwrite_value(row)
+            log_error(value, errors)
+          end
+
+          [row, errors]
+        end
+
+        def severity
+          1
+        end
+
+        private
+
+        def overwrite_value(row)
+          row[column.database_name] = column.default_value
+        end
+
+        def log_error(value, errors)
+          errors << error_hash(value)
+        end
+
+        def error_hash(value)
+          {
+            :process_name => "StandardTransformations",
+            :process_version => 2,
+            :table => table_name.to_s,
+            :column => column.database_name.to_s,
+            :severity => severity,
+            :error => @error_name
+          }
+        end
+
+        def applies?(value)
+        end
+      end
+    end
+  end
+end

data/lib/chicago/etl/screens/composite_screen.rb

@@ -0,0 +1,17 @@
+module Chicago
+  module ETL
+    module Screens
+      class CompositeScreen
+        def initialize(*screens)
+          @screens = screens.flatten
+        end
+
+        def call(row, errors=[])
+          @screens.inject([row,errors]) do |(row, errors), screen| 
+            screen.call(row, errors) 
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chicago/etl/screens/invalid_element.rb

@@ -0,0 +1,27 @@
+module Chicago
+  module ETL
+    module Screens
+      class InvalidElement < ColumnScreen
+        def self.for_columns(table_name, columns)
+          screens = columns.select(&:elements).
+            map {|column| new(table_name, column) }
+          CompositeScreen.new(screens)
+        end
+
+        def severity
+          3
+        end
+
+        def applies?(value)
+          column.elements && 
+            !column.elements.map(&:downcase).include?(value.to_s.downcase)
+        end
+
+        def error_hash(value)
+          super(value).
+            merge(:error_detail => "'#{value}' is not a valid value.")
+        end
+      end
+    end
+  end
+end

data/lib/chicago/etl/screens/missing_value.rb

@@ -0,0 +1,22 @@
+module Chicago
+  module ETL
+    module Screens
+      class MissingValue < ColumnScreen
+        def severity
+          column.descriptive? ? 1 : 2
+        end
+
+        def log_error(value, errors)
+          if ! (column.column_type == :boolean || column.optional?)
+            errors << error_hash(value)
+          end
+        end
+
+        def applies?(value)
+          value.nil? ||
+            (column.column_type == :string && value.blank?)
+        end
+      end
+    end
+  end
+end

data/lib/chicago/etl/screens/out_of_bounds.rb

@@ -0,0 +1,33 @@
+module Chicago
+  module ETL
+    module Screens
+      class OutOfBounds < ColumnScreen
+        def severity
+          2
+        end
+
+        def applies?(value)
+          return false unless value
+
+          (column.numeric? && applies_to_numeric?(value)) ||
+            (column.column_type == :string && applies_to_string?(value))
+        end
+
+        def overwrite_value(row)
+        end
+
+        private
+
+        def applies_to_numeric?(value)
+          (column.min && value < column.min) || 
+            (column.max && value > column.max)
+        end
+
+        def applies_to_string?(value)
+          (column.min && value.length < column.min) || 
+            (column.max && value.length > column.max)
+        end
+      end
+    end
+  end
+end

data/lib/chicago/etl/sequel/dependant_tables.rb

@@ -0,0 +1,48 @@
+require 'sequel'
+
+module Chicago
+  module ETL
+    module SequelExtensions
+      module DependantTables
+        # Returns an Array of table names used in this dataset.
+        #
+        # Handles joins and if the recurse flag is true, unions and
+        # nested datasets.
+        def dependant_tables(recurse=true)
+          tables = extract_dependant_tables_in_clause(opts[:from].first, recurse)
+
+          if opts[:compounds]
+            tables += opts[:compounds].map {|(_, dataset, _)|
+              dataset.dependant_tables
+            }
+          end
+
+          if opts[:join]
+            tables += opts[:join].map {|join| 
+              extract_dependant_tables_in_clause(join.table, recurse)
+            }
+          end
+
+          tables.flatten.uniq
+        end
+
+        private
+        
+        def extract_dependant_tables_in_clause(clause, recurse)
+          case clause
+          when Symbol
+            [clause]
+          when Sequel::SQL::AliasedExpression
+            extract_dependant_tables_in_clause(clause.expression, recurse)
+          when Sequel::Dataset
+            recurse ? clause.dependant_tables : []
+          else
+            []
+          end
+        end
+      end
+    end
+  end
+end
+
+Sequel::Dataset.send :include, Chicago::ETL::SequelExtensions::DependantTables

data/lib/chicago/etl/sequel/filter_to_etl_batch.rb

@@ -0,0 +1,53 @@
+module Chicago
+  module ETL
+    module SequelExtensions
+      module FilterToEtlBatch
+        def filter_to_etl_batch(etl_batch)
+          conditions = (opts[:from] + (opts[:join] || [])).
+            select {|e| has_etl_batch_column?(e) }.
+            map {|e| make_etl_batch_filter(e, etl_batch) }
+
+          ds = conditions.any? ? filter(conditions.inject {|a,b| a | b}) : dup
+
+          if ds.opts[:compounds]
+            ds.opts[:compounds].each do |compound|
+              compound[1] = compound[1].filter_to_etl_batch(etl_batch)
+            end
+          end
+
+          ds
+        end
+
+        private
+
+        def make_etl_batch_filter(expression, etl_batch)
+          table = case expression
+                  when Sequel::SQL::AliasedExpression
+                    expression.aliaz
+                  when Sequel::SQL::JoinClause
+                    expression.table_alias || expression.table
+                  else
+                    expression
+                  end
+
+          {:etl_batch_id.qualify(table) => etl_batch.id}
+        end
+
+        def has_etl_batch_column?(expression)
+          case expression
+          when Sequel::SQL::AliasedExpression
+            has_etl_batch_column?(expression.expression)
+          when Symbol
+            db.schema(expression).map(&:first).include?(:etl_batch_id)
+          when Sequel::SQL::JoinClause
+            has_etl_batch_column?(expression.table)
+          else
+            false
+          end
+        end
+      end
+    end
+  end
+end
+
+Sequel::Dataset.send :include, Chicago::ETL::SequelExtensions::FilterToEtlBatch

data/lib/chicago/etl/sequel/load_data_infile.rb

@@ -0,0 +1,19 @@
+module Chicago
+  module ETL
+    module SequelExtensions
+      module LoadDataInfile
+        # Loads the CSV data columns in filepath into this dataset's table.
+        def load_csv_infile(filepath, columns)
+          execute_dui(load_csv_infile_sql(filepath, columns))
+        end
+        
+        def load_csv_infile_sql(filepath, columns)
+          replacement = opts[:insert_ignore] ? "IGNORE" : "REPLACE"
+          "LOAD DATA INFILE '#{filepath}' #{replacement} INTO TABLE `#{opts[:from].first}` CHARACTER SET 'utf8' FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\"' (`#{columns.join('`,`')}`);"
+        end
+      end
+    end
+  end
+end
+
+Sequel::Dataset.send :include, Chicago::ETL::SequelExtensions::LoadDataInfile

data/lib/chicago/etl/sink.rb

@@ -0,0 +1,61 @@
+require 'set'
+
+module Chicago
+  module ETL
+    # An end point to write rows.
+    #
+    # @abstract
+    # @api public
+    class Sink
+      # Returns the column names expected to be written to this sink.
+      # @api public
+      attr_reader :column_names
+
+      # @abstract
+      def initialize(output, column_names, unique_row_key=nil)
+        @output = output
+        @column_names = column_names
+        @written_rows = Set.new
+        @unique_row_key = unique_row_key
+      end
+
+      # Writes a row to the output.
+      #
+      # Row will not be written to the output if it has already been
+      # written, as identified by the unique row key.
+      #
+      # Should not be overridden by subclasses - overwrite write instead.
+      def <<(row)
+        unless written?(row)
+          write row
+          @written_rows << row[@unique_row_key]
+        end
+      end
+      
+      # Flushes any remaining writes to the output.
+      #
+      # By default does nothing, subclasses should override where
+      # necessary.
+      def flush
+      end
+
+      # Returns true if this row has previously been written to the
+      # output.
+      #
+      # Always returns false if no key to determine row uniqueness has
+      # been provided.
+      def written?(row)
+        return false if @unique_row_key.nil?
+        @written_rows.include?(row[@unique_row_key])
+      end
+
+      protected
+
+      attr_reader :output
+
+      # @abstract
+      def write(row)
+      end
+    end
+  end
+end