cranium 0.2.0

data/lib/cranium/import_strategy/truncate_insert.rb

@@ -0,0 +1,19 @@
+class Cranium::ImportStrategy::TruncateInsert < Cranium::ImportStrategy::Base
+
+  def import_from(source_table)
+    @source_table = source_table
+
+    database[target_table].truncate
+    import_new_records
+    database[@source_table].count
+  end
+
+
+
+  private
+
+  def import_new_records
+    database.run database[target_table].insert_sql(target_fields, database[@source_table].select(*source_fields))
+  end
+
+end

data/lib/cranium/import_strategy.rb

@@ -0,0 +1,9 @@
+module Cranium::ImportStrategy
+
+  autoload :Base, 'cranium/import_strategy/base'
+  autoload :DeleteInsert, 'cranium/import_strategy/delete_insert'
+  autoload :TruncateInsert, 'cranium/import_strategy/truncate_insert'
+  autoload :Delta, 'cranium/import_strategy/delta'
+  autoload :Merge, 'cranium/import_strategy/merge'
+
+end

data/lib/cranium/logging.rb

@@ -0,0 +1,15 @@
+module Cranium::Logging
+
+  def record_metric(name, value)
+    log :info, "[metrics/#{name}] #{value}"
+  end
+
+
+
+  def log(level, message)
+    Cranium.configuration.loggers.each do |logger|
+      logger.send level, message
+    end
+  end
+
+end

data/lib/cranium/profiling.rb

@@ -0,0 +1,13 @@
+BEGIN {
+  require 'ruby-prof'
+  RubyProf.start
+}
+
+END {
+  profile = RubyProf.stop
+
+  printer = RubyProf::CallStackPrinter.new profile
+  profile_path = "/tmp/" + File.basename($0).gsub(".", "_") + "_profile.html"
+  printer.print File.open(profile_path, "w"), min_percent: 1
+  puts "Profiling information saved to: " + profile_path
+}

data/lib/cranium/progress_output.rb

@@ -0,0 +1,37 @@
+require 'progressbar'
+
+class Cranium::ProgressOutput
+
+  def self.show_progress(title, total)
+    progress_bar = create_progress_bar title, total
+    yield progress_bar
+    progress_bar.finish
+  end
+
+
+
+  private
+
+  def self.create_progress_bar(title, total)
+    if STDOUT.tty?
+      ProgressBar.new(title, total, STDOUT)
+    else
+      NullProgressBar.new
+    end
+  end
+
+
+
+  class NullProgressBar
+
+    def inc
+    end
+
+
+
+    def finish
+    end
+
+  end
+
+end

data/lib/cranium/sequel/hash.rb

@@ -0,0 +1,32 @@
+require 'sequel'
+
+class Cranium::Sequel::Hash < Hash
+
+  def qualify(options)
+    invalid_options = options.keys - [:keys_with, :values_with]
+    raise ArgumentError, "Unsupported option for qualify: #{invalid_options.first}" unless invalid_options.empty?
+    Hash[qualify_fields(options[:keys_with], keys).zip qualify_fields(options[:values_with], values)]
+  end
+
+
+
+  def qualified_keys(qualifier)
+    qualify_fields qualifier, keys
+  end
+
+
+
+  def qualified_values(qualifier)
+    qualify_fields qualifier, values
+  end
+
+
+
+  private
+
+  def qualify_fields(qualifier, fields)
+    return fields if qualifier.nil?
+    fields.map { |field| Sequel.qualify qualifier, field }
+  end
+
+end

data/lib/cranium/sequel.rb

@@ -0,0 +1,5 @@
+module Cranium::Sequel
+
+  autoload :Hash, 'cranium/sequel/hash'
+
+end

data/lib/cranium/source_registry.rb

@@ -0,0 +1,21 @@
+class Cranium::SourceRegistry
+
+  def initialize
+    @sources = {}
+  end
+
+
+
+  def [](name)
+    @sources[name] or raise "Undefined source '#{name}'"
+  end
+
+
+
+  def register_source(name, &block)
+    source = Cranium::DSL::SourceDefinition.new name
+    source.instance_eval &block
+    @sources[name] = source
+  end
+
+end

data/lib/cranium/test_framework/cucumber_table.rb

@@ -0,0 +1,140 @@
+class Cranium::TestFramework::CucumberTable
+
+  def self.from_ast_table(ast_table)
+    column_types, hashes = process_ast_table ast_table
+    new remove_comment_columns(hashes), column_types
+  end
+
+
+
+  def initialize(array_of_hashes, column_types = {})
+    @pattern_replacements, @data, @column_types = {}, array_of_hashes, column_types
+  end
+
+
+
+  def with_patterns(pattern_replacements)
+    self.tap { @pattern_replacements = pattern_replacements }
+  end
+
+
+
+  def to_step_definition_arg
+    dup
+  end
+
+
+
+  def accept(_)
+  end
+
+
+
+  def fields
+    @data.first.keys
+  end
+
+
+
+  def data
+    column_count = @column_types.count
+
+    evaluate_cells.tap do |array_of_hashes|
+      array_of_hashes.define_singleton_method(:columns) do
+
+        result = self.reduce(Hash.new { |hash, key| hash[key] = [] }) do |result, current_hash|
+          current_hash.each { |key, value| result[key] << value }
+          result
+        end.values
+
+        result == [] ? Array.new(column_count) { [] } : result
+      end
+    end
+  end
+
+
+
+  def data_array
+    data.map { |hash| hash.values.first }
+  end
+
+
+
+  private
+
+  def evaluate_cells
+    @data.map do |row|
+      Hash.new.tap do |evaluated_row|
+
+        row.each do |key, value|
+          evaluated_value = evaluate_field(key, value)
+          evaluated_row[key] = evaluated_value
+        end
+      end
+    end
+
+  end
+
+
+
+  def evaluate_field(key, value)
+    if @pattern_replacements.keys.include? value
+      (@pattern_replacements[value].is_a? Proc) ? @pattern_replacements[value].() : @pattern_replacements[value]
+    else
+      case @column_types[key]
+        when :integer
+          value.to_i
+        when :numeric
+          BigDecimal.new value
+        else
+          value
+      end
+    end
+  end
+
+
+
+  def self.process_ast_table(ast_table)
+    column_types = {}
+    ast_table = ast_table.map_headers do |header|
+      header.match /^(?<name>.*?)(\s+\((?<type>\w{1,2})\))?$/ do |match|
+        match[:name].to_sym.tap do |field_name|
+          next if comment_field? field_name
+          column_types[field_name] = column_type match[:type]
+        end
+      end
+    end
+
+    return column_types, ast_table.hashes
+  end
+
+
+
+  def self.comment_field?(field_name)
+    field_name.to_s.start_with? "#"
+  end
+
+
+
+  def self.column_type(type_specifier)
+    case type_specifier
+      when "i"
+        :integer
+      when "n"
+        :numeric
+      when "s", nil
+        :string
+      else
+        raise StandardError, "Invalid type specified: #{type_specifier}"
+    end
+  end
+
+
+
+  def self.remove_comment_columns(hashes)
+    hashes.map do |hash|
+      hash.delete_if { |key| key.to_s.start_with? "#" }
+    end
+  end
+
+end

data/lib/cranium/test_framework/database_entity.rb

@@ -0,0 +1,29 @@
+class Cranium::TestFramework::DatabaseEntity
+
+  attr_reader :entity_name
+  attr_reader :connection
+
+
+
+  def initialize(entity, db_connection)
+    @entity_name, @connection = entity, db_connection
+  end
+
+
+
+  class << self
+
+    def entities_created
+      @entities_created ||= []
+    end
+
+
+
+    def cleanup
+      entities_created.each { |entity| entity.destroy }
+      @entities_created = []
+    end
+
+  end
+
+end

data/lib/cranium/test_framework/database_sequence.rb

@@ -0,0 +1,16 @@
+class Cranium::TestFramework::DatabaseSequence < Cranium::TestFramework::DatabaseEntity
+
+
+  def create
+    connection.run "CREATE sequence #{entity_name}"
+    self.class.entities_created << self
+  end
+
+
+
+  def destroy
+    connection.run "DROP SEQUENCE #{entity_name}"
+  end
+
+
+end

data/lib/cranium/test_framework/database_table.rb

@@ -0,0 +1,33 @@
+class Cranium::TestFramework::DatabaseTable < Cranium::TestFramework::DatabaseEntity
+
+
+  def create(fields)
+    connection.run "CREATE TABLE #{entity_name} (#{fields.map { |field| "#{field[:field_name]} #{field[:field_type]}" }.join ", " })"
+    self.class.entities_created << self
+  end
+
+
+
+  def destroy
+    connection.run "DROP TABLE #{entity_name}"
+  end
+
+
+
+  def content(fields = ["*".to_sym])
+    connection[entity_name].select(*fields).all
+  end
+
+
+
+  def insert(data)
+    connection[entity_name].multi_insert data
+  end
+
+
+
+  def clear
+    connection[entity_name].truncate
+  end
+
+end

data/lib/cranium/test_framework/upload_directory.rb

@@ -0,0 +1,39 @@
+require 'fileutils'
+
+class Cranium::TestFramework::UploadDirectory
+
+  def initialize(working_directory)
+    @working_directory = working_directory
+  end
+
+
+
+  def file_exists?(file_name)
+    File.exists? File.join(@working_directory, file_name)
+  end
+
+
+
+  def read_file(file_name)
+    File.read File.join(@working_directory, file_name)
+  end
+
+
+
+  def save_file(file_name, content)
+    File.open(File.join(@working_directory, file_name), "w:UTF-8") { |file| file.write content }
+  end
+
+
+
+  def delete_file(file_name)
+    File.unlink File.join(@working_directory, file_name)
+  end
+
+
+
+  def remove_directory(path)
+    FileUtils.rm_rf path
+  end
+
+end

data/lib/cranium/test_framework/world.rb

@@ -0,0 +1,66 @@
+require 'open3'
+
+class Cranium::TestFramework::World
+
+  DEFINITION_FILE = "definition.rb"
+
+  attr_reader :output, :error_output, :result_code
+
+
+
+  def initialize(working_directory, greenplum_connection)
+    @greenplum_connection = greenplum_connection
+    @directory = working_directory
+  end
+
+
+
+  def upload_directory
+    @upload_directory ||= Cranium::TestFramework::UploadDirectory.new @directory
+  end
+
+
+
+  def save_definition(definition)
+    config = <<-config_string
+      require 'logger'
+
+      Cranium.configure do |config|
+        config.greenplum_connection_string = "#{Cranium.configuration.greenplum_connection_string}"
+        config.gpfdist_url = "#{Cranium.configuration.gpfdist_url}"
+        config.gpfdist_home_directory = "#{Cranium.configuration.gpfdist_home_directory}"
+        config.upload_directory = "#{Cranium.configuration.upload_directory}"
+        config.loggers << Logger.new("log/application.log")
+      end
+    config_string
+
+    upload_directory.save_file DEFINITION_FILE, config + definition
+  end
+
+
+
+  def execute_definition
+    @output, @error_output, status = Open3.capture3("bundle exec bin/cranium --cranium-load #{@directory}/#{DEFINITION_FILE}")
+    @result_code = status.exitstatus
+  end
+
+
+
+  def script_output
+    "Output: #{@output}\n"\
+    "Error: #{@error_output}\n"\
+    "Exit status: #{@result_code}\n"
+  end
+
+
+
+  def database_table(table_name)
+    Cranium::TestFramework::DatabaseTable.new table_name.to_sym, @greenplum_connection
+  end
+
+
+  def database_sequence(sequence_name)
+    Cranium::TestFramework::DatabaseSequence.new sequence_name, @greenplum_connection
+  end
+
+end

data/lib/cranium/test_framework.rb

@@ -0,0 +1,10 @@
+module Cranium::TestFramework
+
+  autoload :CucumberTable, 'cranium/test_framework/cucumber_table'
+  autoload :DatabaseEntity, 'cranium/test_framework/database_entity'
+  autoload :DatabaseSequence, 'cranium/test_framework/database_sequence'
+  autoload :DatabaseTable, 'cranium/test_framework/database_table'
+  autoload :UploadDirectory, 'cranium/test_framework/upload_directory'
+  autoload :World, 'cranium/test_framework/world'
+
+end

data/lib/cranium/transformation/duplication_index.rb

@@ -0,0 +1,42 @@
+require 'set'
+
+class Cranium::Transformation::DuplicationIndex
+
+  def self.[](*fields)
+    raise ArgumentError, "Cannot build duplication index for empty fieldset" if fields.empty?
+    @instances ||= {}
+    @instances[fields] ||= new(*fields)
+  end
+
+
+
+  def initialize(*fields)
+    @fields = fields
+    @fingerprints = Set.new
+  end
+
+
+
+  def duplicate?(record)
+    fingerprint = take_fingerprint(record)
+
+    if @fingerprints.include? fingerprint
+      true
+    else
+      @fingerprints.add fingerprint
+      false
+    end
+  end
+
+
+
+  private
+
+  def take_fingerprint(record)
+    @fields.map do |field_name|
+      raise StandardError, "Missing deduplication key from record: #{field_name}" unless record.has_key? field_name
+      record[field_name]
+    end
+  end
+
+end

data/lib/cranium/transformation/index.rb

@@ -0,0 +1,83 @@
+class Cranium::Transformation::Index
+
+  def initialize
+    @indexes = {}
+  end
+
+
+
+  def lookup(field_name, options)
+    validate options
+
+    cache = cache_for(options[:from_table], key_fields(options), field_name)
+
+    if cache.has_key? keys(options)
+      cache[keys(options)]
+    elsif options.has_key? :if_not_found_then
+      case options[:if_not_found_then]
+        when Proc
+          options[:if_not_found_then].call
+        else
+          options[:if_not_found_then]
+      end
+    elsif options.has_key? :if_not_found_then_insert
+      cache[keys(options)] = Cranium::DimensionManager.for(options[:from_table], key_fields(options)).insert(field_name, default_value_record(options))
+    else
+      :not_found
+    end
+  end
+
+
+
+  def insert(field_name, options)
+    Cranium::DimensionManager.for(options[:table], [field_name]).insert(field_name, options[:record])
+  end
+
+
+
+  def validate(options)
+    raise ArgumentError, "Cannot specify both :if_not_found_then and :if_not_found_then_insert options" if options.has_key? :if_not_found_then_insert and options.has_key? :if_not_found_then
+  end
+
+
+
+  private
+
+  def default_value_record(options)
+    if options.has_key? :match
+      key_values = options[:match]
+    else
+      key_values = {options[:match_column] => options[:to_value]}
+    end
+    options[:if_not_found_then_insert].merge(key_values)
+  end
+
+
+
+  def cache_for(table_name, key_fields, value_field)
+    @indexes[[table_name, key_fields, value_field]] ||= Cranium::DimensionManager.for(table_name, key_fields).create_cache_for_field(value_field)
+  end
+
+
+
+  def key_fields(options)
+    if options.has_key? :match
+      key_fields = options[:match].keys
+    else
+      key_fields = [options[:match_column]]
+    end
+    key_fields
+  end
+
+
+
+  def keys(options)
+    if options.has_key? :match
+      keys = options[:match].values
+    else
+      keys = [options[:to_value]]
+    end
+    keys
+  end
+
+end