cranium 0.2.0

data/features/transform/raw_ruby_transformation.feature

@@ -0,0 +1,69 @@
+Feature: Raw Ruby transformation
+
+  Scenario: A transform block can use the record as a Hash
+    Given a "products.csv" data file containing:
+    """
+    id,name,category
+    JNI-123,Just a product name,Main category > Subcategory > Sub-subcategory
+    CDI-234,Another product name,Smart Insight > Cool stuff > Scripts
+    """
+    And the following definition:
+    """
+    source :products do
+      field :id, String
+      field :name, String
+      field :category, String
+    end
+
+    source :transformed_products do
+      field :item, String
+      field :title, String
+      field :category, String
+    end
+
+    transform :products => :transformed_products do |record|
+      record[:item] = "*#{record[:id]}*"
+      record[:title] = record[:name].chars.first
+      output record
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "transformed_products.csv" data file in the upload directory containing:
+    """
+    item,title,category
+    *JNI-123*,J,Main category > Subcategory > Sub-subcategory
+    *CDI-234*,A,Smart Insight > Cool stuff > Scripts
+    """
+
+
+  Scenario: Records can be skipped
+    Given a "products.csv" data file containing:
+    """
+    id
+    1
+    2
+    3
+    """
+    And the following definition:
+    """
+    source :products do
+      field :id, Integer
+    end
+
+    source :transformed_products do
+      field :id, Integer
+    end
+
+    transform :products => :transformed_products do |record|
+      output record unless "2" == record[:id]
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "transformed_products.csv" data file in the upload directory containing:
+    """
+    id
+    1
+    3
+    """

data/features/transform/split_field.feature

@@ -0,0 +1,39 @@
+Feature: Split field
+
+  Scenario: A single field can be split into multiple fields
+    Given a "products.csv" data file containing:
+    """
+    id,name,category
+    JNI-123,Just a product name,Main category > Subcategory > Sub-subcategory > Ultra-subcategory
+    CDI-234,Another product name,Smart Insight > Cool stuff | 3dim > 2dim > 1dim
+    """
+    And the following definition:
+    """
+    source :products do
+      field :item, String
+      field :title, String
+      field :category, String
+    end
+
+    source :transformed_products do
+      field :item, String
+      field :title, String
+      field :main_category, String
+      field :sub_category, String
+      field :department, String
+    end
+
+    transform :products => :transformed_products do |record|
+      record.split_field :category, into: [:category], by: "|"
+      record.split_field :category, into: [:main_category, :sub_category, :department], by: ">"
+      output record
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "transformed_products.csv" data file in the upload directory containing:
+    """
+    item,title,main_category,sub_category,department
+    JNI-123,Just a product name,Main category,Subcategory,Sub-subcategory
+    CDI-234,Another product name,Smart Insight,Cool stuff,Cool stuff
+    """

data/lib/cranium/application.rb

@@ -0,0 +1,104 @@
+class Cranium::Application
+
+  include Cranium::Logging
+
+  attr_reader :sources
+
+
+
+  def initialize(arguments)
+    @sources = Cranium::SourceRegistry.new
+    @hooks = {}
+
+    @options = Cranium::CommandLineOptions.new arguments
+  end
+
+
+
+  def load_arguments
+    options.load_arguments
+  end
+
+
+
+  def cranium_arguments
+    options.cranium_arguments
+  end
+
+
+
+  def register_source(name, &block)
+    @sources.register_source(name, &block).resolve_files
+  end
+
+
+
+  def run
+    process_file = validate_file options.cranium_arguments[:load]
+
+    begin
+      load process_file
+    rescue Exception => ex
+      log :error, ex
+      raise
+    ensure
+      apply_hook :after
+    end
+  end
+
+
+
+  def after_import(&block)
+    register_hook :after_import, &block
+  end
+
+
+
+  def register_hook(name, &block)
+    @hooks[name] ||= []
+    @hooks[name] << block
+  end
+
+
+
+  def apply_hook(name)
+    unless @hooks[name].nil?
+      @hooks[name].each do |block|
+        block.call
+      end
+    end
+  end
+
+
+
+  private
+
+  attr_reader :options
+
+
+
+  def validate_file(load_file)
+    exit_if_no_file_specified load_file
+    exit_if_no_such_file_exists load_file
+    load_file
+  end
+
+
+
+  def exit_if_no_file_specified(file)
+    if file.nil? || file.empty?
+      $stderr.puts "ERROR: No file specified"
+      exit 1
+    end
+  end
+
+
+
+  def exit_if_no_such_file_exists(file)
+    unless File.exists? file
+      $stderr.puts "ERROR: File '#{file}' does not exist"
+      exit 1
+    end
+  end
+
+end

data/lib/cranium/archiver.rb

@@ -0,0 +1,36 @@
+require 'fileutils'
+
+module Cranium::Archiver
+
+  def self.archive(*files)
+    create_archive_directory
+    archive_files files
+  end
+
+
+
+  def self.remove(*files)
+    files.each do |file_name|
+      FileUtils.rm File.join(Cranium.configuration.upload_path, file_name)
+    end
+  end
+
+
+
+  private
+
+  def self.create_archive_directory
+    FileUtils.mkpath Cranium.configuration.archive_directory unless Dir.exists? Cranium.configuration.archive_directory
+  end
+
+
+
+  def self.archive_files(files)
+    archive_datetime = Time.now.strftime("%Y-%m-%d_%Hh%Mm%Ss")
+    files.each do |file_name|
+      FileUtils.mv File.join(Cranium.configuration.upload_path, file_name),
+                   File.join(Cranium.configuration.archive_directory, "#{archive_datetime}_#{file_name}")
+    end
+  end
+
+end

data/lib/cranium/attribute_dsl.rb

@@ -0,0 +1,43 @@
+module Cranium::AttributeDSL
+
+  def define_attribute(name)
+    class_eval <<-attribute_method
+
+      def #{name}(*args)
+        return @#{name} if args.count.zero?
+
+        @#{name} = args.first
+      end
+
+    attribute_method
+  end
+
+
+
+  def define_array_attribute(name)
+    class_eval <<-attribute_method
+
+      def #{name}(*args)
+        return @#{name} || [] if args.count.zero?
+
+        @#{name} = args
+      end
+
+    attribute_method
+  end
+
+
+
+  def define_boolean_attribute(name)
+    class_eval <<-attribute_method
+
+      def #{name}(*args)
+        return !!@#{name} if args.count.zero?
+
+        @#{name} = !!args
+      end
+
+    attribute_method
+  end
+
+end

data/lib/cranium/command_line_options.rb

@@ -0,0 +1,27 @@
+require "slop"
+
+class Cranium::CommandLineOptions
+
+  def initialize(arguments)
+    @arguments = Slop.parse(arguments, autocreate: true).to_hash
+  end
+
+
+
+  def cranium_arguments
+    @cranium_arguments ||= Hash[arguments.map { |k, v| [$1.to_sym, v] if k.to_s =~ /\Acranium\-(.*)/ }.compact]
+  end
+
+
+
+  def load_arguments
+    @load_arguments ||= Hash[arguments.map { |k, v| [k, v] unless k.to_s =~ /\Acranium\-(.*)/ }.compact]
+  end
+
+
+
+  private
+
+  attr_reader :arguments
+
+end

data/lib/cranium/configuration.rb

@@ -0,0 +1,33 @@
+class Cranium::Configuration
+
+  STORAGE_DIRECTORY_NAME = ".cranium"
+
+  attr_writer :storage_directory
+  attr_accessor :archive_directory
+  attr_accessor :greenplum_connection_string
+  attr_accessor :mysql_connection_string
+  attr_accessor :upload_directory
+  attr_accessor :gpfdist_home_directory
+  attr_accessor :gpfdist_url
+  attr_accessor :loggers
+
+
+
+  def initialize
+    @loggers = []
+  end
+
+
+
+  def upload_path
+    File.join gpfdist_home_directory, upload_directory
+  end
+
+
+
+  def storage_directory
+    return @storage_directory unless @storage_directory.nil?
+    File.join upload_path, STORAGE_DIRECTORY_NAME
+  end
+
+end

data/lib/cranium/data_importer.rb

@@ -0,0 +1,35 @@
+class Cranium::DataImporter
+
+  include Cranium::Logging
+
+
+  def import(import_definition)
+    number_of_items_imported = 0
+    Cranium::Database.connection.transaction do
+      number_of_items_imported = importer_for_definition(import_definition).import
+      Cranium.application.apply_hook(:after_import)
+    end
+
+    record_metric import_definition.name, number_of_items_imported.to_s
+  end
+
+
+  private
+
+  def importer_for_definition(import_definition)
+    if [!import_definition.merge_fields.empty?, !import_definition.delete_insert_on.empty?, import_definition.truncate_insert].count(true) > 1
+      raise StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+    end
+
+    if !import_definition.merge_fields.empty?
+      Cranium::ImportStrategy::Merge.new(import_definition)
+    elsif !import_definition.delete_insert_on.empty?
+      Cranium::ImportStrategy::DeleteInsert.new(import_definition)
+    elsif import_definition.truncate_insert
+      Cranium::ImportStrategy::TruncateInsert.new(import_definition)
+    else
+      Cranium::ImportStrategy::Delta.new(import_definition)
+    end
+  end
+
+end

data/lib/cranium/data_reader.rb

@@ -0,0 +1,48 @@
+require 'csv'
+require 'cranium/extensions/file'
+
+class Cranium::DataReader
+
+  def initialize(source)
+    @source = source
+    @source_field_names = @source.fields.keys
+  end
+
+
+
+  def read(&block)
+    @source.files.each do |input_file|
+      read_input_file File.join(Cranium.configuration.upload_path, input_file), block
+    end
+  end
+
+
+
+  private
+
+  def read_input_file(input_file, read_block)
+    Cranium::ProgressOutput.show_progress File.basename(input_file), File.line_count(input_file) do |progress_bar|
+      line_number = 0
+      CSV.foreach input_file, csv_read_options_for(@source) do |row|
+        next if 1 == (line_number += 1)
+
+        record = Hash[@source_field_names.zip row]
+        self.instance_exec record, &read_block
+
+        progress_bar.inc
+      end
+    end
+  end
+
+
+
+  def csv_read_options_for(source_definition)
+    {
+      encoding: source_definition.encoding,
+      col_sep: source_definition.delimiter,
+      quote_char: source_definition.quote,
+      return_headers: false
+    }
+  end
+
+end

data/lib/cranium/data_transformer.rb

@@ -0,0 +1,126 @@
+require 'csv'
+require 'cranium/extensions/file'
+
+class Cranium::DataTransformer
+
+  def initialize(source, target)
+    @source, @target = source, target
+    @index = Cranium::Transformation::Index.new
+    @target_fields = @target.fields.keys
+    @record = Cranium::TransformationRecord.new @source.fields.keys, @target_fields
+  end
+
+
+
+  def transform(&block)
+    raise StandardError, "Source definition '#{@target.name}' cannot overrride the file name because it is a transformation target" if @target.file_name_overriden?
+
+    CSV.open "#{Cranium.configuration.upload_path}/#{@target.file}", "w:#{@target.encoding}", csv_write_options_for(@target) do |target_file|
+      @target_file = target_file
+      @source.files.each do |input_file|
+        transform_input_file File.join(Cranium.configuration.upload_path, input_file), block
+      end
+    end
+
+    @target.resolve_files
+  end
+
+
+
+  private
+
+  def transform_input_file(input_file, transformation_block)
+    Cranium::ProgressOutput.show_progress File.basename(input_file), File.line_count(input_file) do |progress_bar|
+      line_number = 0
+      CSV.foreach input_file, csv_read_options_for(@source) do |row|
+        next if 1 == (line_number += 1)
+
+        @record.input_data = row
+        self.instance_exec @record, &transformation_block
+
+        progress_bar.inc
+      end
+    end
+  end
+
+
+
+  def csv_write_options_for(source_definition)
+    {
+      col_sep: source_definition.delimiter,
+      quote_char: source_definition.quote,
+      write_headers: true,
+      headers: source_definition.fields.keys
+    }
+  end
+
+
+
+  def csv_read_options_for(source_definition)
+    {
+      encoding: source_definition.encoding,
+      col_sep: source_definition.delimiter,
+      quote_char: source_definition.quote,
+      return_headers: false
+    }
+  end
+
+
+
+  def output(record)
+    @target_file << prepare_for_output(case record
+                                         when Cranium::TransformationRecord
+                                           record.data
+                                         when Hash
+                                           record
+                                         else
+                                           raise ArgumentError, "Cannot write '#{record.class}' to file as CSV record"
+                                       end)
+  end
+
+
+
+  def prepare_for_output(hash)
+    hash.
+      keep_if { |key| @target_fields.include? key }.
+      sort_by { |field, _| @target_fields.index(field) }.
+      map { |item| item[1] }.
+      map { |value| strip(value) }
+  end
+
+
+
+  def strip(value)
+    return value unless value.respond_to? :strip
+    value.strip
+  end
+
+
+
+  def unique_on_fields?(*fields)
+    not Cranium::Transformation::DuplicationIndex[*fields].duplicate? @record
+  end
+
+
+
+  def lookup(field_name, settings)
+    @index.lookup field_name, settings
+  end
+
+
+
+  def insert(field_name, settings)
+    @index.insert field_name, settings
+  end
+
+
+
+  def next_value_in_sequence(name)
+    Cranium::Transformation::Sequence.by_name name
+  end
+
+
+
+  alias_method :sequence, :next_value_in_sequence
+
+end

data/lib/cranium/database.rb

@@ -0,0 +1,36 @@
+require 'sequel'
+require 'sequel/extensions/connection_validator'
+
+module Cranium::Database
+
+  def self.connection
+    @connection ||= setup_connection(Cranium.configuration.greenplum_connection_string)
+  end
+
+
+
+  def self.[](name)
+    @connections ||= {}
+    @connections[name] ||= setup_connection(@definitions[name].connect_to)
+  end
+
+
+
+  def self.register_database(name, &block)
+    @definitions ||= Cranium::DefinitionRegistry.new Cranium::DSL::DatabaseDefinition
+    @definitions.register_definition name, &block
+  end
+
+
+
+  private
+
+
+  def self.setup_connection(connection_string)
+    connection = Sequel.connect connection_string, loggers: Cranium.configuration.loggers
+    connection.extension :connection_validator
+    connection.pool.connection_validation_timeout = -1
+    return connection
+  end
+
+end

data/lib/cranium/definition_registry.rb

@@ -0,0 +1,21 @@
+class Cranium::DefinitionRegistry
+
+  def initialize(definition_class)
+    @definition_class = definition_class
+    @definitions = {}
+  end
+
+
+
+  def [](name)
+    @definitions[name]
+  end
+
+
+
+  def register_definition(name, &block)
+    definition = @definition_class.new name
+    definition.instance_eval &block
+    @definitions[name] = definition
+  end
+end

data/lib/cranium/dimension_manager.rb

@@ -0,0 +1,65 @@
+class Cranium::DimensionManager
+
+  attr_reader :rows
+
+
+
+  def self.for(table_name, key_fields)
+    @instances ||= {}
+    @instances[[table_name, key_fields]] ||= self.new table_name, key_fields
+  end
+
+
+
+  def initialize(table_name, key_fields)
+    @table_name, @key_fields = table_name, key_fields
+    @rows = []
+
+    Cranium.application.after_import { flush }
+  end
+
+
+
+  def insert(target_key, row)
+    raise ArgumentError, "Required attribute '#{target_key}' missing" unless row.has_key? target_key
+
+    @rows << resolve_sequence_values(row)
+    row[target_key]
+  end
+
+
+
+  def create_cache_for_field(value_field)
+    to_multi_key_cache(db.select_map(@key_fields + [value_field]))
+  end
+
+
+
+  def flush
+    db.multi_insert(@rows) unless @rows.empty?
+    @rows = []
+  end
+
+
+
+  private
+
+  def to_multi_key_cache(table_data)
+    Hash[table_data.map { |row| [row[0..-2], row.last] }]
+  end
+
+
+
+  def resolve_sequence_values(row)
+    row.each do |key, value|
+      row[key] = value.next_value if value.is_a? Cranium::Transformation::Sequence
+    end
+  end
+
+
+
+  def db
+    Cranium::Database.connection[@table_name]
+  end
+
+end