cranium 0.2.0

data/lib/cranium/transformation/join.rb

@@ -0,0 +1,141 @@
+require 'csv'
+
+class Cranium::Transformation::Join
+
+  attr_accessor :source_left, :source_right, :target, :match_fields, :type
+
+
+
+  def execute
+    validate_parameters
+    cache_commonly_used_values
+    join_sources
+  end
+
+
+
+  private
+
+  def validate_parameters
+    raise "Missing left source for join transformation" if source_left.nil?
+    raise "Missing right source for join transformation" if source_right.nil?
+    raise "Missing target for join transformation" if target.nil?
+    raise "Invalid match fields for join transformation" unless match_fields.nil? or match_fields.is_a? Hash
+    raise "Invalid type for join transformation" unless %i(inner left).include?(type)
+  end
+
+
+
+  def cache_commonly_used_values
+    @left_source_field_names = source_left.fields.keys
+    @right_source_field_names = source_right.fields.keys
+    @target_field_names = target.fields.keys
+  end
+
+
+
+  def join_sources
+    build_join_table_from_right_source_files
+    write_output_file
+  end
+
+
+
+  def build_join_table_from_right_source_files
+    @join_table = {}
+    source_right.files.each do |file|
+      build_join_table_from_file file
+    end
+  end
+
+
+
+  def build_join_table_from_file(file)
+    line_number = 0
+    CSV.foreach File.join(Cranium.configuration.upload_path, file), csv_read_options_for(source_right) do |row|
+      next if 1 == (line_number += 1)
+
+      record = Hash[@right_source_field_names.zip row]
+      index_key = right_index_key_for record
+      if @join_table.has_key? index_key
+        @join_table[index_key] << record
+      else
+        @join_table[index_key] = [record]
+      end
+    end
+  end
+
+
+
+  def write_output_file
+    CSV.open "#{Cranium.configuration.upload_path}/#{target.file}", "w:#{target.encoding}", csv_write_options_for(target) do |target_file|
+      @target_file = target_file
+      source_left.files.each do |file|
+        process_left_source_file File.join(Cranium.configuration.upload_path, file), target_file
+      end
+    end
+
+    @target.resolve_files
+  end
+
+
+
+  def process_left_source_file(input_file, output_file)
+    line_number = 0
+    CSV.foreach input_file, csv_read_options_for(source_left) do |row|
+      next if 1 == (line_number += 1)
+
+      record = Hash[@left_source_field_names.zip row]
+
+      joined_records = joined_records_for(record)
+      joined_records << record if joined_records.empty? && type == :left
+
+      joined_records.each do |record_to_output|
+        output_file << @target_field_names.map { |field| record_to_output[field] }
+      end
+    end
+  end
+
+
+
+  def joined_records_for(record)
+    index_key = left_index_key_for record
+    return [] unless @join_table.has_key? index_key
+    @join_table[index_key].map { |matching_record| record.merge matching_record }
+  end
+
+
+
+  def left_index_key_for(record)
+    record.select { |field, _| match_fields.values.include? field }.values
+  end
+
+
+
+  def right_index_key_for(record)
+    record.select { |field, _| match_fields.keys.include? field }.values
+  end
+
+
+
+  def csv_write_options_for(source_definition)
+    {
+      col_sep: source_definition.delimiter,
+      quote_char: source_definition.quote,
+      write_headers: true,
+      headers: source_definition.fields.keys
+    }
+  end
+
+
+
+  def csv_read_options_for(source_definition)
+    {
+      encoding: source_definition.encoding,
+      col_sep: source_definition.delimiter,
+      quote_char: source_definition.quote,
+      return_headers: false
+    }
+  end
+
+end

data/lib/cranium/transformation/sequence.rb

@@ -0,0 +1,42 @@
+class Cranium::Transformation::Sequence
+
+  attr_reader :name
+
+
+
+  def initialize(name)
+    @name = name
+  end
+
+
+
+  def next_value
+    if @current_value.nil?
+      @current_value = Cranium::Database.connection["SELECT nextval('#{@name}') AS next_value"].first[:next_value]
+    else
+      @current_value += 1
+    end
+  end
+
+
+
+  def flush
+    Cranium::Database.connection.run "SELECT setval('#{@name}', #{@current_value})" unless @current_value.nil?
+  end
+
+
+
+  class << self
+
+    def by_name(name)
+      @sequences ||= {}
+      if @sequences[name].nil?
+        @sequences[name] = new name
+        Cranium.application.after_import { @sequences[name].flush }
+      end
+      @sequences[name]
+    end
+
+  end
+
+end

data/lib/cranium/transformation.rb

@@ -0,0 +1,8 @@
+module Cranium::Transformation
+
+  autoload :DuplicationIndex, 'cranium/transformation/duplication_index'
+  autoload :Index, 'cranium/transformation/index'
+  autoload :Join, 'cranium/transformation/join'
+  autoload :Sequence, 'cranium/transformation/sequence'
+
+end

data/lib/cranium/transformation_record.rb

@@ -0,0 +1,45 @@
+class Cranium::TransformationRecord
+
+  attr_reader :data
+
+
+
+  def initialize(source_fields, target_fields)
+    @source_fields, @target_fields = source_fields, target_fields
+  end
+
+
+
+  def input_data=(values)
+    @data = Hash[@source_fields.zip values]
+  end
+
+
+
+  def [](field)
+    @data[field]
+  end
+
+
+
+  def []=(field, value)
+    @data[field] = value
+  end
+
+
+
+  def split_field(field, options)
+    values = @data[field].split(options[:by])
+
+    options[:into].each_with_index do |target_field, index|
+      @data[target_field] = values[index] || options[:default_value] || values.last
+    end
+  end
+
+
+
+  def has_key?(key)
+    @data.has_key? key
+  end
+
+end

data/lib/cranium.rb

@@ -0,0 +1,57 @@
+module Cranium
+
+  autoload :Application, 'cranium/application'
+  autoload :Archiver, 'cranium/archiver'
+  autoload :AttributeDSL, 'cranium/attribute_dsl'
+  autoload :CommandLineOptions, 'cranium/command_line_options'
+  autoload :Configuration, 'cranium/configuration'
+  autoload :Database, 'cranium/database'
+  autoload :DataImporter, 'cranium/data_importer'
+  autoload :DataReader, 'cranium/data_reader'
+  autoload :DataTransformer, 'cranium/data_transformer'
+  autoload :DefinitionRegistry, 'cranium/definition_registry'
+  autoload :DimensionManager, 'cranium/dimension_manager'
+  autoload :DSL, 'cranium/dsl'
+  autoload :ExternalTable, 'cranium/external_table'
+  autoload :Extract, 'cranium/extract'
+  autoload :ImportStrategy, 'cranium/import_strategy'
+  autoload :Logging, 'cranium/logging'
+  autoload :ProgressOutput, 'cranium/progress_output'
+  autoload :Sequel, 'cranium/sequel'
+  autoload :SourceRegistry, 'cranium/source_registry'
+  autoload :TestFramework, 'cranium/test_framework'
+  autoload :TransformationRecord, 'cranium/transformation_record'
+  autoload :Transformation, 'cranium/transformation'
+
+  class << self
+
+    def application(argv = [])
+      @application ||= Application.new(argv)
+    end
+
+
+
+    def configuration
+      @configuration ||= Configuration.new.freeze
+    end
+
+
+
+    def configure
+      mutable_configuration = configuration.dup
+      yield mutable_configuration
+      @configuration = mutable_configuration
+      @configuration.freeze
+    end
+
+
+
+    def load_arguments
+      application.load_arguments
+    end
+
+  end
+
+end
+
+self.extend Cranium::DSL

data/rake/test.rake

@@ -0,0 +1,31 @@
+require 'cucumber'
+require 'cucumber/rake/task'
+require 'rspec/core/rake_task'
+
+task :default => :test
+
+desc "Run test suite (all RSpec examples and Cucumber features)"
+task :test => [:'test:spec', :'test:features']
+
+desc "Run RSpec code examples (options: RSPEC_SEED=seed)"
+task :spec => :'test:spec'
+
+desc "Run Cucumber features (options: CUCUMBER_SEED=seed)"
+task :features => :'test:features'
+
+
+namespace :test do
+
+  desc "Run RSpec code examples (options: RSPEC_SEED=seed)"
+  RSpec::Core::RakeTask.new :spec do |task|
+    task.verbose = false
+    task.rspec_opts = "--order random"
+    task.rspec_opts << " --seed #{ENV['RSPEC_SEED']}" if ENV['RSPEC_SEED']
+  end
+
+
+  Cucumber::Rake::Task.new(:features, "Run Cucumber features (options: CUCUMBER_SEED=seed)") do |task|
+    task.cucumber_opts = %w[--profile build]
+  end
+
+end

data/spec/cranium/application_spec.rb

@@ -0,0 +1,166 @@
+require_relative '../spec_helper'
+
+describe Cranium::Application do
+
+  let(:application) { Cranium::Application.new [] }
+
+
+  describe "#load_arguments" do
+    it "should return the provided load arguments" do
+      app = Cranium::Application.new ["--cranium-load", "load", "--customer_name", "my_customer"]
+      expect(app.load_arguments).to eq customer_name: "my_customer"
+    end
+  end
+
+  describe "#cranium_arguments" do
+    it "should return the provided load arguments" do
+      app = Cranium::Application.new ["--cranium-load", "loads/load_file", "--customer_name", "my_customer"]
+      expect(app.cranium_arguments).to eq load: "loads/load_file"
+    end
+  end
+
+
+  describe "Application" do
+    it "should include metrics logging capabilities" do
+      expect(application.respond_to?(:log)).to be_truthy
+    end
+  end
+
+
+  describe "#sources" do
+    it "should return a SourceRegistry" do
+      expect(application.sources).to be_a Cranium::SourceRegistry
+    end
+  end
+
+
+  describe "#register_source" do
+    it "should register a source in the source registry and resolve its files" do
+      source = double "SourceDefinition"
+
+      expect(application.sources).to receive(:register_source).with(:source1).and_return(source)
+      expect(source).to receive(:resolve_files)
+
+      application.register_source(:source1) { file "test*.csv" }
+    end
+  end
+
+
+  describe "#run" do
+    before(:each) do
+      @original_stderr = $stderr
+      $stderr = StringIO.new
+    end
+
+    after(:each) do
+      $stderr = @original_stderr
+    end
+
+    context "when no files are specified as an argument" do
+      it "should exit with an error" do
+        expect { application.run }.to raise_error(SystemExit) { |exit| expect(exit.status).to eq(1) }
+      end
+
+      it "should log an error to STDOUT" do
+        expect { application.run }.to raise_error(SystemExit)
+
+        expect($stderr.string.chomp).to eq "ERROR: No file specified"
+      end
+    end
+
+
+    context "when a non-existent file is specified as an argument" do
+      let(:application) { Cranium::Application.new ["--cranium-load", "no-such-file.exists"] }
+
+      it "should exit with an error" do
+        expect { application.run }.to raise_error(SystemExit) { |exit| expect(exit.status).to eq(1) }
+      end
+
+      it "should log an error to STDOUT" do
+        expect { application.run }.to raise_error(SystemExit)
+
+        expect($stderr.string.chomp).to eq "ERROR: File 'no-such-file.exists' does not exist"
+      end
+    end
+
+
+    context "when called with an existing file" do
+      let(:file) { "products.rb" }
+
+      let(:application) { Cranium::Application.new ["--cranium-load", file] }
+
+      before(:each) do
+        allow(File).to receive(:exists?).with(file).and_return(true)
+      end
+
+
+      it "should load the first file specified as a command line parameter" do
+        expect(application).to receive(:load).with(file)
+
+        application.run
+      end
+
+
+      it "should run any registered after hooks" do
+        allow(application).to receive :load
+
+        hook_ran = false
+        application.register_hook :after do
+          hook_ran = true
+        end
+
+        application.run
+
+        expect(hook_ran).to be_truthy
+      end
+
+
+      context "when the execution of the process raises an error" do
+        let(:error) { StandardError.new }
+        before(:each) { allow(application).to receive(:load).and_raise error }
+
+        it "should propagate the error" do
+          expect { application.run }.to raise_error
+        end
+
+        it "should log an error" do
+          expect(application).to receive(:log).with(:error, error)
+
+          expect { application.run }.to raise_error
+        end
+
+        it "should still run any registered after hooks" do
+          hook_ran = false
+          application.register_hook :after do
+            hook_ran = true
+          end
+
+          begin
+            application.run
+          rescue
+          end
+
+          expect(hook_ran).to be_truthy
+        end
+      end
+    end
+  end
+
+
+  describe "#after_import" do
+
+    it "should register the given block" do
+      block_called = false
+
+      application.after_import do
+        block_called = true
+      end
+
+      application.apply_hook(:after_import)
+
+      expect(block_called).to be_truthy
+    end
+
+  end
+
+end

data/spec/cranium/archiver_spec.rb

@@ -0,0 +1,44 @@
+require_relative '../spec_helper'
+
+describe Cranium::Archiver do
+
+  before(:each) do
+    allow(Cranium).to receive_messages(configuration: Cranium::Configuration.new.tap do |config|
+      config.gpfdist_home_directory = "gpfdist_home"
+      config.upload_directory = "upload_dir"
+      config.archive_directory = "path/to/archive"
+    end)
+  end
+
+
+  describe ".archive" do
+    it "should create the archive directory if it doesn't exist" do
+      allow(Dir).to receive(:exists?).with("path/to/archive").and_return(false)
+
+      expect(FileUtils).to receive(:mkpath).with "path/to/archive"
+
+      Cranium::Archiver.archive
+    end
+
+    it "should move files to the archive directory" do
+      allow(Dir).to receive(:exists?).with("path/to/archive").and_return(true)
+      allow(Time).to receive(:now).and_return Time.new(2000, 1, 1, 1, 2, 3)
+
+      expect(FileUtils).to receive(:mv).with "gpfdist_home/upload_dir/file.txt", "path/to/archive/2000-01-01_01h02m03s_file.txt"
+      expect(FileUtils).to receive(:mv).with "gpfdist_home/upload_dir/another_file.txt", "path/to/archive/2000-01-01_01h02m03s_another_file.txt"
+
+      Cranium::Archiver.archive "file.txt", "another_file.txt"
+    end
+  end
+
+
+  describe ".remove" do
+    it "should remove files from the upload directory" do
+      expect(FileUtils).to receive(:rm).with "gpfdist_home/upload_dir/file.txt"
+      expect(FileUtils).to receive(:rm).with "gpfdist_home/upload_dir/another_file.txt"
+
+      Cranium::Archiver.remove "file.txt", "another_file.txt"
+    end
+  end
+
+end

data/spec/cranium/command_line_options_spec.rb

@@ -0,0 +1,32 @@
+require_relative '../spec_helper'
+
+describe Cranium::CommandLineOptions do
+
+  let(:argv) {
+    %w[
+      --cranium-initializer my_initializer
+      --cranium-load my_load
+      --some-param some_value
+      --another-param another_value
+    ]
+  }
+
+
+  subject { Cranium::CommandLineOptions.new argv }
+
+  describe "#cranium_arguments" do
+
+    it "should return only arguments used by Cranium" do
+      expect(subject.cranium_arguments).to eq(initializer: "my_initializer", load: "my_load")
+    end
+
+  end
+
+
+  describe "#load_arguments" do
+    it "should return non-cranium arguments" do
+      expect(subject.load_arguments).to eq(:"some-param" => "some_value", :"another-param" => "another_value")
+    end
+  end
+
+end

data/spec/cranium/configuration_spec.rb

@@ -0,0 +1,31 @@
+require_relative '../spec_helper'
+
+describe Cranium::Configuration do
+
+  let(:config) { Cranium::Configuration.new }
+
+  describe "#upload_path" do
+    it "should return the full upload path" do
+      config.gpfdist_home_directory = "/gpfdist/home/dir"
+      config.upload_directory = "uploads/customer"
+
+      expect(config.upload_path).to eq "/gpfdist/home/dir/uploads/customer"
+    end
+  end
+
+
+  describe "#storage_directory" do
+    it "should return the previously set value" do
+      config.storage_directory = "/some/path"
+      expect(config.storage_directory).to eq "/some/path"
+    end
+
+    it "should return the default storage directory if one wasn't explicitly set" do
+      config.gpfdist_home_directory = "/gpfdist/home/dir"
+      config.upload_directory = "uploads/customer"
+
+      expect(config.storage_directory).to eq "/gpfdist/home/dir/uploads/customer/.cranium"
+    end
+  end
+
+end

data/spec/cranium/data_importer_spec.rb

@@ -0,0 +1,55 @@
+require_relative '../spec_helper'
+
+describe Cranium::DataImporter do
+
+  before do
+    connection = double
+    allow(Cranium::Database).to receive(:connection).and_return connection
+    allow(connection).to receive(:transaction).and_yield
+  end
+
+  let(:importer) { Cranium::DataImporter.new }
+  let(:definition) { Cranium::DSL::ImportDefinition.new "definition_name" }
+
+  describe "#import" do
+
+    context "when called with both merge and delete_insert fields set" do
+      it "should raise an exception" do
+        definition.delete_insert_on :some_field
+        definition.merge_on :another_field
+
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+      end
+    end
+
+    context "when called with both merge and truncate_insert fields set" do
+      it "should raise an exception" do
+        definition.truncate_insert true
+        definition.merge_on :another_field
+
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+      end
+    end
+
+    context "when called with both delete_insert and truncate_insert fields set" do
+      it "should raise an exception" do
+        definition.delete_insert_on :some_field
+        definition.truncate_insert true
+
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+      end
+    end
+
+    context "when called with both merge, delete_insert and truncate_insert fields set" do
+      it "should raise an exception" do
+        definition.delete_insert_on :some_field
+        definition.merge_on :another_field
+        definition.truncate_insert true
+
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+      end
+    end
+
+  end
+
+end

data/spec/cranium/data_transformer_spec.rb

@@ -0,0 +1,16 @@
+require_relative '../spec_helper'
+
+describe Cranium::DataTransformer do
+
+  describe "#transform" do
+    it "should raise an error if the target definition's file name has been overriden" do
+      source = Cranium::DSL::SourceDefinition.new :source
+      target = Cranium::DSL::SourceDefinition.new :target
+
+      target.file "overriden filename"
+
+      expect { Cranium::DataTransformer.new(source, target).transform }.to raise_error StandardError, "Source definition 'target' cannot overrride the file name because it is a transformation target"
+    end
+  end
+
+end