cranium 0.2.0

data/spec/cranium/external_table_spec.rb

@@ -0,0 +1,71 @@
+require_relative '../spec_helper'
+require 'ostruct'
+
+describe Cranium::ExternalTable do
+
+  let(:connection) { double "Greenplum connection" }
+  let(:source) do
+    Cranium::DSL::SourceDefinition.new(:products).tap do |source|
+      source.file "test_products.csv"
+      source.field :text_field, String
+      source.field :integer_field, Integer
+      source.field :numeric_field, Float
+      source.field :date_field, Date
+      source.field :timestamp_field, Time
+      source.field :boolean_field1, TrueClass
+      source.field :boolean_field2, FalseClass
+      source.delimiter ';'
+      source.quote '"'
+      source.escape "'"
+    end
+  end
+  let(:external_table) { Cranium::ExternalTable.new source, connection }
+
+
+  describe "#create" do
+    it "should create an external table from the specified source" do
+      allow(Cranium).to receive_messages configuration: OpenStruct.new(
+        gpfdist_url: "gpfdist-url",
+        gpfdist_home_directory: "/gpfdist-home",
+        upload_directory: "upload-dir"
+      )
+
+      allow(source).to receive_messages files: %w(test_products_a.csv test_products_b.csv)
+
+      expect(connection).to receive(:run).with(<<-sql
+      CREATE EXTERNAL TABLE "external_products" (
+          "text_field" TEXT,
+          "integer_field" INTEGER,
+          "numeric_field" NUMERIC,
+          "date_field" DATE,
+          "timestamp_field" TIMESTAMP WITHOUT TIME ZONE,
+          "boolean_field1" BOOLEAN,
+          "boolean_field2" BOOLEAN
+      )
+      LOCATION ('gpfdist://gpfdist-url/upload-dir/test_products_a.csv', 'gpfdist://gpfdist-url/upload-dir/test_products_b.csv')
+      FORMAT 'CSV' (DELIMITER ';' ESCAPE '''' QUOTE '"' HEADER)
+      ENCODING 'UTF8'
+      sql
+      )
+
+      external_table.create
+    end
+  end
+
+
+  describe "#destroy" do
+    it "should drop the external table" do
+      expect(connection).to receive(:run).with(%Q[DROP EXTERNAL TABLE "external_products"])
+
+      external_table.destroy
+    end
+  end
+
+
+  describe "#name" do
+    it "should return the name of the external table based on the source's name" do
+      expect(external_table.name).to eq(:external_products)
+    end
+  end
+
+end

data/spec/cranium/extract/storage_spec.rb

@@ -0,0 +1,125 @@
+require_relative '../../spec_helper'
+
+describe Cranium::Extract::Storage do
+
+  let(:storage) { Cranium::Extract::Storage.new :extract_name }
+  let(:storage_dir) { "/storage/directory/.cranium" }
+  let(:storage_file) { "#{storage_dir}/extracts" }
+
+  before do
+    allow(Cranium).to receive(:configuration).and_return(Cranium::Configuration.new.tap { |config| config.storage_directory = storage_dir })
+  end
+
+  describe "#last_value_of" do
+    context "when storage file doesn't exist" do
+      it "should return nil if no storage file was created yet" do
+        allow(File).to receive(:exists?).with(storage_file).and_return(false)
+        expect(storage.last_value_of(:field)).to eq(nil)
+      end
+    end
+
+    context "when storage file already exists" do
+      before { allow(File).to receive(:exists?).with(storage_file).and_return(true) }
+
+      it "should return nil if no value was saved for this extract yet" do
+        allow(File).to receive(:read).with(storage_file).and_return(YAML.dump(other_extract_name: {last_values: {}}))
+        expect(storage.last_value_of(:field)).to eq(nil)
+      end
+
+      it "should return nil if no value was saved for the field" do
+        allow(File).to receive(:read).with(storage_file).and_return(YAML.dump(extract_name: {last_values: {}}))
+        expect(storage.last_value_of(:field)).to eq(nil)
+      end
+
+      it "should return the last saved value of the specified field" do
+        allow(File).to receive(:read).with(storage_file).and_return(YAML.dump(extract_name: {last_values: {field: 15}}))
+        expect(storage.last_value_of(:field)).to eq(15)
+      end
+    end
+  end
+
+
+  describe "#save_last_value_of" do
+    context "when storage file doesn't exist" do
+      before { allow(File).to receive(:exists?).with(storage_file).and_return(false) }
+
+      it "should create the storage file and save the specified value if the storage directory already exists" do
+        allow(Dir).to receive(:exists?).with(storage_dir).and_return(true)
+
+        expect(File).to receive(:write).with(storage_file, YAML.dump(extract_name: {last_values: {field: 15}}))
+
+        storage.save_last_value_of(:field, 15)
+      end
+
+      it "should create the storage directory if it doesn't exist yet" do
+        allow(Dir).to receive(:exists?).with(storage_dir).and_return(false)
+        allow(File).to receive :write
+
+        expect(FileUtils).to receive(:mkdir_p).with(storage_dir)
+
+        storage.save_last_value_of(:field, 15)
+      end
+    end
+
+    context "when there are previously saved values" do
+      before do
+        allow(Dir).to receive(:exists?).with(storage_dir).and_return(true)
+        allow(File).to receive(:exists?).with(storage_file).and_return(true)
+      end
+
+      it "should overwrite the specified field's value and preserve all others" do
+        allow(File).to receive(:read).with(storage_file).and_return(YAML.dump({
+                                                                                extract_name: {
+                                                                                  last_values: {
+                                                                                    field1: 1,
+                                                                                    field2: 2,
+                                                                                    field3: 3
+                                                                                  }
+                                                                                }
+                                                                              }))
+
+        expect(File).to receive(:write).with(storage_file, YAML.dump({
+                                                                       extract_name: {
+                                                                         last_values: {
+                                                                           field1: 1,
+                                                                           field2: 5,
+                                                                           field3: 3
+                                                                         }
+                                                                       }
+                                                                     }))
+
+        storage.save_last_value_of(:field2, 5)
+      end
+
+      it "should create the new entry if it doesn't exist yet" do
+        allow(File).to receive(:read).with(storage_file).and_return(YAML.dump({
+                                                                                other_extract_name: {
+                                                                                  last_values: {
+                                                                                    field1: 1,
+                                                                                    field2: 2,
+                                                                                    field3: 3
+                                                                                  }
+                                                                                }
+                                                                              }))
+
+        expect(File).to receive(:write).with(storage_file, YAML.dump({
+                                                                       other_extract_name: {
+                                                                         last_values: {
+                                                                           field1: 1,
+                                                                           field2: 2,
+                                                                           field3: 3
+                                                                         }
+                                                                       },
+                                                                       extract_name: {
+                                                                         last_values: {
+                                                                           field2: 5
+                                                                         }
+                                                                       }
+                                                                     }))
+
+        storage.save_last_value_of(:field2, 5)
+      end
+    end
+  end
+
+end

data/spec/cranium/logging_spec.rb

@@ -0,0 +1,37 @@
+require_relative '../spec_helper'
+
+describe Cranium::Logging do
+
+  let(:logging_object) { Object.new.tap { |object| object.extend Cranium::Logging } }
+  let(:loggers) { [double("Logger 1"), double("Logger 2")] }
+
+  before(:each) do
+    allow(Cranium).to receive_message_chain(:configuration, :loggers).and_return loggers
+  end
+
+
+
+  def all_loggers_should_receive(level, message)
+    loggers.each { |logger| expect(logger).to receive(level).with(message) }
+  end
+
+
+
+  describe "#record_metric" do
+    it "should record an arbitrary metric in every registered logger" do
+      all_loggers_should_receive :info, "[metrics/products] 1234"
+
+      logging_object.record_metric "products", 1234
+    end
+  end
+
+
+  describe "#log" do
+    it "should log a message with the specified reporting level in every registered logger" do
+      all_loggers_should_receive :error, "error message"
+
+      logging_object.log :error, "error message"
+    end
+  end
+
+end

data/spec/cranium/sequel/hash_spec.rb

@@ -0,0 +1,56 @@
+require_relative '../../spec_helper'
+
+describe Cranium::Sequel::Hash do
+
+  let(:source_hash) { { :field1 => :field2, :field3 => :field4 } }
+  let(:sequel_hash) { Cranium::Sequel::Hash[source_hash] }
+
+  before(:each) do
+    allow(Sequel).to receive(:qualify) { |qualifier, field| :"#{qualifier}_#{field}" }
+  end
+
+
+  it "should be a Hash" do
+    expect(Cranium::Sequel::Hash.new).to be_a Hash
+  end
+
+
+  describe "#qualify" do
+    context "when called with 'keys_with'" do
+      it "should qualify only the key fields of the hash" do
+        expect(sequel_hash.qualify(keys_with: :table1)).to eq({ :table1_field1 => :field2, :table1_field3 => :field4 })
+      end
+    end
+
+    context "when called with 'values_with'" do
+      it "should qualify only the value fields of the hash" do
+        expect(sequel_hash.qualify(values_with: :table1)).to eq({ :field1 => :table1_field2, :field3 => :table1_field4 })
+      end
+    end
+
+    context "when called with both 'keys_with' and 'values_with'" do
+      it "should qualify both keys and value fields of the hash" do
+        expect(sequel_hash.qualify(keys_with: :table1, values_with: :table2)).to eq({ :table1_field1 => :table2_field2, :table1_field3 => :table2_field4 })
+      end
+    end
+
+    it "should raise an error if called with unsupported options" do
+      expect { sequel_hash.qualify key_with: :table }.to raise_error ArgumentError, "Unsupported option for qualify: key_with"
+    end
+  end
+
+
+  describe "#qualified_keys" do
+    it "should return an array with the hash's keys qualified with the specified qualifier" do
+      expect(sequel_hash.qualified_keys(:table)).to eq([:table_field1, :table_field3])
+    end
+  end
+
+
+  describe "#qualified_values" do
+    it "should return an array with the hash's values qualified with the specified qualifier" do
+      expect(sequel_hash.qualified_values(:table)).to eq([:table_field2, :table_field4])
+    end
+  end
+
+end

data/spec/cranium/source_registry_spec.rb

@@ -0,0 +1,31 @@
+require_relative '../spec_helper'
+
+describe Cranium::SourceRegistry do
+
+  let(:registry) { Cranium::SourceRegistry.new }
+
+  describe "#[]" do
+    it "should raise an error if a source with the specified name wasn't registered yet" do
+      expect { registry[:name] }.to raise_error "Undefined source 'name'"
+    end
+  end
+
+
+  describe "#register_source" do
+    it "should register a new source and configure it through the block passed" do
+      source = Cranium::DSL::SourceDefinition.new :test_source
+      source.field :test_field, String
+
+      registry.register_source :test_source do
+        field :test_field, String
+      end
+
+      expect(registry[:test_source]).to eq(source)
+    end
+
+    it "should return the newly registered source" do
+      expect(registry.register_source(:test_source) {}).to be_a Cranium::DSL::SourceDefinition
+    end
+  end
+
+end

data/spec/cranium/test_framework/cucumber_table_spec.rb

@@ -0,0 +1,144 @@
+require_relative "../../spec_helper"
+require 'cucumber/ast/table'
+require 'date'
+
+module Cranium::TestFramework
+  describe CucumberTable do
+
+    context "class method" do
+      describe ".from_cucumber_table" do
+
+        let(:table) { CucumberTable.from_ast_table(Cucumber::Ast::Table.new(@table_data)) }
+
+        it "should return a CucumberTable" do
+          @table_data = [{ "column" => "value" }]
+
+          expect(table).to be_a CucumberTable
+        end
+
+
+        it "should convert header values to symbols" do
+          @table_data = [{ "column1" => "value1", "column2" => "value2" }]
+
+          expect(table.fields).to eq([:column1, :column2])
+        end
+
+
+        it "should discard comment columns" do
+          @table_data = [{ "column" => "value1", "#comment column" => "value2" }]
+          expect(CucumberTable).to receive(:new).with([{ column: "value1" }], { column: :string })
+
+          table
+        end
+
+
+        it "should discard type specifiers in column names" do
+          @table_data = [{
+                           "integer_column (i)" => "one",
+                           "string_column (s)" => "two",
+                           "numeric_column (n)" => "five",
+                           "some_column" => "else"
+                         }]
+
+          expect(table.fields).to match_array([:integer_column, :string_column, :numeric_column, :some_column])
+        end
+
+
+        it "should raise an exception if invalid type is specified" do
+          @table_data = [{ "column (x)" => "value" }]
+
+          expect { table.fields }.to raise_error StandardError, "Invalid type specified: x"
+        end
+
+
+        it "should instantiate the new table with the correct column types" do
+          @table_data = [{
+                           "integer_column (i)" => "one",
+                           "string_column (s)" => "two",
+                           "numeric_column (n)" => "five",
+                           "some_column" => "else"
+                         }]
+          expect(CucumberTable).to receive(:new).with(
+            [{
+               integer_column: "one",
+               string_column: "two",
+               numeric_column: "five",
+               some_column: "else"
+             }],
+            {
+              integer_column: :integer,
+              string_column: :string,
+              numeric_column: :numeric,
+              some_column: :string
+            }
+          )
+
+          table
+        end
+      end
+    end
+
+
+    context "instance methods" do
+      let(:data) { [{ "one" => "two", "three" => "four" }, { "five" => "six" }] }
+
+      describe "#fields" do
+        it "should return the keys of the first row" do
+          expect(CucumberTable.new(data).fields).to eq(%w[one three])
+        end
+      end
+
+
+      describe "#with_patterns" do
+        it "should set replacement patterns and return the object" do
+          table = CucumberTable.new(data)
+          table_with_patterns = table.with_patterns({ "a" => "b" })
+
+          expect(table_with_patterns).to be_equal table
+        end
+      end
+
+
+      describe "#data" do
+        it "should return all data as an array of hashes" do
+          expect(CucumberTable.new(data).data).to eq(data)
+        end
+
+
+        it "should make all substitutions set up as replacement patterns" do
+          table = CucumberTable.new [{ first: "NULL", second: "apple", third: "something else entirely" }]
+          table.with_patterns(
+            "NULL" => nil,
+            "apple" => lambda { "pear" }
+          )
+
+          expect(table.data).to eq([first: nil, second: "pear", third: "something else entirely"])
+        end
+
+
+        it "should evaluate integer fields" do
+          table = CucumberTable.new([{ integer_column: "20" }], { integer_column: :integer })
+          expect(table.data).to eq([{ integer_column: 20 }])
+        end
+
+
+        describe "#columns" do
+          it "should return an array of empty arrays if there are no data rows" do
+            table = CucumberTable.new [], { argument: :string }
+
+            expect(table.data.columns).to eq([[]])
+          end
+
+
+          it "should return the data in columns as an array of arrays, discarding all header information" do
+            table = CucumberTable.new [{ header1: "value1", header2: "value2" }, { header1: "value3", header2: "value4" }]
+
+            expect(table.data.columns).to eq([%w[value1 value3], %w[value2 value4]])
+          end
+        end
+      end
+
+    end
+
+  end
+end

data/spec/cranium/transformation/duplication_index_spec.rb

@@ -0,0 +1,75 @@
+require_relative '../../spec_helper'
+
+describe Cranium::Transformation::DuplicationIndex do
+
+  let(:index) { Cranium::Transformation::DuplicationIndex.new :field1, :field2 }
+  let(:record) { Cranium::TransformationRecord.new [:field1, :field2, :field3], [:field1, :field2, :field3] }
+
+  describe ".[]" do
+    before(:each) { Cranium::Transformation::DuplicationIndex.instance_variable_set :@instances, nil }
+
+    it "should return a DuplicationIndex instance for the specified fields" do
+      allow(Cranium::Transformation::DuplicationIndex).to receive(:new).with(:field1, :field2).and_return(index)
+
+      expect(Cranium::Transformation::DuplicationIndex[:field1, :field2]).to eq index
+    end
+
+    it "should memoize the previously created instances" do
+      expect(Cranium::Transformation::DuplicationIndex[:field1, :field2]).to eq(Cranium::Transformation::DuplicationIndex[:field1, :field2])
+    end
+
+    it "should raise an error if empty fieldset was passed" do
+      expect { Cranium::Transformation::DuplicationIndex[] }.to raise_error ArgumentError, "Cannot build duplication index for empty fieldset"
+    end
+  end
+
+
+  describe "#duplicate?" do
+    it "should return false for the first entry" do
+      record.input_data = ["one", "two", "three"]
+      expect(index.duplicate?(record)).to be_falsey
+    end
+
+    it "should return true the second time it's called for the same record" do
+      record.input_data = ["one", "two", "three"]
+      index.duplicate?(record)
+      expect(index.duplicate?(record)).to be_truthy
+    end
+
+    it "should only use the specified fieldset for duplication detection" do
+      record1 = record
+      record2 = record.clone
+      index = Cranium::Transformation::DuplicationIndex.new :field1
+
+      record1.input_data = ["one", "two", "three"]
+      index.duplicate? record1
+
+      record2.input_data = ["one", "four", "five"]
+      expect(index.duplicate?(record2)).to be_truthy
+    end
+
+    it "should handle multiple fields for detection" do
+      record1 = record
+      record2 = record.clone
+      record3 = record.clone
+      index = Cranium::Transformation::DuplicationIndex.new :field1, :field2
+
+      record1.input_data = ["one", "two", "three"]
+      index.duplicate? record1
+
+      record2.input_data = ["one", "four", "five"]
+      expect(index.duplicate?(record2)).to be_falsey
+
+      record3.input_data = ["one", "two", "five"]
+      expect(index.duplicate?(record3)).to be_truthy
+    end
+
+    it "should raise an error if record fieldset doesn't contain index fieldset" do
+      record.input_data = ["one", "two", "three"]
+      index = Cranium::Transformation::DuplicationIndex.new :field5
+
+      expect { index.duplicate? record }.to raise_error StandardError, "Missing deduplication key from record: field5"
+    end
+  end
+
+end