cranium 0.2.0

data/spec/cranium/database_spec.rb

@@ -0,0 +1,69 @@
+require_relative '../spec_helper'
+require 'sequel'
+require 'sequel/adapters/mock'
+
+describe Cranium::Database do
+
+  let(:database) { Cranium::Database }
+  let(:connection) { Sequel::Mock::Database.new }
+  let(:other_connection) { Sequel::Mock::Database.new }
+
+  before(:each) do
+    allow(Cranium).to receive(:configuration).and_return(Cranium::Configuration.new.tap do |config|
+                                                           config.greenplum_connection_string = "connection string"
+                                                           config.loggers = "loggers"
+                                                         end)
+  end
+
+
+  describe ".connection" do
+    before { Cranium::Database.instance_variable_set :@connection, nil }
+
+    it "should connect to the DB" do
+      expect(Sequel).to receive(:connect).with("connection string", :loggers => "loggers").and_return connection
+
+      expect(database.connection).to eq connection
+    end
+
+    it "should return the same object every time" do
+      allow(Sequel).to receive(:connect).and_return(connection, other_connection)
+
+      expect(database.connection).to eq database.connection
+    end
+  end
+
+
+  describe ".[]" do
+    before do
+      database.instance_variable_set :@connections, nil
+      database.instance_variable_set :@definitions, nil
+
+      database.register_database :dwh do
+        connect_to "other connection string"
+      end
+    end
+
+    it "should return the specified database connection" do
+      expect(Sequel).to receive(:connect).with("other connection string", :loggers => "loggers").and_return connection
+
+      expect(database[:dwh]).to eq connection
+    end
+
+    it "should memoize the result of a previous call" do
+      allow(Sequel).to receive(:connect).and_return(connection, other_connection)
+
+      expect(database[:dwh]).to eq database[:dwh]
+    end
+
+    it "should memoize connections by name" do
+      database.register_database :dwh2 do
+        connect_to "other connection string 2"
+      end
+
+      allow(Sequel).to receive(:connect).and_return(connection, other_connection)
+
+      expect(database[:dwh]).not_to eq database[:dwh2]
+    end
+  end
+
+end

data/spec/cranium/definition_registry_spec.rb

@@ -0,0 +1,45 @@
+require_relative '../spec_helper'
+
+describe Cranium::DefinitionRegistry do
+
+  let(:registry) { Cranium::DefinitionRegistry.new Cranium::DSL::DatabaseDefinition }
+
+  describe "#[]" do
+    it "should return nil if a definition with the specified name wasn't registered yet" do
+      expect(registry[:name]).to be_nil
+    end
+  end
+
+
+  describe "#register_definition" do
+    it "should return the newly registered source" do
+      expect(registry.register_definition(:test_database) {}).to be_a Cranium::DSL::DatabaseDefinition
+    end
+
+    it "should register a new database definition and configure it through the block passed" do
+      expected_definition = Cranium::DSL::DatabaseDefinition.new :test_database
+      expected_definition.connect_to "connection string"
+
+      registry.register_definition :test_database do
+        connect_to "connection string"
+      end
+
+      expect(registry[:test_database]).to eq expected_definition
+    end
+
+
+    it "should register a new source definition and configure it through the block passed" do
+      registry = Cranium::DefinitionRegistry.new Cranium::DSL::SourceDefinition
+      expected_definition = Cranium::DSL::SourceDefinition.new :test_source
+      expected_definition.file "test.csv"
+
+      registry.register_definition :test_source do
+        file "test.csv"
+      end
+
+      expect(registry[:test_source]).to eq expected_definition
+    end
+
+  end
+
+end

data/spec/cranium/dimension_manager_spec.rb

@@ -0,0 +1,63 @@
+require_relative '../spec_helper'
+
+describe Cranium::DimensionManager do
+  describe "#insert" do
+
+    context "with single key" do
+      let(:manager) { Cranium::DimensionManager.new :table, :source_key }
+
+      it "should store a new record for insertion" do
+        manager.insert :target_key, target_key: 123, name: "John"
+
+        expect(manager.rows).to eq [{target_key: 123, name: "John"}]
+      end
+
+      it "should raise an error if the key field's value isn't specified in the record" do
+        expect { manager.insert :target_key, name: "John" }.to raise_error ArgumentError, "Required attribute 'target_key' missing"
+      end
+
+      it "should return the key field's value in an array" do
+        expect(manager.insert(:target_key, target_key: 123, name: "John")).to eq 123
+      end
+
+      context "if one of the values in the record is a sequence" do
+        it "should insert the next value of the specified sequence" do
+          sequence = Cranium::Transformation::Sequence.new :id_seq
+          allow(sequence).to receive(:next_value).and_return(123)
+
+          manager.insert :target_key, target_key: sequence
+
+          expect(manager.rows).to eq [{target_key: 123}]
+        end
+      end
+    end
+  end
+
+  describe "#create_cache_for_field" do
+    context "with single key" do
+      let(:manager) { Cranium::DimensionManager.new :table, [:key] }
+
+      it "should load key-value pairs into a hash" do
+        database = double("Database connection")
+        allow(manager).to receive(:db).and_return database
+
+        expect(database).to receive(:select_map).with([:key, :value_field]).and_return([[1, 10], [2, 20]])
+
+        expect(manager.create_cache_for_field(:value_field)).to eq [1] => 10, [2] => 20
+      end
+    end
+
+    context "with multiple keys" do
+      let(:manager) { Cranium::DimensionManager.new :table, [:key_1, :key_2] }
+
+      it "should load key-value pairs into a hash" do
+        database = double("Database connection")
+        allow(manager).to receive(:db).and_return database
+
+        expect(database).to receive(:select_map).with([:key_1, :key_2, :value_field]).and_return([[1, 1, 11], [1, 2, 12], [2, 1, 21]])
+
+        expect(manager.create_cache_for_field(:value_field)).to eq [1, 1] => 11, [1, 2] => 12, [2, 1] => 21
+      end
+    end
+  end
+end

data/spec/cranium/dsl/database_definition_spec.rb

@@ -0,0 +1,23 @@
+require_relative '../../spec_helper'
+
+describe Cranium::DSL::DatabaseDefinition do
+
+  let(:database) { Cranium::DSL::DatabaseDefinition.new "name" }
+
+
+  describe "#name" do
+    it "should return the name of the database definition" do
+      expect(database.name).to eq("name")
+    end
+  end
+
+
+  describe "#connect_to" do
+    it "should set the attribute to the specified value" do
+      database.connect_to "value"
+
+      expect(database.connect_to).to eq("value")
+    end
+  end
+
+end

data/spec/cranium/dsl/extract_definition_spec.rb

@@ -0,0 +1,76 @@
+require_relative '../../spec_helper'
+
+describe Cranium::DSL::ExtractDefinition do
+
+  let(:extract) { Cranium::DSL::ExtractDefinition.new :extract_name }
+
+
+  describe "#name" do
+    it "should return the name of the extract definition" do
+      expect(extract.name).to eq(:extract_name)
+    end
+  end
+
+
+  describe "#storage" do
+    it "should return the persistent storage corresponding to the extract" do
+      expect(extract.storage).to be_a Cranium::Extract::Storage
+    end
+  end
+
+
+  describe "#from" do
+    it "should set the attribute to the specified value" do
+      extract.from :database
+      expect(extract.from).to eq(:database)
+    end
+  end
+
+
+  describe "#query" do
+    it "should set the attribute to the specified value" do
+      extract.query "extract query"
+      expect(extract.query).to eq("extract query")
+    end
+  end
+
+
+  describe "#columns" do
+    it "should set the attribute to the specified value" do
+      extract.query %w(id name status)
+      expect(extract.query).to eq(%w(id name status))
+    end
+  end
+
+
+  describe "#incrementally_by" do
+    it "should set the attribute to the specified value" do
+      extract.incrementally_by :id
+      expect(extract.incrementally_by).to eq(:id)
+    end
+  end
+
+
+  describe "#last_extracted_value_of" do
+    let(:storage) { double "extract storage" }
+    before { allow(Cranium::Extract::Storage).to receive(:new).with(:extract_name).and_return(storage) }
+
+    context "when there is no last extracted value for the field" do
+      before { allow(storage).to receive(:last_value_of).with(:id).and_return(nil) }
+
+      it "should return nil" do
+        expect(extract.last_extracted_value_of(:id)).to be_nil
+      end
+
+      it "should return the default value if one was specified" do
+        expect(extract.last_extracted_value_of(:id, 0)).to eq(0)
+      end
+    end
+
+    it "should return the last extracted value of the field" do
+      allow(storage).to receive(:last_value_of).with(:id).and_return(15)
+      expect(extract.last_extracted_value_of(:id)).to eq(15)
+    end
+  end
+
+end

data/spec/cranium/dsl/import_definition_spec.rb

@@ -0,0 +1,153 @@
+require_relative '../../spec_helper'
+
+describe Cranium::DSL::ImportDefinition do
+
+  let(:import) { Cranium::DSL::ImportDefinition.new "import_name" }
+
+  describe "#into" do
+    it "should set the attribute to the specified value" do
+      import.into "new value"
+
+      expect(import.into).to eq("new value")
+    end
+  end
+
+
+  describe "#name" do
+    it "should return the name of the import definition" do
+      expect(import.name).to eq("import_name")
+    end
+  end
+
+
+  describe "#field_associations" do
+    context "when no fields are set" do
+      it "should return empty hash" do
+        expect(import.field_associations).to eq({})
+      end
+    end
+  end
+
+
+  describe "#put" do
+    context "when called with a Hash" do
+      it "should store the field associations" do
+        import.put :item => :id, :title => :name
+
+        expect(import.field_associations).to eq({:item => :id, :title => :name})
+      end
+    end
+
+    context "when called with a Symbol" do
+      it "should store a field association between fields with the same name" do
+        import.put :item
+
+        expect(import.field_associations).to eq({:item => :item})
+      end
+    end
+
+    context "when called with an unsupported type" do
+      it "should raise an error" do
+        expect { import.put "unsupported" }.to raise_error ArgumentError, "Unsupported argument for Import::put"
+      end
+    end
+
+    context "when called multiple times" do
+      it "should merge all specified associations" do
+        import.put :item => :id
+        import.put :title => :name
+        import.put :category => :category, :brand => :brand
+
+        expect(import.field_associations).to eq({
+            :item => :id,
+            :title => :name,
+            :category => :category,
+            :brand => :brand
+        })
+      end
+    end
+  end
+
+
+  describe "#merge_fields" do
+    context "when no fields are set" do
+      it "should return empty hash" do
+        expect(import.merge_fields).to eq({})
+      end
+    end
+  end
+
+
+  describe "#merge_on" do
+    context "when called with a Hash" do
+      it "should set merge field associations" do
+        import.merge_on :item => :id, :title => :name
+
+        expect(import.merge_fields).to eq({:item => :id, :title => :name})
+      end
+    end
+
+    context "when called with a Symbol" do
+      it "should store a merge field association between fields with the same name" do
+        import.merge_on :item
+
+        expect(import.merge_fields).to eq({:item => :item})
+      end
+    end
+
+    context "when called with an unsupported type" do
+      it "should raise an error" do
+        expect { import.merge_on "unsupported" }.to raise_error ArgumentError, "Unsupported argument for Import::merge_on"
+      end
+    end
+
+    context "when called multiple times" do
+      it "should overwrite existing merge fields" do
+        import.merge_on :item => :id
+        import.merge_on :title => :name
+
+        expect(import.merge_fields).to eq({:title => :name})
+      end
+    end
+  end
+
+
+  describe "#delete_insert_on" do
+
+    it "should return an empty array when no value was set" do
+      expect(import.delete_insert_on).to eq([])
+    end
+
+
+    it "should store and return the value passed" do
+      import.delete_insert_on :some_field
+
+      expect(import.delete_insert_on).to eq([:some_field])
+    end
+
+
+    it "should handle multiple arguments" do
+      import.delete_insert_on :some_field, :another_field
+
+      expect(import.delete_insert_on).to eq([:some_field, :another_field])
+    end
+
+  end
+
+
+  describe "#truncate_insert" do
+
+    it "should return false when no value was set" do
+      expect(import.truncate_insert).to eq(false)
+    end
+
+
+    it "should store and return the value passed" do
+      import.truncate_insert true
+
+      expect(import.truncate_insert).to eq(true)
+    end
+
+  end
+
+end

data/spec/cranium/dsl/source_definition_spec.rb

@@ -0,0 +1,84 @@
+require_relative '../../spec_helper'
+require 'ostruct'
+
+describe Cranium::DSL::SourceDefinition do
+
+  let(:source) { Cranium::DSL::SourceDefinition.new "name" }
+
+
+  { file: "name.csv",
+    delimiter: ",",
+    escape: '"',
+    quote: '"',
+    encoding: "UTF-8" }.each do |attribute, default_value|
+
+    describe "#attribute" do
+      context "if called without a parameter" do
+        it "should return the (default) value of the attribute" do
+          expect(source.send(attribute)).to eq(default_value)
+        end
+      end
+
+      context "if called with a parameter" do
+        it "should set the attribute to the specified value" do
+          source.send(attribute, "new value")
+
+          expect(source.send(attribute)).to eq("new value")
+        end
+      end
+    end
+
+  end
+
+
+  describe "#fields" do
+    it "should return an empty Hash if no fields are set" do
+      expect(source.fields).to eq({})
+    end
+
+    it "should return the fields and types that were set" do
+      source.field :field1, String
+      source.field :field2, Fixnum
+
+      expect(source.fields).to eq({
+        field1: String,
+        field2: Fixnum
+      })
+    end
+  end
+
+
+  describe "#file_name_overriden?" do
+    it "should signal if the file name parameter of the source definition has been set to something other than the default" do
+      expect(source.file_name_overriden?).to be_falsey
+
+      source.file "overriden.csv"
+      expect(source.file_name_overriden?).to be_truthy
+    end
+  end
+
+
+  describe "#files" do
+    it "should return nil by default" do
+      expect(source.files).to be_nil
+    end
+  end
+
+
+  describe "#resolve_files" do
+    it "should store the file names of all files matching the file pattern" do
+      allow(Cranium).to receive_messages configuration: (Cranium::Configuration.new.tap do |config|
+        config.gpfdist_home_directory = "/home/gpfdist"
+        config.upload_directory = "customer"
+      end)
+      source.file "product*.csv"
+      allow(Dir).to receive(:[]).with("/home/gpfdist/customer/product*.csv").and_return(["/home/gpfdist/customer/product2.csv",
+                                                                            "/home/gpfdist/customer/product1.csv"])
+
+      source.resolve_files
+
+      expect(source.files).to eq(["product1.csv", "product2.csv"])
+    end
+  end
+
+end

data/spec/cranium/dsl_spec.rb

@@ -0,0 +1,119 @@
+require_relative '../spec_helper'
+
+describe Cranium::DSL do
+
+  let(:dsl_object) { Object.new.tap { |object| object.extend Cranium::DSL } }
+
+  describe "#database" do
+    it "should register a database connection in the application" do
+      block = lambda {}
+
+      expect(Cranium::Database).to receive(:register_database) do |arg, &blk|
+        expect(arg).to eq :name
+        expect(blk).to be block
+      end
+
+      dsl_object.database(:name, &block)
+    end
+  end
+
+
+  describe "#source" do
+    it "should register a source in the application" do
+      expect(Cranium.application).to receive(:register_source).with(:name)
+
+      dsl_object.source(:name)
+    end
+  end
+
+
+  describe "#extract" do
+    it "should create an extract definition and execute it" do
+      extract_definition = double "ExtractDefinition"
+      block = lambda {}
+
+      allow(Cranium::DSL::ExtractDefinition).to receive(:new).with(:contacts).and_return(extract_definition)
+      expect(extract_definition).to receive(:instance_eval) { |&blk| expect(blk).to be block }
+
+      extractor = double "DataExtractor"
+      allow(Cranium::Extract::DataExtractor).to receive_messages new: extractor
+      expect(extractor).to receive(:execute).with(extract_definition)
+
+      dsl_object.extract :contacts, &block
+    end
+  end
+
+
+  describe "#deduplicate" do
+    it "should call transform with correct source and target arguments" do
+      expect(dsl_object).to receive(:transform).with(:sales_items => :products)
+      dsl_object.deduplicate :sales_items, into: :products, by: [:item]
+    end
+  end
+
+
+  describe "#import" do
+    it "should create an import definition and execute it" do
+      import_definition = double "ImportDefinition"
+      block = lambda {}
+
+      allow(Cranium::DSL::ImportDefinition).to receive(:new).with(:contacts).and_return(import_definition)
+      expect(import_definition).to receive(:instance_eval) { |&blk| expect(blk).to be block }
+
+      importer = double "DataImporter"
+      allow(Cranium::DataImporter).to receive_messages new: importer
+      expect(importer).to receive(:import).with(import_definition)
+
+      dsl_object.import :contacts, &block
+    end
+  end
+
+
+  describe "#archive" do
+    it "should archive files for the specified sources" do
+      allow(Cranium.application).to receive_messages sources: {first_source: double(files: ["file1", "file2"]),
+                                         second_source: double(files: ["file3"]),
+                                         third_source: double(files: ["file4"])}
+
+      expect(Cranium::Archiver).to receive(:archive).with "file1", "file2"
+      expect(Cranium::Archiver).to receive(:archive).with "file3"
+
+      dsl_object.archive :first_source, :second_source
+    end
+  end
+
+
+  describe "#remove" do
+    it "should remove files for the specified sources" do
+      allow(Cranium.application).to receive_messages sources: {first_source: double(files: ["file1", "file2"]),
+                                         second_source: double(files: ["file3"]),
+                                         third_source: double(files: ["file4"])}
+
+      expect(Cranium::Archiver).to receive(:remove).with "file1", "file2"
+      expect(Cranium::Archiver).to receive(:remove).with "file3"
+
+      dsl_object.remove :first_source, :second_source
+    end
+  end
+
+
+  describe "#sequence" do
+    it "should return a sequence with the specified name" do
+      result = dsl_object.sequence "test_sequence"
+
+      expect(result).to be_a Cranium::Transformation::Sequence
+      expect(result.name).to eq("test_sequence")
+    end
+  end
+
+
+  describe "#after" do
+    it "should register a new after hook for the application" do
+      block = -> {}
+
+      expect(Cranium.application).to receive(:register_hook).with(:after) { |&blk| expect(blk).to be block }
+
+      dsl_object.after &block
+    end
+  end
+end