datashift 0.16.0 → 0.40.0

data/spec/datashift/binder_spec.rb

@@ -0,0 +1,266 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Jan 2015
+# License::   MIT
+#
+# Details::   Specs for Binder aspect of Active Record Loader
+#             Binder provides the bridge between 'strings' e.g column headings
+#             and a classes different types of assignment operators/associations
+#
+require File.join(File.dirname(__FILE__), '/../spec_helper')
+
+module DataShift
+
+  describe Binder do
+    include_context 'ClearThenManageProject'
+
+    let(:binder) { Binder.new }
+
+    context 'errors binding headers' do
+      let(:headers) { [:value_as_string, :owner, :bad_no_such_column, :value_as_boolean, :value_as_double, :more_rubbish_as_nil] }
+
+      before(:each) do
+        @bindings = binder.map_inbound_headers( Project, headers )
+      end
+
+      let(:bindings) { @bindings }
+
+      it 'uses NoMethodBinding object methods when no such operator' do
+        expect(bindings.size).to eq 6
+
+        expect(bindings[2]).to be_a NoMethodBinding
+        expect(bindings[5]).to be_a NoMethodBinding
+
+        expect(bindings[0]).to be_a MethodBinding
+      end
+
+      it 'should populate missing_bindings  when no such operator' do
+        expect(binder.missing_bindings.size).to eq 2
+      end
+
+      it 'should indicate when bindings  missing' do
+        expect(binder.missing_bindings?).to eq true
+      end
+
+      it 'should indicate names of missing bindings' do
+        missing = binder.headers_missing_bindings
+
+        expect(missing.size).to eq 2
+
+        expect(missing[0]).to eq 'bad_no_such_column'
+        expect(missing[1]).to eq 'more_rubbish_as_nil'
+      end
+
+      it 'should indicate index of missing bindings ' do
+        missing = binder.indexes_missing_bindings
+
+        expect(missing[0]).to eq 2
+        expect(missing[1]).to eq 5
+      end
+
+      it 'uses NoMethodBinding object methods when operator has no such lookup field', fail: true do
+        # Owner  has_many :digitals which has field attachment_file_name
+        headers = ['digitals:attachment_file_name',
+                   'Digitals:nonsense_lookup_field',
+                   'Digitals:nonsense:with a value',
+                   'Digitals:nonsense:with a value:and random data']
+
+        binder.map_inbound_headers( Owner, headers)
+
+        expect(binder.bindings.size).to eq 4
+        expect(binder.missing_bindings.size).to eq 3
+
+        expect(binder.missing_bindings?).to eq true
+        expect(binder.missing_bindings[0].reason).to include 'Field [nonsense_lookup_field] Not Found'
+      end
+    end
+
+    let (:headers) { [:value_as_string, :owner, :value_as_boolean, :value_as_double] }
+
+    it 'should find a set of methods based on a list of column symbols' do
+      bindings = binder.map_inbound_headers( Project, headers )
+      expect(bindings.size).to eq 4
+    end
+
+    it 'should map a list of column names to a set of method details' do
+      headers = %w(value_as_double value_as_string bad_no_such_column value_as_boolean)
+
+      bindings = binder.map_inbound_headers( Project, headers )
+
+      expect(bindings.size).to eq 4
+
+      expect(bindings[2]).to be_a NoMethodBinding
+
+      expect(bindings[0]).to be_a MethodBinding
+      expect(bindings.last).to be_a MethodBinding
+    end
+
+    it 'should populate a method binding instance based on column and database info' do
+      headers = [:value_as_string, :owner, :value_as_boolean, :value_as_double]
+
+      bindings = binder.map_inbound_headers( Project, headers )
+
+      expect(bindings.size).to eq 4
+
+      expect(bindings[0]).to be_a MethodBinding
+
+      headers.each_with_index do |_c, i|
+        expect(bindings[i].inbound_index).to eq i
+        expect(bindings[i].inbound_column.index).to eq i
+      end
+    end
+
+    it 'should map between user name and domain model attribute' do
+      headers = ['Value as string', :value_as_string, 'value_as boolean', 'Value_As_Double']
+
+      operators = %w(value_as_string value_as_string value_as_boolean value_as_double)
+
+      bindings = binder.map_inbound_headers( Project, headers )
+
+      expect(bindings.size).to eq headers.size
+
+      headers.each_with_index do |c, i|
+        expect(bindings[i].valid?).to eq true
+        expect(bindings[i].inbound_index).to eq i
+        expect(bindings[i].inbound_name).to eq c.to_s
+        expect(bindings[i].operator).to eq operators[i]
+      end
+    end
+
+    it 'should map between user name and domain model has_one association' do
+      # Project has_one  :owner
+      headers = ['owner', 'Owner', :owner]
+
+      operators = %w(owner owner owner)
+
+      bindings = binder.map_inbound_headers( Project, headers )
+
+      expect(bindings.size).to eq headers.size
+
+      headers.each_with_index do |c, i|
+        expect(bindings[i].valid?).to eq true
+        expect(bindings[i].inbound_index).to eq i
+        expect(bindings[i].inbound_name).to eq c.to_s
+        expect(bindings[i].operator).to eq operators[i]
+      end
+    end
+
+    it 'should map between user name and domain model has_many association' do
+      # Project has_many :loader_releases
+      headers = ['loader_releases', 'Loader Releases', :loader_releases]
+
+      operators = %w(loader_releases loader_releases loader_releases)
+
+      bindings = binder.map_inbound_headers( Project, headers )
+
+      expect(bindings.size).to eq headers.size
+
+      headers.each_with_index do |c, i|
+        expect(bindings[i].valid?).to eq true
+        expect(bindings[i].inbound_index).to eq i
+        expect(bindings[i].inbound_name).to eq c.to_s
+        expect(bindings[i].operator).to eq operators[i]
+      end
+    end
+
+    it 'should parse header for where  fields and make available through inbound_column' do
+      # Owner has a name and belongs_to Project which has a title i.e lookup on title
+      headers = ['project:title']
+
+      bindings = binder.map_inbound_headers( Owner, headers )
+
+      expect(bindings.size).to eq headers.size
+
+      headers.each_with_index do |_c, i|
+        expect(bindings[i].valid?).to eq true
+        expect(bindings[i].inbound_column).to be_a InboundData::Column
+        expect(bindings[i].inbound_column.lookup_list).to be_a Array
+        expect(bindings[i].inbound_column.lookup_list.size).to eq 1
+        expect(bindings[i].inbound_column.lookup_list[0]).to be_a InboundData::LookupSupport
+        expect(bindings[i].inbound_column.lookups.first).to be_a InboundData::LookupSupport
+        expect(bindings[i].inbound_column.lookups.first).to eq bindings[i].inbound_column.lookup_list[0]
+      end
+    end
+
+    it 'should parse header for where clause field' do
+      # Owner has a name and belongs_to Project which has a title i.e lookup on title
+      headers = ['project:title', 'Project:title']
+
+      bindings = binder.map_inbound_headers( Owner, headers )
+
+      expect(bindings.size).to eq headers.size
+
+      headers.each_with_index do |_c, i|
+        first_lookup = bindings[i].inbound_column.lookups.first
+        expect(first_lookup.klass).to eq Project
+        expect(first_lookup.field).to eq 'title'
+        expect(first_lookup.where_value).to eq nil
+      end
+    end
+
+    it 'should parse header for where clause field & value belongs_to' do
+      # Owner has a name and belongs_to Project
+      headers = ['project:title:my first project', 'Project:title:my first project']
+
+      bindings = binder.map_inbound_headers( Owner, headers )
+
+      expect(bindings.size).to eq headers.size
+
+      headers.each_with_index do |_c, i|
+        first_lookup = bindings[i].inbound_column.lookups.first
+        expect(first_lookup.klass).to eq Project
+        expect(first_lookup.field).to eq 'title'
+        expect(first_lookup.where_value).to eq 'my first project'
+      end
+    end
+
+    it 'should parse header for where clause field & value has_one' do
+      # class Version < ActiveRecord::Base has_many :releases
+      # has_one :long_and_complex_table_linked_to_version
+      headers = ['long_and_complex_table_linked_to_version:price:10.2', 'Long_And Complex Table_linked_to_version:price:10.2']
+
+      bindings = binder.map_inbound_headers( Version, headers )
+
+      expect(bindings.size).to eq headers.size
+
+      headers.each_with_index do |_c, i|
+        first_lookup = bindings[i].inbound_column.lookups.first
+        expect(first_lookup.klass).to eq LongAndComplexTableLinkedToVersion
+        expect(first_lookup.field).to eq 'price'
+        expect(first_lookup.where_value).to eq '10.2'
+      end
+    end
+
+    it 'should parse header for where clause field & value & global data has_many' do
+      # Owner  has_many :digitals which has field attachment_file_name
+      headers = ['digitals:attachment_file_name:my pdf:random data for a loader',
+                 'Digitals:attachment_file_name:my pdf:random data for a loader']
+
+      bindings = binder.map_inbound_headers( Owner, headers )
+
+      expect(bindings.size).to eq headers.size
+
+      headers.each_with_index do |_c, i|
+        first_lookup = bindings[i].inbound_column.lookups.first
+        expect(first_lookup.klass).to eq Digital
+        expect(first_lookup.field).to eq 'attachment_file_name'
+        expect(first_lookup.where_value).to eq 'my pdf'
+        expect(bindings[i].inbound_column.data).to be_a Array
+        expect(bindings[i].inbound_column.data).to include 'random data for a loader'
+      end
+    end
+
+    it 'should enable us to sort bindings into arbitrary processing order' do
+      headers = ['Value as string', :value_as_string, 'value_as boolean', 'Value_As_Double']
+
+      operators = %w(value_as_string value_as_string value_as_boolean value_as_double)
+
+      bindings = binder.map_inbound_headers( Project, headers )
+
+      pending 'sorting methods'
+
+      bindings.sort
+    end
+  end
+
+end

data/spec/datashift/config_generator_spec.rb

@@ -0,0 +1,186 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# License::   MIT
+#
+require File.join(File.dirname(__FILE__), '/../spec_helper')
+
+module DataShift
+
+  describe  DataShift::ConfigGenerator do
+    include_context 'ClearAllCatalogues'
+
+    before(:each) do
+      results_clear
+    end
+
+    let(:config_generator) { DataShift::ConfigGenerator.new }
+
+    let(:expected_config_file) { result_file('mapping_service_project.yaml') }
+
+    let(:generate_config_file) {  config_generator.write_import(expected_config_file, Project) }
+
+    context 'erb generation' do
+
+      context 'basic templates without a class' do
+        it 'should generate an standard ERB template containing default mappings & config' do
+          result = config_generator.create_import_config Project
+
+          expect(result).to be_a String
+          expect(result).to include 'Project:'
+          expect(result).to include 'nodes:'
+        end
+
+        it 'should have a consistent starting title' do
+          result = config_generator.create_import_config Project
+
+          expect(result).to include 'Project'
+        end
+      end
+
+      context 'real mappings' do
+        let(:map_file) { result_file('project_mapper.yaml') }
+
+        it 'should generate a populated mapping doc for a class' do
+          generate_config_file
+
+          expect(File.exist?(expected_config_file)).to be true
+          expect( File.read(expected_config_file) ).to include 'Project:'
+        end
+
+        it 'should be able to extract headers from_excel', fail: true do
+          config_generator.generate_from_excel(ifixture_file('SimpleProjects.xls') )
+
+          expect(config_generator.headers.empty?).to eq false
+          expect(config_generator.headers.class).to eq Headers
+        end
+
+        it 'should be able to extract headers from_excel' do
+          config_generator.generate_from_excel(ifixture_file('SimpleProjects.xls') )
+
+          # bit flakey need to manually st expected spreadsheet values
+          # value_as_string,	Value as Text,	value as datetime,	value_as_boolean,	value_as_double
+
+          expect(config_generator.headers.size).to eq 5
+          expect(config_generator.headers[0]).to eq 'value_as_string'
+          expect(config_generator.headers[4]).to eq 'value_as_double'
+        end
+
+        it 'should be able to generate a mapping from_excel' do
+          expect(File.exist?(map_file)).to be false
+
+          config_generator.generate_from_excel(ifixture_file('SimpleProjects.xls'), file: map_file )
+
+          expect(File.exist?(map_file)).to be true
+        end
+      end
+    end
+
+    context 'Import' do
+
+      let(:expected_columns) { Project.new.serializable_hash.keys }
+
+      it 'should be able to write out a basic configuration document for a class' do
+        expect(File.exist?(expected_config_file)).to_not be true
+
+        expect { generate_config_file }.to_not raise_error
+
+        expect(File.exist?(expected_config_file)).to be true
+      end
+
+      it 'a basic mapping document should contain at least attributes of a class' do
+        generate_config_file
+
+        File.foreach(expected_config_file)
+        expect( $.).to be > expected_columns.size
+      end
+
+
+      it 'a basic mapping document can be configures to contain associations as well' do
+
+        DataShift::Exporters::Configuration.configure do |config|
+          config.with = [:all]
+        end
+
+        generate_config_file
+
+        expect(File.exist?(expected_config_file)).to be true
+
+        File.foreach(expected_config_file)
+        expect( $.).to be > expected_columns.size
+      end
+
+    end
+
+    # TODO: split into two - with and without associations
+
+    context 'Reading' do
+
+      let(:data_flow_schema) { DataFlowSchema.new }
+
+      #let(:config_file) { ifixture_file('ProjectConfiguration.yml') }
+
+      let(:expected_config_file) { result_file('mapping_service_project.yaml') }
+
+      let(:options) {
+        {
+          defaults:  {'value_as_string': 'some default text', 'value_as_double': 45.467 },
+          overrides: {'value_as_double': 45.467 },
+          postfixes: {'value_as_text': 'postfix value_as_text' },
+          substitutions:  { 'owner' => ['sub this text', 'for some other text'] }
+          # prefixs
+        }
+      }
+
+      let(:generate_config_file) { config_generator.write_import(expected_config_file, Project, options) }
+
+      before(:each) do
+
+        config = generate_config_file
+
+        expect(config).to be_a String
+        expect(config).to_not be_empty
+
+        expect(File.exist?(expected_config_file)).to be true
+
+        data_flow_schema.prepare_from_file(expected_config_file)
+      end
+
+      it 'should store the raw mapping data' do
+        expect(data_flow_schema.raw_data).to_not be_empty
+        expect(data_flow_schema.yaml_data).to_not be_empty
+      end
+
+      it 'should read in the nodes for this data schema' do
+        expect(data_flow_schema.nodes).to be_a NodeCollection
+        expect(data_flow_schema.nodes).to_not be_empty
+
+        node = data_flow_schema.nodes[0]
+        expect(node).to be_a NodeContext
+      end
+
+      it 'should provide access to details of the schema' do
+        expect(data_flow_schema.nodes.doc_context).to be_a DocContext
+        expect(data_flow_schema.nodes.doc_context.klass).to eq Project
+
+      end
+
+      it 'should have configured the Transformer', duff: true do
+
+        postfixes =  DataShift::Transformation.factory.postfixes_for(Project)
+
+        expect(postfixes).to be_a Hash
+        expect(postfixes.has_key?('value_as_integer')).to eq false
+        expect(postfixes.has_key?('value_as_text')).to eq true
+        expect(postfixes.size).to eq 1
+
+        expect(DataShift::Transformation.factory.defaults_for(Project).size).to eq 2
+        expect(DataShift::Transformation.factory.overrides_for(Project).size).to eq 1
+        expect(DataShift::Transformation.factory.substitutions_for(Project).size).to eq 1
+        expect(DataShift::Transformation.factory.prefixes_for(Project).size).to eq 0
+
+        expect(DataShift::Transformation.factory.get_postfix_on(Project, :value_as_text)).to eq 'postfix value_as_text'
+      end
+    end
+
+  end
+end # module