datashift 0.16.0 → 0.40.0

data/spec/datashift/populator_spec.rb

@@ -0,0 +1,117 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# License::   MIT
+#
+#
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module DataShift
+
+  describe 'Populator' do
+    include_context 'ClearThenManageProject'
+
+    it 'should be able to create a new populator' do
+      expect(DataShift::Populator.new).to be
+    end
+
+    it 'should process a DSL string into a real hash' do
+      str1 = "{:name => 'the_fox' }"
+
+      x = DataShift::Populator.string_to_hash( str1 )
+
+      expect(x).to be_a Hash
+      expect(x.size).to eq 1
+
+      str2 = "{:name => 'the_fox', 'occupation' => 'fantastic', :food => 'duck soup' }"
+
+      x = DataShift::Populator.string_to_hash( str2 )
+
+      expect(x.size).to eq 3
+      expect(x).to eq('name' => 'the_fox', 'occupation' => 'fantastic', 'food' => 'duck soup')
+    end
+
+    it 'should process simplified syntax string into a real hash' do
+      str3 = "{cost_price: '13.45', price: 23,  sale_price: 4.23 }"
+
+      x = DataShift::Populator.string_to_hash( str3 )
+
+      expect(x.keys).to include 'price'
+      expect(x['cost_price']).to eq '13.45'
+      expect(x['price']).to eq 23
+      expect(x['sale_price']).to eq 4.23
+    end
+
+    it 'should process mixed hash syntax string into a real hash' do
+      str = "{:cost_price => '13.45', price: 23,  :sale_price => 4.23 }"
+
+      x = DataShift::Populator.string_to_hash( str )
+
+      expect(x.size).to eq 3
+      expect(x.keys).to include 'cost_price'
+      expect(x.keys).to include 'price'
+      expect(x['cost_price']).to eq '13.45'
+      expect(x['price']).to eq 23
+      expect(x['sale_price']).to eq 4.23
+    end
+
+    context 'prepare data' do
+      let(:model_method)    { project_collection.search('value_as_string') }
+
+      let(:method_binding)  { MethodBinding.new('value_as_string', 1, model_method) }
+
+      let(:populator)       { DataShift::Populator.new }
+
+      let(:data)            { 'some text for the string' }
+
+      it 'should prepare inbound string data for a method binding' do
+        value, attributes = populator.prepare_data(method_binding, data)
+
+        expect(value).to eq data
+        expect(attributes).to be_a Hash
+        expect(attributes.empty?).to eq true
+      end
+
+      it 'should prepare inbound array data for a method binding' do
+        list = create_list(:milestone, 4)
+
+        method_binding =  MethodBinding.new('milestones', 1, project_collection.search('milestones') )
+
+        value, attributes = populator.prepare_data(method_binding, list)
+
+        expect(value).to eq list
+        expect(attributes).to be_a Hash
+        expect(attributes.empty?).to eq true
+      end
+
+      it 'should prepare inbound active relation a method binding' do
+        create_list(:loader_release, 5)
+
+        list = LoaderRelease.all
+
+        method_binding =  MethodBinding.new('milestones', 1, project_collection.search('milestones') )
+
+        value, attributes = populator.prepare_data(method_binding, list)
+
+        expect(value).to be_a Array
+        expect(value.size).to eq list.size
+
+        expect(attributes).to be_a Hash
+        expect(attributes.empty?).to eq true
+      end
+
+      it 'should process a string value with attributes' do
+        data = 'Get up Lazy fox {:name => \'the_fox\', food: chickens }'
+
+        value, attributes = populator.prepare_data(method_binding, data)
+
+        expect(value).to eq 'Get up Lazy fox '
+
+        expect(attributes).to be_a Hash
+        expect(attributes.size).to eq 2
+        expect(attributes.keys).to include 'name'
+        expect(attributes['name']).to eq 'the_fox'
+      end
+    end
+  end
+
+end # module

data/spec/datashift/thor_spec.rb

@@ -0,0 +1,314 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+#
+# License::   MIT - Free, OpenSource
+#
+# Details::   Specification for Thor tasks supplied with datashift
+#
+#             Test are often within a call to
+#                 run_in(rails_sandbox_path)
+#             so we can access real model data from the dummy app
+#
+require 'thor'
+require 'thor/group'
+require 'thor/runner'
+
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module DataShift
+
+  describe 'Thor high level command line tasks' do
+
+    context 'Loading commands' do
+
+      before(:all) do
+        DataShift.load_commands
+      end
+
+      it 'should list available datashift thor tasks' do
+        x = run_in(rails_sandbox_path) do
+          capture_stream(:stdout){ require 'datashift' ; Thor::Runner.start(["list"]) }
+        end
+
+        expect(x).to start_with("datashift\n--------")
+
+        expect(x).to include "csv"
+        expect(x).to include "excel"
+        expect(x).to include "csv"
+      end
+
+      it "tasks have been instantiated" do
+        expect(Datashift.constants.include?(:Import)).to be
+
+        expect(Datashift::Generate.new).to be
+        expect(Datashift::Config.new).to be
+        expect(Datashift::Export.new).to be
+        expect(Datashift::Import.new).to be
+        expect(Datashift::Generate.new).to be
+        expect(Datashift::Paperclip.new).to be
+      end
+
+    end
+
+    context 'Export CLI' do
+
+      let(:number_of_projects) { 7 }
+
+      before(:all) do
+        DataShift.load_commands
+      end
+
+      before(:each) do
+        clear_everything
+
+        create_list(:project, number_of_projects)
+
+        @user = create( :project_with_user ).user   # creates a Project with associations
+      end
+
+      it 'should run datashift:export:csv to export a model to CSV file' do
+        expected =  result_file 'rspec_thor_project_export.csv'
+
+        args = ['--model', 'Project', '--result', expected, '--associations']
+
+        run_in(rails_sandbox_path) do
+
+          capture_stream(:stdout) { Datashift::Export.new.invoke(:csv, [], args) }
+
+          expect(File.exist?(expected)).to eq true
+
+          File.foreach(expected) {}
+          count = $INPUT_LINE_NUMBER
+          expect(count).to eq number_of_projects + 2     # + 2 cos - 1 header row, 1 row for project_with_user
+        end
+      end
+
+      it 'should run datashift:export:csv to export a model to a EXCEL file' do
+        expected =  result_file 'rspec_thor_project_export.xls'
+
+        args = ['--model', 'Project', '--result', expected, '--associations']
+
+        run_in(rails_sandbox_path) do
+
+          capture_stream(:stdout) { Datashift::Export.new.invoke(:excel, [], args) }
+
+          expect(File.exist?(expected)).to eq true
+
+          excel = DataShift::Excel.new
+          excel.open(expected)
+
+          expected_rows = Project.count + 1
+          last_idx = Project.count
+
+          expect(excel.num_rows).to eq expected_rows
+
+          user_column_index = excel.row(0).index 'user'
+          owner_column_index = excel.row(0).index 'owner'
+
+          expect(user_column_index).to be > -1
+
+          # Factories should have built the project WITHOUT associated owner data first
+          expect( excel[1, owner_column_index] ).to be_nil
+          expect( excel[1, user_column_index] ).to_not be_nil
+
+          # Factories should have built the project WITH real associated user data last
+          expect( excel[last_idx, user_column_index] ).to include 'mr'
+
+          owner_idx = excel.row(0).index 'owner'
+
+          expect(owner_idx).to be > -1
+
+          expect( excel[last_idx, owner_idx] ).to include '10000.23'
+        end
+      end
+
+      it 'should run datashift:export:db to export whole DB to EXCEL', duff: true do
+
+        # Writes one file per model into a PATH
+        expected_path =  results_path
+
+        args = ['--model', 'Project', '--result', expected_path, '--associations']
+
+        run_in(rails_sandbox_path) do
+
+          output = capture_stream(:stdout) { Datashift::Export.new.invoke(:db, [], args) }
+
+          ['users', 'projects', 'milestones', 'owners', 'categories', 'versions', 'loader_releases',
+           'long_and_complex_table_linked_to_versions', 'empties', 'digitals', 'categories'].each do |f|
+
+            expected = File.join(expected_path, "#{f}.xls")
+            expect(File.exist?(expected) ).to be_truthy, "MissingFile\nExport file [#{expected}] not found"
+          end
+
+          excel = Excel.new
+          excel.open(  expected = File.join(expected_path, "projects.xls") )
+
+          expected_rows = Project.count + 1
+          last_idx = Project.count
+
+          expect(excel.num_rows).to eq expected_rows
+
+          user_column_index = excel.row(0).index 'user'
+          owner_column_index = excel.row(0).index 'owner'
+
+          expect(user_column_index).to be > -1
+
+          # Factories should have built the project WITHOUT associated owner data first
+          expect( excel[1, owner_column_index] ).to be_nil
+          expect( excel[1, user_column_index] ).to_not be_nil
+
+          # Factories should have built the project WITH real associated user data last
+          expect( excel[last_idx, user_column_index] ).to include 'mr'
+
+          expect(owner_column_index).to be > -1
+          expect( excel[last_idx, owner_column_index] ).to include '10000.23'
+        end
+      end
+
+    end # end Export CLi
+
+    context 'Import CLI' do
+
+      before(:all) do
+        DataShift.load_commands
+      end
+
+      before(:each) do
+        clear_everything
+      end
+
+      it 'should run datashift:config:import - generate skeleton import config for a model' do
+        expected = result_file('thor_spec_project_config.yaml')
+
+        expect(File.exists?(expected)).to eq false
+
+        #t datashift:config:import -m Spree::Variant -r /tmp
+
+        run_in(rails_sandbox_path) do
+          options = ['--model', 'Project', '--result', expected]
+
+          output = capture_stream(:stdout) { Datashift::Config.new.invoke(:import, [], options) }
+
+          expect(File.exists?(expected)).to eq true
+          expect(output).to include('Creating new configuration file')
+        end
+      end
+
+      it 'should run datashift:import:csv to import data from a CSV file' do
+
+        input_file = ifixture_file('csv/SimpleProjects.csv')
+
+        options = ['--model', 'Project', '--input', input_file]
+
+        run_in(rails_sandbox_path) do
+          count = Project.count
+
+          output = capture_stream(:stdout) { Datashift::Import.new.invoke(:csv, [], options) }
+
+          # TODO for now hard code 3  - should probably count the lines in the file
+          expect(Project.count - count).to eq 3
+
+          expect(output).to include('Processing Summary Report')
+          expect(output).to include('There were NO failures')
+        end
+
+      end
+
+      it 'should run datashift:import:excel to import data from Excel spreadsheet' do
+
+        input_file = ifixture_file('SimpleProjects.xls')
+
+        options = ['--model', 'Project', '--input', input_file]
+
+        run_in(rails_sandbox_path) do
+          count = Project.count
+
+          output = capture_stream(:stdout) { Datashift::Import.new.invoke(:excel, [], options) }
+
+          # TODO for now hard code 3  - should probably count the lines in the file
+          expect(Project.count - count).to eq 3
+
+          expect(output).to include('Processing Summary Report')
+          expect(output).to include('There were NO failures')
+        end
+
+      end
+
+    end
+
+    context 'Generate CLI' do
+      before(:each) do
+        results_clear
+      end
+
+      it 'should provide tasks to generate a mapping doc' do
+
+        # Access real model in dummy
+        run_in(rails_sandbox_path) do
+          cmd = ['--model', 'Project', '--result', File.join(results_path, 'thor_spec_gen_project.csv')]
+
+          output = capture_stream(:stdout) { Datashift::Generate.new.invoke(:csv, [], cmd) }
+
+          puts output
+          expect(File.exists?(File.join(results_path, 'thor_spec_gen_project.csv'))).to eq true
+          expect(output).to include('Datashift: CSV Template COMPLETED')
+        end
+
+      end
+    end
+
+    # thor datashift:export:csv -m, --model=MODEL -r, --result=RESULT              ...
+    # thor datashift:export:excel -m, --model=MODEL -r, --result=RESULT            ...
+    # thor datashift:generate:excel -m, --model=MODEL -r, --result=RESULT          ...
+    # thor datashift:import:csv -i, --input=INPUT -m, --model=MODEL                ...
+    # thor datashift:import:excel -i, --input=INPUT -m, --model=MODEL              ...
+    # thor datashift:paperclip:attach -a, --attachment-klass=ATTACHMENT_KLASS -f, -...                                     ...
+    # thor datashift:tools:zip -p, --path=PATH -r, --results=RESULTS               ...
+
+
+    # N.B Tasks that fire up Rails application need to be run in own Thread or else get
+    #  ...  You cannot have more than one Rails::Application
+
+    it 'should be able to import a model from a complex excel through import CLI' do
+      skip 'How to run once rails already initialzed .. error : database configuration does not specify adapter'
+
+      x = Thread.new {
+        run_in(Sandbox.rails_sandbox_path) do
+          stdout = capture_stream(:stdout){
+            Thor::Runner.start(['datashift:import:excel', '-m', 'Project', '-i', ifixture_file('ProjectsSingleCategories.xls')])
+          }
+          puts stdout
+        end
+      }
+      x.join
+    end
+
+    it 'should attach Images to a specified Class from a directory' do
+      skip 'better understanding of testign thor'
+
+      # Owner has_many :digitals of type Digital
+
+      # Owner has a name by which we can id/look it up
+
+      args = [
+        '--attachment-klass',        'Digital',
+        '--attach-to-klass',         'Owner',
+        '--attach-to-find-by-field', 'name',
+        '--attach-to-field',         'digitals']
+
+      # which boils down to
+      #
+      # Digital.find_by_name( abc ).digitals << :Digital.new( File.read('abc_001.jpg') )
+
+      args << '--input' << File.join(fixtures_path, 'images')
+
+      puts "Running attach with: #{args}"
+
+      x = capture_stream(:stdout) { Thor::Runner.start(['datashift:paperclip:attach', [], args]) }
+
+      expect(x).to include "datashift\n--------"
+      # x.should =~ / csv -i/
+      # x.should =~ / excel -i/
+    end
+  end
+end

data/spec/datashift/transformation/factory_spec.rb

@@ -0,0 +1,195 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# License::   MIT
+#
+# Details::   Specs around Transforming inbound data
+#
+require File.join(File.dirname(__FILE__), '/../../spec_helper')
+
+module  DataShift
+
+  describe Transformation do
+    include_context 'ClearThenManageProject'
+
+    context 'basics' do
+
+      it 'should create standard method formats for all transformations' do
+
+        name = DataShift::Transformation::Factory::TRANSFORMERS_HASH_INSTANCE_NAMES.first
+
+        expect(DataShift::Transformation.factory).to respond_to "#{name}s_for"
+
+        expect(DataShift::Transformation.factory).to respond_to "defaults_for"
+        expect(DataShift::Transformation.factory).to respond_to "set_default"
+        expect(DataShift::Transformation.factory).to respond_to "set_default_on"
+        expect(DataShift::Transformation.factory).to respond_to "default?"
+        expect(DataShift::Transformation.factory).to respond_to "default"
+        expect(DataShift::Transformation.factory).to respond_to "get_default_on"
+      end
+
+
+      it 'should enable settings to be cleared' do
+
+        #  [:default, :override, :substitution, :prefix, :postfix]
+
+        DataShift::Transformation.factory do |factory|
+          factory.set_default_on(Project, "name", 'default name')
+          factory.set_override_on(Project, "name", 'override name')
+          factory.set_prefix_on(Project, "name", 'prefix name')
+          factory.set_postfix_on(Project, "name", 'postfix name')
+
+          factory.set_substitution_on(Project, "name", 'if its blah', 'substitution name')
+        end
+
+        expect(DataShift::Transformation.factory.get_default_on(Project, "name")).to eq 'default name'
+        expect(DataShift::Transformation.factory.get_override_on(Project, "name")).to eq 'override name'
+        expect(DataShift::Transformation.factory.get_prefix_on(Project, "name")).to eq 'prefix name'
+        expect(DataShift::Transformation.factory.get_postfix_on(Project, "name")).to eq 'postfix name'
+        expect(DataShift::Transformation.factory.get_substitution_on(Project, "name")).to be_a Struct::Substitution
+
+        DataShift::Transformation.factory.clear
+
+        expect(DataShift::Transformation.factory.get_default_on(Project, "name")).to be_nil
+        expect(DataShift::Transformation.factory.get_override_on(Project, "blah")).to be_nil
+        expect(DataShift::Transformation.factory.get_prefix_on(Project, "blah")).to be_nil
+        expect(DataShift::Transformation.factory.get_postfix_on(Project, "blah")).to be_nil
+        expect(DataShift::Transformation.factory.get_substitution_on(Project, "blah")).to be_nil
+      end
+    end
+
+    context 'over-rides' do
+      let(:model_method)    { project_collection.search('value_as_string') }
+
+      let(:method_binding)  { MethodBinding.new('value_as_string', 1, model_method) }
+
+      let(:populator)       { DataShift::Populator.new }
+
+      let(:data)            { 'some text for the string' }
+
+      before(:each) do
+        DataShift::Transformation.factory.clear
+
+        DataShift::Exporters::Configuration.reset
+      end
+
+      it 'over-ride should always over-ride value regardless of real value' do
+        DataShift::Transformation.factory do |factory|
+          factory.set_override(method_binding, 'override text')
+        end
+
+        value, _attributes = populator.prepare_data(method_binding, data)
+
+        expect(value).to eq 'override text'
+      end
+
+      context 'DefaultValues for Columns' do
+        it 'should use default value when nil' do
+          DataShift::Transformation.factory do |factory|
+            factory.set_default(method_binding, 'default text')
+          end
+
+          value, _attributes = populator.prepare_data(method_binding, nil)
+          expect(value).to eq 'default text'
+        end
+
+        it 'should use default value when empty string' do
+          DataShift::Transformation.factory do |factory|
+            factory.set_default(method_binding, 'default text')
+          end
+
+          value, _attributes = populator.prepare_data(method_binding, '')
+          expect(value).to eq 'default text'
+        end
+      end
+
+      it 'should use substitution when relevant' do
+        DataShift::Transformation.factory do |factory|
+          factory.set_substitution(method_binding, 'text', ' replaced with me')
+        end
+
+        value, _attributes = populator.prepare_data(method_binding, data)
+        expect(value).to eq 'some  replaced with me for the string'
+      end
+
+      it 'should add a prefix' do
+        DataShift::Transformation.factory do |factory|
+          factory.set_prefix(method_binding, 'added me before')
+        end
+
+        value, _attributes = populator.prepare_data(method_binding, data)
+        expect(value).to eq 'added me before' + data
+      end
+
+      it 'should add a postfix' do
+        DataShift::Transformation.factory do |factory|
+          factory.set_postfix(method_binding, 'added me after')
+        end
+
+        value, _attributes = populator.prepare_data(method_binding, data)
+        expect(value).to eq data + 'added me after'
+      end
+    end
+
+    let(:config_file) {ifixture_file('ProjectConfiguration.yml') }
+
+    context 'Configuration of Transformations' do
+
+      before(:each) do
+        DataShift::Transformation.factory.clear
+        DataShift::Transformation.factory.configure_from(Project, config_file )
+      end
+
+      it 'should provide facility to set default values via YAML configuration', duff: true do
+
+        puts  DataShift::Transformation.factory.inspect
+
+        defaults =  DataShift::Transformation.factory.defaults_for(Project)
+        expect(defaults).to be_a Hash
+        expect(defaults.size).to eq 3
+      end
+
+      it 'should provide facility to set override values via YAML configuration' do
+        override =  DataShift::Transformation.factory.overrides_for(Project )
+        expect(override).to be_a Hash
+        expect(override.has_key?('value_as_integer')).to eq true
+        expect(override.size).to eq 2
+      end
+
+      it 'should provide facility to substitute values via YAML configuration' do
+        substitutes =  DataShift::Transformation.factory.substitutions_for(Project )
+        expect(substitutes).to be_a Hash
+        expect(substitutes.has_key?('value_as_text')).to eq true
+
+        sub = substitutes['value_as_text']
+
+        expect(sub).to be_a Struct::Substitution
+        expect(sub.pattern).to eq "change me"
+        expect(sub.replacement).to eq "i only gone and got meself changed by datashift"
+
+        expect(substitutes.size).to eq 1
+      end
+
+      it 'should provide facility to set prefixes via YAML configuration' do
+        prefixes =  DataShift::Transformation.factory.prefixes_for(Project )
+        expect(prefixes).to be_a Hash
+        expect(prefixes.has_key?('value_as_string')).to eq true
+        expect(prefixes.size).to eq 1
+      end
+
+      it 'should provide facility to set postfixes via YAML configuration' do
+        postfixes =  DataShift::Transformation.factory.postfixes_for(Project )
+        expect(postfixes).to be_a Hash
+        expect(postfixes.has_key?('value_as_integer')).to eq false
+        expect(postfixes.has_key?('value_as_string')).to eq true
+        expect(postfixes.has_key?('value_as_text')).to eq true
+        expect(postfixes.size).to eq 2
+      end
+
+      after(:all) do
+        DataShift::Transformation.factory.clear
+      end
+
+    end
+  end
+
+end