datashift 0.16.0 → 0.40.0

data/spec/loaders/excel_loader_failures_spec.rb

@@ -0,0 +1,67 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# License::   MIT
+#
+#
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module  DataShift
+
+  describe 'Excel Loader Failures' do
+    include_context 'ClearAllCatalogues'
+
+    let(:loader) { ExcelLoader.new }
+
+    let(:expected) { ifixture_file('SimpleProjects.xls') }
+
+    before(:each) do
+      create_list(:category, 5)
+    end
+
+    context 'progress monitor' do
+      it 'should process excel spreadsheet with extra undefined columns', duff: true do
+        expected = ifixture_file('BadAssociationName.xls')
+
+        loader.run(expected, Project)
+
+        puts loader.loaded_count
+        puts loader.failed_count
+
+      end
+    end
+
+    context 'reporters' do
+
+      it 'should be able to perform reporting' do
+        expect(loader.respond_to? :report).to eq true
+      end
+
+      it 'can access list of reporters' do
+        expect(loader.reporters).to be_a Array
+      end
+
+
+      class MyReporter < DataShift::Reporters::Reporter
+        def report
+          "Rspec expects this text"
+        end
+      end
+
+      let(:reporter) { MyReporter.new }
+
+      it 'can configure list of reporters' do
+        loader.reporters = [reporter]
+
+        expect(loader.reporters).to eq [reporter]
+      end
+
+      it 'can use a self configured list of reporters' do
+        loader.reporters = [reporter]
+
+        expect(loader.reporters.first.report).to eq "Rspec expects this text"
+      end
+
+    end
+
+  end
+end

data/spec/loaders/excel_loader_spec.rb

@@ -0,0 +1,294 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# License::   MIT
+#
+#
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module  DataShift
+
+  describe 'Excel Loader' do
+    include_context 'ClearAllCatalogues'
+
+    before(:each) do
+      DataShift::Transformation::Factory.reset
+    end
+
+    let(:loader) { ExcelLoader.new }
+
+    let(:expected) { ifixture_file('SimpleProjects.xls') }
+
+    context 'prepare to load' do
+
+      it 'should be able to create a new excel loader' do
+        expect(loader).to be
+      end
+
+      it 'should provide access to a context for the whole document' do
+        expect(loader.doc_context).to be_a DocContext
+      end
+
+      it 'should have access to open an Excel file' do
+        expect( loader.respond_to?(:start_excel)).to eq true
+        expect( loader.open_excel(expected, sheet_number: 0) ).to be_a Excel
+      end
+    end
+
+    context 'basic load operations' do
+
+      before(:each) do
+        create_list(:category, 5)
+      end
+
+      it 'should provide access to the file_name' do
+        loader.run(expected, Project)
+        expect(loader.file_name).to eq expected
+      end
+
+      it 'should parse the headers' do
+
+        loader.run(expected, Project)
+        expect(loader.headers.class).to eq Headers
+        # TOFIX flakey - is it possible to read from 'SimpleProjects.xls'
+        expect(loader.headers).to eq ['value_as_string',	'Value as Text', 'value as datetime', 'value_as_boolean',	'value_as_double']
+        expect(loader.headers.idx).to eq 0
+      end
+
+      it 'should bind headers to real class methods' do
+        loader.run(expected, Project)
+
+        expect(loader.binder).to be
+
+        binding = loader.binder.bindings[1]
+
+        expect(binding.valid?).to eq true
+        expect(binding.inbound_index).to eq 1
+        expect(binding.inbound_name).to eq 'Value as Text'
+        expect(binding.operator).to eq 'value_as_text'
+
+        expect(Project.new.respond_to?(binding.operator)).to eq true
+      end
+    end
+
+    context 'creates new records' do
+
+      it 'should process a simple .xls spreedsheet' do
+        count = Project.count
+
+        loader.run(expected, Project)
+
+        expect(loader.loaded_count).to eq 3
+        expect(Project.count).to eq count + 3
+      end
+
+      #         it "should provide access to main subject class of the load" do
+      #           expect(loader.load_object_class).to eq Project
+      #         end
+      #
+      #         it "should provide access to main subject of the load" do
+      #           expect(loader.load_object).to be_a Project
+      #           expect(loader.load_object.new_record?).to eq true
+      #         end
+
+      it 'should populate database objects from a simple .xls spreedsheet' do
+        loader.run(expected, Project)
+
+        loaded = Project.last
+
+        expect(loaded.value_as_string).to eq '003 Can handle different column naming styles'
+        expect(loaded.value_as_text).to eq ''
+        expect(loaded.value_as_datetime).to eq '2011-05-19'
+        expect(loaded.value_as_boolean).to eq true
+        expect(loaded.value_as_double).to eq 520.00
+      end
+
+      it 'should process multiple associations from single column' do
+        expect(Project.find_by_title('001')).to be_nil
+
+        count = Project.count
+
+        expected = ifixture_file('ProjectsSingleCategories.xls')
+
+        loader.run(expected, Project)
+
+        expect(loader.loaded_count).to eq 4
+
+        expect(loader.loaded_count).to eq (Project.count - count)
+
+        { '001' => 2, '002' => 1, '003' => 3, '099' => 0 }.each do |title, expected|
+          project = Project.find_by_title(title)
+
+          expect(project).to_not be_nil
+
+          expect(project.categories.size).to eq expected
+        end
+      end
+
+      it 'should process multiple associations in excel spreadsheet' do
+        count = Project.count
+
+        expected = ifixture_file('ProjectsMultiCategories.xls')
+
+        loader.run(expected, Project)
+
+        expect(loader.loaded_count).to eq (Project.count - count)
+
+        { '004' => 3, '005' => 1, '006' => 0, '007' => 1 }.each do |title, expected|
+          project = Project.find_by_title(title)
+
+          expect(project).to_not be_nil
+
+          expect(project.categories.size).to eq expected
+        end
+      end
+
+      it 'should process multiple associations with lookup specified in column from excel spreadsheet' do
+        count = Project.count
+
+        expected = ifixture_file('ProjectsMultiCategoriesHeaderLookup.xls')
+
+        loader.run(expected, Project)
+
+        expect(loader.loaded_count).to eq 4
+        expect(Project.count).to eq count + 4
+
+        { '004' => 4, '005' => 1, '006' => 0, '007' => 1 }.each do |title, expected|
+          project = Project.find_by_title(title)
+
+          expect(project).to_not be_nil
+
+          expect(project.categories.size).to eq expected
+        end
+      end
+
+      it 'should process excel spreadsheet with extra undefined columns' do
+        expected = ifixture_file('BadAssociationName.xls')
+
+        expect { loader.run(expected, Project) }.to_not raise_error
+      end
+
+      it 'should NOT process excel spreadsheet with extra undefined columns when strict mode' do
+        expected = ifixture_file('BadAssociationName.xls')
+
+        loader.configuration.strict = true
+
+        expect { loader.run(expected, Project) }.to raise_error(MappingDefinitionError)
+      end
+
+      it 'should raise an error when mandatory columns missing' do
+        expected = ifixture_file('ProjectsMultiCategories.xls')
+
+        expect {
+
+          loader.configuration.mandatory = %w(not_an_option must_be_there)
+
+          loader.run(expected, Project)
+        }.to raise_error(DataShift::MissingMandatoryError)
+      end
+    end
+
+    context 'update existing records' do
+    end
+
+    context 'internal configuration of loader' do
+      let(:expected)  { ifixture_file('ProjectsSingleCategories.xls') }
+
+      before(:each) do
+        DataShift::Transformation.factory.clear
+      end
+
+      it 'should use global transforms to set default values' do
+        texpected = Time.now.to_s(:db)
+
+        DataShift::Transformation.factory do |factory|
+          factory.set_default_on(Project, 'value_as_string', 'some default text' )
+          factory.set_default_on(Project, 'value_as_double', 45.467 )
+          factory.set_default_on(Project, 'value_as_boolean', true )
+          factory.set_default_on(Project, 'value_as_datetime', texpected )
+        end
+
+        # value_as_string	Value as Text	value as datetime	value_as_boolean	value_as_double	category
+
+        loader.run(expected, Project)
+
+        p = Project.find_by_title( '099' )
+
+        expect(p).to_not be_nil
+
+        expect(p.value_as_string).to eq 'some default text'
+        expect(p.value_as_double).to eq 45.467
+        expect(p.value_as_boolean).to eq true
+        expect(p.value_as_datetime.to_s(:db)).to eq texpected
+
+        # expected: "2012-09-17 10:00:52"
+        # got: Mon Sep 17 10:00:52 +0100 2012 (using ==)
+
+        p_no_defs = Project.first
+
+        expect(p_no_defs.value_as_string).to_not eq 'some default text'
+        expect(p_no_defs.value_as_double).to_not eq 45.467
+        expect(p_no_defs.value_as_datetime).to_not eq texpected
+      end
+
+      it 'should use global transforms to set pre and post fix values' do
+        DataShift::Transformation.factory do |factory|
+          factory.set_prefix_on(Project, 'value_as_string', 'myprefix' )
+          factory.set_postfix_on(Project, 'value_as_string', 'my post fix' )
+        end
+
+        loader.run(expected, Project)
+
+        p = Project.find_by_title( '001' )
+
+        expect(p).to_not be_nil
+
+        expect(p.value_as_string).to eq 'myprefixDemo stringmy post fix'
+      end
+    end
+
+    context 'external configuration of loader' do
+      let(:expected)  { ifixture_file('ProjectsSingleCategories.xls') }
+
+      before(:each) do
+        DataShift::Transformation.factory.clear
+        loader.setup_load_class(Project)
+
+        loader.configure_from( ifixture_file('ProjectConfiguration.yml') )
+
+        expect(Project.where(title: '099').first).to be_nil
+
+        loader.run(expected, Project)
+      end
+
+      it 'should provide facility to set DEFAULT values via YAML configuration', duff: true  do
+        loader.run(expected, Project)
+
+        p = Project.find_by_title( '099' )
+
+        expect(p).to_not be_nil
+
+        # yaml has snippet to set Time to 'now' .. at least match the date part
+        expect(p.value_as_datetime.to_s).to include(Date.today.to_s)
+      end
+
+      it 'should combine PREFIX and POSTFIX transformations set via YAML configuration' do
+        p = Project.find_by_title( '099' )
+
+        expect(p).to_not be_nil
+
+        expect(p.value_as_string).to eq 'prefix me every-time Default Project Value postfix me every-time'
+      end
+
+      it 'should provide facility to OVER RIDE values via YAML configuration' do
+
+        expected = DataShift::Transformation.factory.overrides_for(Project)[:value_as_double]
+
+        Project.all.each { |p|
+          # TOFIX - the scale of the DB column from migration is 4 -- can we get that dynamically ?
+          expect(p.value_as_double).to be_within(0.0001).of(expected)
+        }
+      end
+
+    end
+  end
+end

data/spec/loaders/loader_base_spec.rb

@@ -0,0 +1,29 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Details::   Specs for base class Loader
+#
+require File.dirname(__FILE__) + '/../spec_helper'
+
+require 'erb'
+
+module DataShift
+  describe 'LoaderBase' do
+    let(:loader) { LoaderBase.new }
+
+    it 'should be able to create an empty loader with basic load object' do
+      expect(loader.load_object).to be_a DataShift::LoadObject
+      expect(loader.file_name).to eq ''
+      expect(loader.doc_context).to be_a(DocContext)
+      expect(loader.binder).to be_a(Binder)
+    end
+
+    it 'should be able to set the file_name ot load' do
+      loader.file_name = 'Test.csv'
+      expect(loader.file_name).to eq 'Test.csv'
+    end
+  end
+
+end

data/spec/loaders/paperclip_loader_spec.rb

@@ -0,0 +1,106 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# License::   MIT
+#
+#
+require File.dirname(__FILE__) + '/../spec_helper'
+
+require 'paperclip/attachment_loader'
+
+Paperclip.options[:command_path] = '/usr/local/bin/'
+
+describe 'PaperClip Bulk Loader' do
+  include DataShift::Logging
+
+  before(:each) do
+    DataShift::Transformation::Factory.reset
+  end
+
+  module Paperclip
+    module Interpolations
+
+      # Returns the Rails.root constant.
+      def rails_root(_attachment, _style_name)
+        '.'
+      end
+    end
+  end
+
+  let(:path) { File.join(fixtures_path, 'images') }
+
+  let(:common_options) { { verbose: true } }
+
+  # Owner.where(:name = 'jeff mills').first.digitals << DigitalAttachmentLoadedFromPath
+  let(:attachment_options) do
+    {
+      attach_to_klass: Owner,
+      attach_to_find_by_field: :name,
+      attach_to_field: :digitals
+    }
+  end
+
+  let(:paper_clip_attachment_class) { Digital }
+
+  let(:loader) { DataShift::Paperclip::AttachmentLoader.new }
+
+  it 'should create a new paperclip loader to load a directory of attachments' do
+    expect(loader).to be
+    expect(loader.attach_to_klass).to be_nil
+    expect(loader.attach_to_find_by_field).to be_nil
+    expect(loader.attach_to_field).to be_nil
+  end
+
+  it 'can be configured via Hash to load attachments against class found via field name, attached to another field' do
+    loader.init_from_options(attachment_options)
+    expect(loader.attach_to_klass).to eq attachment_options[:attach_to_klass]
+    expect(loader.attach_to_find_by_field).to eq attachment_options[:attach_to_find_by_field]
+    expect(loader.attach_to_field).to eq attachment_options[:attach_to_field]
+  end
+
+  it 'can be configured to load attachments against class as a String' do
+    attachment_options[:attach_to_klass] = 'Owner'
+    loader.init_from_options(attachment_options)
+    expect(loader.attach_to_klass).to eq  Owner
+  end
+
+  it 'can be initialised directly with owning class, found via field name, attached to another field' do
+    loader.init(Owner, :image, :digitals)
+    expect(loader.attach_to_klass).to eq Owner
+    expect(loader.attach_to_find_by_field).to eq :image
+    expect(loader.attach_to_field).to eq :digitals
+  end
+
+  context('Loading attachments') do
+
+    let(:owner_names) { %w(DEMO_001 DEMO_002 DEMO_003 DEMO_004) }
+
+    before(:each) do
+      # these Owner names should be embeeded in the attachment FILE NAME somewhere
+      owner_names.each do |n|
+        Owner.create( name: n )
+      end
+
+      loader.init_from_options attachment_options
+    end
+
+    it 'should bulk load from a directory file system', duff: true do
+      loader.split_file_name_on = '_'
+
+      loader.run(path, Digital)
+
+      expect(Owner.first.digitals.size).to be > 0
+    end
+
+    it 'should save failed images to folder when unable to find matching record' do
+
+      # use a non existent field to cause error
+      loader.attach_to_find_by_field = :junk
+      loader.split_file_name_on = '_'
+
+      loader.run(path, Digital)
+
+      expect(Dir.glob('MissingAttachmentRecords/*.jpeg', File::FNM_CASEFOLD).size).to eq owner_names.size
+    end
+  end
+
+end