go_import 3.0.39 → 3.0.40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 10b4c342bbed4907bb37223c88ae78ca2dccdd99
-  data.tar.gz: c441c76c21db917c6dbc7839565e7f1721494922
+  metadata.gz: 59427aef6763d25193bcb19a34b834207e0a1afa
+  data.tar.gz: 171379596422ffb7b8bf31d2d071ebb05682db26
 SHA512:
-  metadata.gz: 4bf2826818be40929267eb8495265ad4b833dda32752cd9c9484683bcc17e2d5b01be9c728fbb9cd7a34d4de503d7d15310f6765369cb19d72cb78fcf416d1c5
-  data.tar.gz: 3f5727edd5253712c0dfc7e296887d0e45c2be471a0511598edf63e69c07068662250e65a4ecff51046b0dfbd055dcfb5e2bb40d4f4b8b163582d2936595800b
+  metadata.gz: b8e6aa91dec72dbffe0bcf9d430b78a51c28208c60758a3dd85f959dce032b50d7f0f95402c4996eec20cace636173d7010a23909df1008c03ee983a9c33cfaf
+  data.tar.gz: f3d1e0faa54a918fce9136b5247f8af2dd6d9d6b06dffb895d9d3550d87cc24a9ba92a069f59f7b7e9ce125a5ccf3959dd468f5cb94b741b9a8cb264469fcc9f

data/bin/go-import

@@ -67,6 +67,10 @@ class GoImportCommandLine < Thor
            :desc => "Name of the file to put imported documents in (default in same as --output)",
            :type => :string,
            :required => false)
+    option(:shard_size,
+           :desc => "Large imports are sharded into several zip-files. This property sets how many objects each zip-file should contain. Default is 25 000",
+           :type => :numeric,
+           :required => false)
     def run_import()
         if !options.log_to_file.nil?
             $stdout = File.new(options.log_to_file == "log_to_file" ? "go-import.log" : options.log_to_file, 'w')
@@ -99,14 +103,25 @@ class GoImportCommandLine < Thor
                 log_and_remove_invalid_files model, max_file_size
             end
 
-            go_data_zip = options.output.nil? == true ? "go.zip" : options.output
-            go_files = options.output_documents.nil? == true ? nil : ::File.basename(options.output_documents,File.extname(options.output_documents))
-            model.save_to_zip(go_data_zip, go_files)
-            puts "Source has been been converted into '#{go_data_zip}'."
-            puts "  - and files into '#{go_files}.zip'." if !go_files.nil?
-            if !warnings_msg.empty? 
-                puts "WARNINGS: "
-                puts warnings_msg
+            puts "Starting sharding of model..."
+            sharder = GoImport::ShardHelper.new(options.shard_size)
+            models_to_serialize = sharder.shard_model(model)
+
+            if models_to_serialize.length > 1
+                puts "Import is large and will be broken into #{models_to_serialize.length} files"
+            end
+
+            models_to_serialize.each_with_index do |model, i| 
+                go_data_zip = options.output.nil? == true ? "go" : options.output
+                go_data_zip += "_#{i}.zip"
+                go_files = options.output_documents.nil? == true ? nil : ::File.basename(options.output_documents,File.extname(options.output_documents))
+                model.save_to_zip(go_data_zip, go_files)
+                puts "Source has been been converted into '#{go_data_zip}'."
+                puts "  - and files into '#{go_files}.zip'." if !go_files.nil?
+                if !warnings_msg.empty? 
+                    puts "WARNINGS: "
+                    puts warnings_msg
+                end
             end
         else
             puts "ERROR: Source could not be converted due to:"

data/lib/go_import.rb

@@ -16,4 +16,5 @@ module GoImport
     require 'go_import/excel_helper'
     require 'go_import/templating'
     require 'go_import/source'
+    require 'go_import/shard_helper'
 end

data/lib/go_import/shard_helper.rb

@@ -0,0 +1,96 @@
+module GoImport
+    class ShardHelper
+            
+            attr_accessor :shards, :current_shard_count, :current_shard
+            
+            def initialize(shard_size = nil)
+                @shard_size = shard_size || 25000
+                setup()
+            end
+
+            def shard_model(model)
+                @current_shard.configuration = model.configuration
+
+                model.coworkers.each{ |key, coworker| 
+                    if coworker.integration_id != "import"
+                        add_coworker(coworker)
+                    end
+                }
+                model.organizations.each{|key, org| add_organization(org)}
+                model.deals.each{|key, deal| add_deal(deal)}
+                model.notes.each{|key, note| add_note(note)}
+                add_documents(model.documents)
+
+                return_value = @shards
+                setup()
+                return return_value
+            end
+
+            private
+            def setup()
+                @current_shard = GoImport::RootModel.new
+                @shards = [@current_shard]
+                @current_shard_count = 0
+            end
+
+            private
+            def add_note(note)
+                check_or_create_new_chard()
+                @current_shard.add_note(note)
+                @current_shard_count += 1
+            end
+
+            private
+            def add_deal(deal)
+                check_or_create_new_chard()
+                @current_shard.add_deal(deal)
+                @current_shard_count += 1
+            end
+
+            private
+            def add_coworker(coworker)
+                check_or_create_new_chard()
+                @current_shard.add_coworker(coworker)
+                @current_shard_count += 1
+            end
+
+            private
+            def add_organization(org)
+                check_or_create_new_chard()
+                if org.employees != nil
+                    @current_shard_count += org.employees.length
+                end
+                @current_shard.add_organization(org)
+                @current_shard_count += 1 
+            end
+
+            private
+            def add_documents(doc)
+                    doc.files.each{|file| add_file(file)}
+                    doc.links.each{|link| add_link(link)}
+            end
+
+            private
+            def add_file(file)
+                check_or_create_new_chard()
+                @current_shard.add_file(file)
+                @current_shard_count += 1 
+            end
+
+            private
+            def add_link(link)
+                check_or_create_new_chard()
+                @current_shard.add_link(link)
+                @current_shard_count += 1 
+            end
+            
+            private
+            def check_or_create_new_chard()
+                if @current_shard_count > @shard_size
+                    @current_shard = GoImport::RootModel.new
+                    @shards.push(@current_shard)
+                    @current_shard_count = 0
+                end
+            end
+    end
+end

data/sources/VISMA/.go_import/runner.rb

@@ -39,6 +39,7 @@ def convert_source
             if not row["NAMN"] == ""
                 organization = converter.to_organization(row, rootmodel)
                 rootmodel.add_organization(organization)
+                converter.organization_hook(row, organization, rootmodel) if defined? converter.organization_hook
             end
         end
     end

data/sources/csv/.go_import/runner.rb

@@ -4,8 +4,8 @@ require_relative("../converter")
 
 # COWORKER_FILE and other file names should be defined ../converter.rb
 
-def process_rows(file_name)
-    data = File.open(file_name, 'r').read.encode('UTF-8',"ISO-8859-1")
+def process_rows(file_name, source_encoding)
+    data = File.read(file_name, :encoding => source_encoding)
     rows = GoImport::CsvHelper::text_to_hashes(data)
     rows.each do |row|
         yield row
@@ -22,28 +22,30 @@ def convert_source
     rootmodel = GoImport::RootModel.new
 
     converter.configure(rootmodel)
+    source_encoding = defined?(SOURCE_ENCODING) ? SOURCE_ENCODING : 'ISO-8859-1'
 
     # coworkers
     # start with these since they are referenced
     # from everywhere....
     if defined?(COWORKER_FILE) && !COWORKER_FILE.nil? && !COWORKER_FILE.empty?
-        process_rows COWORKER_FILE do |row|
+        process_rows(COWORKER_FILE, source_encoding) do |row|
             rootmodel.add_coworker(converter.to_coworker(row))
         end
     end
 
     # organizations
     if defined?(ORGANIZATION_FILE) && !ORGANIZATION_FILE.nil? && !ORGANIZATION_FILE.empty?
-        process_rows ORGANIZATION_FILE do |row|
+        process_rows(ORGANIZATION_FILE, source_encoding) do |row|
             organization = converter.to_organization(row, rootmodel)
             rootmodel.add_organization(organization)
+            converter.organization_hook(row, organization, rootmodel) if defined? converter.organization_hook
         end
     end
 
     # persons
     # depends on organizations
     if defined?(PERSON_FILE) && !PERSON_FILE.nil? && !PERSON_FILE.empty?
-        process_rows PERSON_FILE do |row|
+        process_rows(PERSON_FILE, source_encoding) do |row|
             # adds it self to the employer
             converter.to_person(row, rootmodel)
         end
@@ -53,7 +55,7 @@ def convert_source
     # deals can reference coworkers (responsible), organizations
     # and persons (contact)
     if defined?(DEAL_FILE) && !DEAL_FILE.nil? && !DEAL_FILE.empty?
-        process_rows DEAL_FILE do |row|
+        process_rows(DEAL_FILE, source_encoding) do |row|
             rootmodel.add_deal(converter.to_deal(row, rootmodel))
         end
     end

data/sources/csv/converter.rb

@@ -12,9 +12,16 @@ COWORKER_FILE = "data/coworkers.csv"
 ORGANIZATION_FILE = "data/organizations.csv"
 PERSON_FILE = "data/persons.csv"
 DEAL_FILE = "data/deals.csv"
-
 # Ie if you dont want to import deals, set DEAL_FILE = ""
 
+# Default encoding for files are 'ISO-8859-1' (aka latin1)
+# If the file is in any other encoding this must be specified with 
+# SOURCE_ENCODING set to correct value
+# Ruby can handle lots of encodings, but common ones are:
+# SOURCE_ENCODING = "UTF-8"
+# SOURCE_ENCODING = "bom|UTF-8"
+# SOURCE_ENCODING = "UTF-16"
+
 # If you are importing files then you must set the FILES_FOLDER
 # constant. FILES_FOLDER should point to the folder where the files
 # are stored. FILES_FOLDER can be relative to the project directory

data/sources/excel-basic/.go_import/runner.rb

@@ -35,6 +35,7 @@ def convert_source
     if defined?(ORGANIZATION_SHEET)
         if excel_workbook.has_sheet?(ORGANIZATION_SHEET)
             organization_rows = excel_workbook.rows_for_sheet ORGANIZATION_SHEET
+            converter.organization_hook(row, organization, rootmodel) if defined? converter.organization_hook
         else
             puts "WARNING: can't find sheet '#{ORGANIZATION_SHEET}'"
         end

data/sources/excel-basic/converter.rb

@@ -30,10 +30,17 @@ class Converter
         # are :String and :Link. If no type is specified :String is
         # used as default.
 
+        # Organizastion
         # rootmodel.settings.with_organization do |organization|
         #     organization.set_custom_field( { :integration_id => 'source', :title => 'Källa', :type => :Link } )
         # end
-    end
+
+        #Person
+        #rootmodel.settings.with_person  do |person|
+            # person.set_custom_field( { :integration_id => 'shoe_size', :title => 'Shoe Size', :type => :String} )
+        #end
+
+        end
 
     def import_person_to_organization(row, rootmodel)
         person = to_person(row, rootmodel)

data/sources/excel/.go_import/runner.rb

@@ -36,6 +36,7 @@ def convert_source
     if defined?(ORGANIZATION_SHEET)
         if excel_workbook.has_sheet?(ORGANIZATION_SHEET)
             organization_rows = excel_workbook.rows_for_sheet ORGANIZATION_SHEET
+            converter.organization_hook(row, organization, rootmodel) if defined? converter.organization_hook
         else
             puts "WARNING: can't find sheet '#{ORGANIZATION_SHEET}'"
         end

data/sources/excel/converter.rb

@@ -51,6 +51,17 @@ class Converter
         # rootmodel.settings.with_organization do |organization|
         #     organization.set_custom_field( { :integration_id => 'source', :title => 'Källa', :type => :Link } )
         # end
+
+        # rootmodel.settings.with_person  do |person|
+        #     person.set_custom_field( { :integration_id => 'shoe_size', :title => 'Shoe size', :type => :String} )
+        # end
+
+        # rootmodel.settings.with_deal do |deal|
+        # assessment is default DealState::NoEndState
+        #     deal.add_status( {:label => '1. Kvalificering' })
+        #     deal.add_status( {:label => '2. Deal closed', :assessment => GoImport::DealState::PositiveEndState })
+        #     deal.add_status( {:label => '4. Deal lost', :assessment => GoImport::DealState::NegativeEndState })
+        # end
     end
 
     def import_person_to_organization(row, rootmodel)

data/sources/lime-easy/converter.rb

@@ -67,14 +67,39 @@ FILES_FOLDER_AT_CUSTOMER = "m:\\documents\\"
 class Converter
     # Reads a row from the Easy exported Company.txt
     # and ads custom fields to the go_import organization.
-
-    # NOTE!!! You should customize this method to include
+        # NOTE!!! You should customize this method to include
     # and transform the fields you want to import to LIME Go.
     # The method includes examples of different types of
     # fields and how you should handle them.
     # Sometimes it's enough to uncomment some code and
     # change the row name but in most cases you need to
     # do some thinking of your own.
+
+    def configure(rootmodel)
+        #####################################################################
+        ## LIME Go custom fields.
+        # This is how you add a custom field in LIME Go.
+        # Custom fields can be added to organization, deal and person.
+        # Valid types are :String and :Link. If no type is specified
+        # :String is used as default.
+
+        #rootmodel.settings.with_organization do |organization|
+            #organization.set_custom_field( { :integration_id => 'building_size', :title => 'Building Size', :type => :String } )
+        #end
+
+        # rootmodel.settings.with_person  do |person|
+        #     person.set_custom_field( { :integration_id => 'shoe_size', :title => 'Shoe size', :type => :String} )
+        # end
+
+        # rootmodel.settings.with_deal do |deal|
+            # assessment is default DealState::NoEndState
+        #     deal.add_status( {:label => '1. Kvalificering' })
+        #     deal.add_status( {:label => '2. Deal closed', :assessment => GoImport::DealState::PositiveEndState })
+        #     deal.add_status( {:label => '4. Deal lost', :assessment => GoImport::DealState::NegativeEndState })
+        # end
+    end
+
+
     def to_organization(organization, row)
         # Here are some standard fields that are present
         # on a LIME Go organization and are usually represented
@@ -315,25 +340,7 @@ class Converter
         # return classification
     end
 
-    def configure(rootmodel)
-        #####################################################################
-        ## LIME Go custom fields.
-        # This is how you add a custom field in LIME Go.
-        # Custom fields can be added to organization, deal and person.
-        # Valid types are :String and :Link. If no type is specified
-        # :String is used as default.
-
-        # rootmodel.settings.with_person  do |person|
-        #     person.set_custom_field( { :integration_id => 'shoe_size', :title => 'Shoe size', :type => :String} )
-        # end
-
-        # rootmodel.settings.with_deal do |deal|
-            # assessment is default DealState::NoEndState
-        #     deal.add_status( {:label => '1. Kvalificering' })
-        #     deal.add_status( {:label => '2. Deal closed', :assessment => GoImport::DealState::PositiveEndState })
-        #     deal.add_status( {:label => '4. Deal lost', :assessment => GoImport::DealState::NegativeEndState })
-        # end
-    end
+    
 
     # HOOKS
     #

data/sources/lime-pro-basic/.go_import/runner.rb

@@ -108,6 +108,7 @@ def convert_source
     con.fetch_data "company" do |row|
         organization = init_organization(row, con.get_class_by_name('company'), rootmodel)
         rootmodel.add_organization(converter.to_organization(organization, row))
+        converter.organization_hook(row, organization, rootmodel) if defined? converter.organization_hook
     end
    
     # persons

data/sources/salesforce/.go_import/runner.rb

@@ -115,6 +115,8 @@ def account_to_organization(row, rootmodel)
     
     organization.responsible_coworker =
         rootmodel.find_coworker_by_integration_id(row['OwnerId'])
+   
+    converter.organization_hook(row, organization, rootmodel) if defined? converter.organization_hook
 
     return organization
 end

data/spec/helpers/shard_helper_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+require 'go_import'
+
+describe GoImport::ShardHelper do
+    it "should shard 50 objects of a single type into two shards" do
+        # given
+        model =  GoImport::RootModel.new
+
+        (1..50).each do |n|
+            organization = GoImport::Organization.new
+            organization.name = "Ankeborgs bibliotek"
+            organization.integration_id = n.to_s
+            model.add_organization(organization)
+        end
+
+        sharder = GoImport::ShardHelper.new(25)
+
+        # when, the
+        sharder.shard_model(model).length.should eq 2
+    end
+
+    it "should shard 60 objects of different type into three shards" do
+        # given
+        model =  GoImport::RootModel.new
+
+        (1..10).each do |n|
+            organization = GoImport::Organization.new
+            organization.name = "Ankeborgs bibliotek"
+            organization.integration_id = n.to_s
+            
+            person = GoImport::Person.new
+            person.first_name = "Kalle"
+            person.last_name = "Kula"
+            organization.add_employee(person)
+            
+            person = GoImport::Person.new
+            person.first_name = "Nisse"
+            person.last_name = "Nice"
+            organization.add_employee(person)
+
+            model.add_organization(organization)
+        end
+
+        (1..10).each do |n|
+            deal = GoImport::Deal.new
+            deal.name = "Big deal"
+            deal.integration_id = n.to_s
+            model.add_deal(deal)
+        end
+
+        (1..10).each do |n|
+            note = GoImport::Note.new
+            note.text = "Important note"
+            model.add_note(note)
+        end
+
+        (1..10).each do |n|
+            link = GoImport::Link.new
+            link.url = "https://go.lime-go.com"
+            link.name = "Our url"
+            model.add_link(link)
+        end
+
+        sharder = GoImport::ShardHelper.new(20)
+
+        # when, then
+        sharder.shard_model(model).length.should eq 3
+    end
+
+    it "should be able to change the shard size" do
+        # given
+        model =  GoImport::RootModel.new
+
+        (1..20).each do |n|
+            organization = GoImport::Organization.new
+            organization.name = "Ankeborgs bibliotek"
+            organization.integration_id = n.to_s
+            model.add_organization(organization)
+        end
+
+        sharder = GoImport::ShardHelper.new(5)
+        
+        # when, then
+        sharder.shard_model(model).length.should eq 4
+    end
+
+    it "should add an organization into a shard" do
+        # given
+
+        model =  GoImport::RootModel.new
+
+        organization = GoImport::Organization.new
+        organization.name = "Ankeborgs bibliotek"
+        organization.integration_id = "123"
+
+        sharder = GoImport::ShardHelper.new()
+        model.add_organization(organization)
+
+        # when, then
+        sharder.shard_model(model)[0].find_organization_by_integration_id("123").should eq organization
+    end
+
+    it "should add a deal into a shard" do
+        # given
+
+        model =  GoImport::RootModel.new
+
+        deal = GoImport::Deal.new
+        deal.name = "Big deal"
+        deal.integration_id = "123"
+
+        sharder = GoImport::ShardHelper.new()
+        model.add_deal(deal)
+
+        # when, then
+        sharder.shard_model(model)[0].find_deal_by_integration_id("123").should eq deal
+    end
+
+end
+
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: go_import
 version: !ruby/object:Gem::Version
-  version: 3.0.39
+  version: 3.0.40
 platform: ruby
 authors:
 - Petter Sandholdt
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-17 00:00:00.000000000 Z
+date: 2016-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iso_country_codes
@@ -211,6 +211,7 @@ files:
 - lib/go_import/phone_helper.rb
 - lib/go_import/roo_helper.rb
 - lib/go_import/serialize_helper.rb
+- lib/go_import/shard_helper.rb
 - lib/go_import/source.rb
 - lib/go_import/templating.rb
 - sources/VISMA/.gitignore
@@ -288,6 +289,7 @@ files:
 - spec/helpers/phone_helper_spec.rb
 - spec/helpers/roo_helper_spec.rb
 - spec/helpers/serialize_helper_spec.rb
+- spec/helpers/shard_helper_spec.rb
 - spec/helpers/xsd_validate_spec.rb
 - spec/link_spec.rb
 - spec/note_spec.rb
@@ -333,6 +335,7 @@ test_files:
 - spec/helpers/phone_helper_spec.rb
 - spec/helpers/roo_helper_spec.rb
 - spec/helpers/serialize_helper_spec.rb
+- spec/helpers/shard_helper_spec.rb
 - spec/helpers/xsd_validate_spec.rb
 - spec/link_spec.rb
 - spec/note_spec.rb