go_import 3.0.26 → 3.0.27

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZjBiMjk3NmIxNjgwZDNhYjI2M2M2MWEwYjUxMzgyODQ2MmNmNmE4NQ==
+    NGUwNDcxNGFmZDlkZjExNTJkZDY3MjBiMjI2NGNhNTZmNGM5Y2I0ZQ==
   data.tar.gz: !binary |-
-    NTRkOGI4YTZiYjY5YzI5OTk0ZThlOWY4ZWY3NzM3YTBlNzMwMDk1MQ==
+    M2I0YmRkOWY2NmY0NmVkN2UwYmZjMDg2NmJlNjQ1ODJmNGNlYzU5NQ==
 SHA512:
   metadata.gz: !binary |-
-    M2JjMjNhNTc4ZTI0ODc2YTRjODU5YzFmODA2NDEzM2ZjZDI2YjE2ZTgwNGM5
-    Yjk3NjBhN2IxN2NhMzFhNzEzOGY0NTUxZWUzNjMxMmIxMzc3YWZiMDAwOTI3
-    M2E0Y2U3MzI0OGUxMWExNGI0ZTg2YTRkNDdiOGQzMzUwNjRkNGE=
+    Y2I1NzUxOGIxZDAyZTE4MWJkYjQ4YjBhZmViYTJmODVjNjhmMzFkMmQwMWE2
+    NWFmMDllMjJjZTk1ZTA4MDBkM2Y0NWU1NzI2NjU4OWIzNGQ3OGNmZTM1M2My
+    MWU4MWEzYTk4MjE4MWFiZWIyZTBiMTMzZmNhZDgyMmM3NWYxYWI=
   data.tar.gz: !binary |-
-    MDE0MmJmNjliY2Y4MjBkNDk3MTNhNGNjMjljZTMyNmE5YTNiOGNjMzEyMjNk
-    NzdiZWFkZjhiZjQ0ZTYzODRhNmIzM2YwZTYwOWRhM2ZhNzA3NGMxMmI0OTgw
-    MDViZGJhODNiYjJkYTdmMmRmOTVkY2YyYzE0OWYyNzQzZGZiZDQ=
+    NGI5NjdmOWZmYTA0ODM2Njk5MDkwM2NiZTQ3ZmNiNjRjNzIxYjA0ZjhiNGNh
+    MjM1OGMxYWRmNWJmMGQ5Yjk5NTY4MTIyOGI3NzQ4ZThhMzg0ODdhN2Q2NmU1
+    MmNjYzcwZTY2OWY2MDFmYzJjZjEwOTY0ODJhZjkxMDU3NzlmZGY=

data/bin/go-import

@@ -2,6 +2,7 @@
 
 require "thor"
 require "go_import"
+require 'progress'
 
 RUNNER_DIR = ".go_import"
 
@@ -58,17 +59,26 @@ class GoImportCommandLine < Thor
            :desc => "Console output will be redirected to file",
            :type => :string,
            :required => false)
+    option(:max_file_size,
+           :desc => "Maximum size in bytes of documents included in zip",
+           :type => :numeric,
+           :required => false)
+    option(:output_documents,
+           :desc => "Name of the file to put imported documents in (default in same as --output)",
+           :type => :string,
+           :required => false)
     def run_import()
         if !options.log_to_file.nil?
             $stdout = File.new(options.log_to_file == "log_to_file" ? "go-import.log" : options.log_to_file, 'w')
             $stdout.sync = true
         end
+        max_file_size = options.max_file_size.nil? ? GoImport::File::DEFAULT_MAX_FILE_SIZE : options.max_file_size 
 
         if !is_valid_goimport_project?
             return
         end
 
-        runner_file = File.expand_path("./#{RUNNER_DIR}/runner.rb", Dir.pwd)
+        runner_file = ::File.expand_path("./#{RUNNER_DIR}/runner.rb", Dir.pwd)
         require(runner_file)
         model = convert_source()
 
@@ -82,15 +92,18 @@ class GoImportCommandLine < Thor
             puts "WARNING: This means that files with an absolute path will be imported with their original path. Set this constant if you want to get files from the FILES_FOLDER directory."
         end
 
-        is_ok, error_msg, warnings_msg = can_be_serialized?(model, options.ignore_invalid_files)
+        is_ok, error_msg, warnings_msg = can_be_serialized?(model, options.ignore_invalid_files, max_file_size)
         if is_ok
+
             if options.ignore_invalid_files && model.documents.files.length > 0
-                log_and_remove_invalid_files model
+                log_and_remove_invalid_files model, max_file_size
             end
 
             go_data_zip = options.output.nil? == true ? "go.zip" : options.output
-            model.save_to_zip(go_data_zip)
+            go_files = options.output_documents.nil? == true ? nil : ::File.basename(options.output_documents,File.extname(options.output_documents))
+            model.save_to_zip(go_data_zip, go_files)
             puts "Source has been been converted into '#{go_data_zip}'."
+            puts "  - and files into '#{go_files}.zip'." if !go_files.nil?
             if !warnings_msg.empty? 
                 puts "WARNINGS: "
                 puts warnings_msg
@@ -107,17 +120,16 @@ class GoImportCommandLine < Thor
     end
 
     private
-    def log_and_remove_invalid_files(model)
+    def log_and_remove_invalid_files(model, max_file_size)
         if model.documents.files.length > 0
-            puts "Trying to log files that can't be found..."
             file_log_header = "name;integration_id;path;organization.integrationid;organization.name;deal.integrationid;deal.name;file.size"
             file_log = ""
             files_to_remove = []
-            model.documents.files.each do |file|
+            model.documents.files.with_progress(" - Trying to log files that can't be found...").each do |file|
                 if !::File.exists?(file.path_for_project)
                     file_log = "#{file_log}#{file.name};#{file.integration_id};#{file.path};#{file.organization.nil? ? '' : file.organization.integration_id};#{file.organization.nil? ? '' : file.organization.name};#{file.deal.nil? ? '' : file.deal.integration_id};#{file.deal.nil? ? '' : file.deal.name};0\n"
                     files_to_remove.push file
-                elsif ::File.size(file.path_for_project) > GoImport::File::MAX_FILE_SIZE
+                elsif ::File.size(file.path_for_project) > max_file_size
                     file_log = "#{file_log}#{file.name};#{file.integration_id};#{file.path};#{file.organization.nil? ? '' : file.organization.integration_id};#{file.organization.nil? ? '' : file.organization.name};#{file.deal.nil? ? '' : file.deal.integration_id};#{file.deal.nil? ? '' : file.deal.name};#{::File.size(file.path_for_project)}\n"
                     files_to_remove.push file
                 end
@@ -133,7 +145,7 @@ class GoImportCommandLine < Thor
                     f.puts file_log_header
                     f.puts file_log
                 }
-                puts "WARNING: go-import has invalid files. Filenames of all ignored files has been written to '#{log_filename}'."
+                puts "WARNING: go-import has invalid files (#{files_to_remove.length} of #{model.documents.files.length}). Filenames of all ignored files has been written to '#{log_filename}'."
             else
                 puts "All files are OK."
             end
@@ -141,11 +153,11 @@ class GoImportCommandLine < Thor
     end
 
     private
-    def can_be_serialized?(rootmodel, ignore_invalid_files)
+    def can_be_serialized?(rootmodel, ignore_invalid_files, max_file_size)
         is_ok = false
         error = rootmodel.sanity_check
         if error.empty?
-            error, warnings = rootmodel.validate(ignore_invalid_files)
+            error, warnings = rootmodel.validate(ignore_invalid_files, max_file_size)
 
             if error.empty?
                 is_ok = true

data/lib/go_import/model/file.rb

@@ -6,7 +6,7 @@ require_relative '../serialize_helper'
 
 module GoImport
     class File
-        MAX_FILE_SIZE = 100000000 # 100 Mb
+        DEFAULT_MAX_FILE_SIZE = 100000000 # 100 Mb
 
         include SerializeHelper
         attr_accessor :id, :integration_id, :description
@@ -62,6 +62,8 @@ module GoImport
             if (@name.nil? || @name.empty?) && (!@path.nil? && !@path.empty?)
                 @name = Pathname.new(path).basename.to_s
             end
+            @location_in_zip_file = "files/#{SecureRandom.uuid}#{::File.extname(@path).to_s}"
+
         end
 
         def name=(name)
@@ -152,12 +154,10 @@ module GoImport
         end
 
         def add_to_zip_file(zip_file)
-            @location_in_zip_file = "files/#{SecureRandom.uuid}#{::File.extname(@path).to_s}"
-
             zip_file.add(@location_in_zip_file, path_for_project)
         end
 
-        def validate(ignore_invalid_files = false)
+        def validate(ignore_invalid_files = false, max_file_size = DEFAULT_MAX_FILE_SIZE)
             error = String.new
             warning = String.new
 
@@ -167,15 +167,14 @@ module GoImport
 
             if @path.nil? || @path.empty?
                 error = "Path is required for file.\n"
-            elsif !ignore_invalid_files 
+            elsif !ignore_invalid_files
                 if !::File.exists?(path_for_project())
                     error = "#{error}Can't find file with name '#{@name}' and original path '#{@path}' at '#{path_for_project()}'."
-                elsif ::File.size(path_for_project()) > MAX_FILE_SIZE
-                    error = "#{error}File '#{@name}' is bigger than #{MAX_FILE_SIZE} bytes."
+                elsif ::File.exists?(path_for_project) && ::File.size(path_for_project()) > max_file_size
+                    error = "#{error}File '#{@name}' is bigger than #{max_file_size} bytes."
                 end
             end
 
-
             if @created_by_reference.nil?
                 error = "#{error}Created_by is required for file (#{@name}).\n"
             end

data/lib/go_import/model/rootmodel.rb

@@ -2,6 +2,7 @@
 
 require 'zip'
 require 'securerandom'
+require "progress"
 
 module GoImport
     # The root model for Go import. This class is the container for everything else.
@@ -310,7 +311,7 @@ module GoImport
             return error.strip
         end
 
-        def validate(ignore_invalid_files = false)
+        def validate(ignore_invalid_files = false, max_file_size)
             errors = String.new
             warnings = String.new
 
@@ -322,8 +323,9 @@ module GoImport
                 end
             end
 
+            converter_deal_statuses = @settings.deal.statuses.map {|status| status.label} if @settings.deal != nil
             @deals.each do |deal|
-                error, warning = deal.validate @settings.deal.statuses.map {|status| status.label}
+                error, warning = deal.validate converter_deal_statuses
 
                 if !error.empty?
                     errors = "#{errors}\n#{error}"
@@ -349,7 +351,7 @@ module GoImport
             end
 
             @documents.files.each do |file|
-                validation_message = file.validate(ignore_invalid_files)
+                validation_message = file.validate(ignore_invalid_files, max_file_size)
                 if !validation_message.empty?
                     errors = "#{errors}\n#{validation_message}"
                 end
@@ -367,8 +369,8 @@ module GoImport
 
         # @!visibility private
         # zip-filename is the name of the zip file to create
-        def save_to_zip(zip_filename)
-            puts "Trying to save to '#{zip_filename}'..."
+        def save_to_zip(zip_filename, files_filename)
+            puts "Trying to save to zip..."
             # saves the model to a zipfile that contains xml data and
             # document files.
 
@@ -376,47 +378,70 @@ module GoImport
                 ::File.delete zip_filename
             end
 
-            Zip::File.open(zip_filename, Zip::File::CREATE) do |zip_file|
-                puts "Trying to add files to zip..."
-                # We must add files first to the zip file since we
-                # will set each file's location_in_zip_file when the
-                # zip file is created.
-
-                if defined?(FILES_FOLDER) && !FILES_FOLDER.empty?()
-                    puts "Files with relative path are imported from '#{FILES_FOLDER}'."
-                    root_folder = FILES_FOLDER
-                else
-                    puts "Files with relative path are imported from the current folder (#{Dir.pwd})."
-                    root_folder = Dir.pwd
-                end
-
-                # If a file's path is absolute, then we probably dont
-                # have the files in the same location here. For
-                # example, the customer might have stored their files
-                # at f:\lime-easy\documents. We must replace this part
-                # of each file with the root_folder from above.
-                if defined?(FILES_FOLDER_AT_CUSTOMER) && !FILES_FOLDER_AT_CUSTOMER.empty?()
-                    files_folder_at_customer = FILES_FOLDER_AT_CUSTOMER
-                    puts "Files with absolute paths will have the part '#{files_folder_at_customer}' replaced with '#{root_folder}'."
-                else
-                    files_folder_at_customer = ""
-                    puts "Files with absolute paths will be imported from their origial location."
+            go_data_file = Tempfile.new('go')
+            puts "Creating go.xml file with data..."
+            if !files_filename.nil?
+                saved_documents = @documents
+                @documents = Documents.new
+            end
+            serialize_to_file(go_data_file)
+            create_zip(zip_filename, go_data_file, documents.files)
+            
+            if !files_filename.nil?
+                go_files_file = Tempfile.new('go-files')
+                puts "Creating go.xml file with documents information..."
+                @organizations = []
+                @coworkers = []
+                @deals = []
+                @notes = []
+                @documents = saved_documents
+                serialize_to_file(go_files_file)
+
+                files_zip_filename = files_filename+".zip"
+                if ::File.exists?(files_zip_filename)
+                    ::File.delete files_zip_filename
                 end
+                create_zip(files_zip_filename, go_files_file, documents.files)
+            end            
+        end
 
-                # 1) files/ - a folder with all files referenced from
-                # the source.
-                documents.files.each do |file|
-                    # we dont need to check that the file exists since
-                    # we assume that rootmodel.validate has been
-                    # called before save_to_zip.
-                    file.add_to_zip_file(zip_file)
+        def create_zip(filename, xml, files)
+            Zip::File.open(filename, Zip::File::CREATE) do |zip_file|
+                puts "Add go.xml file to zip '#{filename}'..."
+                zip_file.add('go.xml', xml)
+
+                if files.length > 0
+                    if defined?(FILES_FOLDER) && !FILES_FOLDER.empty?()
+                        puts "Files with relative path are imported from '#{FILES_FOLDER}'."
+                        root_folder = FILES_FOLDER
+                    else
+                        puts "Files with relative path are imported from the current folder (#{Dir.pwd})."
+                        root_folder = Dir.pwd
+                    end
+
+                    # If a file's path is absolute, then we probably dont
+                    # have the files in the same location here. For
+                    # example, the customer might have stored their files
+                    # at f:\lime-easy\documents. We must replace this part
+                    # of each file with the root_folder from above.
+                    if defined?(FILES_FOLDER_AT_CUSTOMER) && !FILES_FOLDER_AT_CUSTOMER.empty?()
+                        files_folder_at_customer = FILES_FOLDER_AT_CUSTOMER
+                        puts "Files with absolute paths will have the part '#{files_folder_at_customer}' replaced with '#{root_folder}'."
+                    else
+                        files_folder_at_customer = ""
+                        puts "Files with absolute paths will be imported from their origial location."
+                    end
+
+                    # 1) files/ - a folder with all files referenced from
+                    # the source.
+                    files.with_progress(" - Trying to add files to zip...").each do |file|
+                        # we dont need to check that the file exists since
+                        # we assume that rootmodel.validate has been
+                        # called before save_to_zip.
+                        file.add_to_zip_file(zip_file)
+                    end
                 end
-
-                # 2) go.xml - with all data from source
-                puts "Trying to add organizations, persons, etc to zip..."
-                go_data_file = Tempfile.new('go')
-                serialize_to_file(go_data_file)
-                zip_file.add('go.xml', go_data_file)
+                puts "Compressing zip file ... "
             end
         end
 

data/sources/VISMA/converter.rb

@@ -116,4 +116,20 @@ class Converter
 
         return person
     end
+
+    # HOOKS
+    #
+    # Sometimes you need to add exra information to the rootmodel, this can be done
+    # with hooks, below is an example of an organization hook that adds a note to
+    # an organization if a field has a specific value
+    #def organization_hook(row, organization, rootmodel)
+    #    if not row['fieldname'].empty?
+    #        note = GoImport::Note.new
+    #        note.text = row['fieldname']
+    #        note.organization = organization
+    #        note.created_by = rootmodel.import_coworker
+    #        rootmodel.add_note(note)
+    #    end
+    #end
+
 end

data/sources/csv/converter.rb

@@ -192,4 +192,20 @@ class Converter
 
         return deal
     end
+
+    # HOOKS
+    #
+    # Sometimes you need to add exra information to the rootmodel, this can be done
+    # with hooks, below is an example of an organization hook that adds a note to
+    # an organization if a field has a specific value
+    #def organization_hook(row, organization, rootmodel)
+    #    if not row['fieldname'].empty?
+    #        note = GoImport::Note.new
+    #        note.text = row['fieldname']
+    #        note.organization = organization
+    #        note.created_by = rootmodel.import_coworker
+    #        rootmodel.add_note(note)
+    #    end
+    #end
+
 end

data/sources/excel/converter.rb

@@ -158,4 +158,20 @@ class Converter
 
         return file
     end
+
+    # HOOKS
+    #
+    # Sometimes you need to add exra information to the rootmodel, this can be done
+    # with hooks, below is an example of an organization hook that adds a note to
+    # an organization if a field has a specific value
+    #def organization_hook(row, organization, rootmodel)
+    #    if not row['fieldname'].empty?
+    #        note = GoImport::Note.new
+    #        note.text = row['fieldname']
+    #        note.organization = organization
+    #        note.created_by = rootmodel.import_coworker
+    #        rootmodel.add_note(note)
+    #    end
+    #end
+
 end

data/sources/lime-easy/.go_import/runner.rb

@@ -1,6 +1,7 @@
 # encoding: UTF-8
 
 require 'go_import'
+require 'progress'
 require_relative("../converter")
 
 EXPORT_FOLDER = 'export'
@@ -34,59 +35,61 @@ def convert_source
     # coworkers
     # start with these since they are referenced
     # from everywhere....
-    process_rows COWORKER_FILE do |row|
+    
+    process_rows(" - Reading Coworkers '#{COWORKER_FILE}'", COWORKER_FILE) do |row|
         rootmodel.add_coworker(to_coworker(row))
     end
 
     # organizations
-    process_rows ORGANIZATION_FILE do |row|
+    process_rows(" - Reading Organizations '#{ORGANIZATION_FILE}'", ORGANIZATION_FILE) do |row|
         organization = init_organization(row, rootmodel)
         rootmodel.add_organization(
             converter.to_organization(organization, row))
+        converter.organization_hook(row, organization, rootmodel) if defined? converter.organization_hook
     end
 
     # persons
     # depends on organizations
-    process_rows PERSON_FILE do |row|
+    process_rows(" - Reading Persons '#{PERSON_FILE}'", PERSON_FILE) do |row|
         # init method also adds the person to the employer
         person = init_person(row, rootmodel)
         converter.to_person(person, row)
     end
 
     # organization notes
-    process_rows ORGANIZATION_NOTE_FILE do |row|
+    process_rows(" - Reading Organization Notes '#{ORGANIZATION_NOTE_FILE}'", ORGANIZATION_NOTE_FILE) do |row|
         # adds itself if applicable
         rootmodel.add_note(to_organization_note(converter, row, rootmodel))
     end
 
     # Organization - Deal connection
     # Reads the includes.txt and creats a hash
-    # that connect organizations to deals
-    process_rows INCLUDE_FILE do |row|
+    # that connect organizations to deals 
+    process_rows(" - Reading Organization Deals '#{INCLUDE_FILE}'", INCLUDE_FILE) do |row|
         includes[row['idProject']] = row['idCompany']
     end
 
     # deals
     # deals can reference coworkers (responsible), organizations
-    # and persons (contact)
-    process_rows DEAL_FILE do |row|
+    # and persons (contact) 
+    process_rows(" - Reading Deals '#{DEAL_FILE}'", DEAL_FILE) do |row|
         deal = init_deal(row, rootmodel, includes)
         rootmodel.add_deal(converter.to_deal(deal, row))
     end
 
     # deal notes
-    process_rows DEAL_NOTE_FILE do |row|
+    process_rows(" - Reading Deal Notess '#{DEAL_NOTE_FILE}'", DEAL_NOTE_FILE) do |row|
         # adds itself if applicable
         rootmodel.add_note(to_deal_note(converter, row, rootmodel))
     end
 
     # documents
     if defined?(IMPORT_DOCUMENTS) && !IMPORT_DOCUMENTS.nil? && IMPORT_DOCUMENTS
-        process_rows ORGANIZATION_DOCUMENT_FILE do |row|
+        process_rows(" - Reading Organization Documents", ORGANIZATION_DOCUMENT_FILE) do |row|
             rootmodel.add_file(to_organization_document(row, rootmodel))
         end
 
-        process_rows PROJECT_DOCUMENT_FILE do |row|
+        process_rows(" - Reading Project Documents", PROJECT_DOCUMENT_FILE) do |row|
             rootmodel.add_file(to_deal_document(row, rootmodel))
         end
     end
@@ -321,13 +324,13 @@ def validate_constants()
 end
 
 
-def process_rows(file_name)
+def process_rows(description, file_name)
     data = File.open(file_name, 'r').read.encode('UTF-8',"ISO-8859-1").strip().gsub('"', '')
     data = '"' + data.gsub("\t", "\"\t\"") + '"'
     data = data.gsub("\n", "\"\n\"")
 
     rows = GoImport::CsvHelper::text_to_hashes(data, "\t", "\n", '"')
-        rows.each do |row|
+    rows.with_progress(description).each do |row|
         yield row
     end
 end

data/sources/lime-easy/converter.rb

@@ -314,7 +314,7 @@ class Converter
         
         # return classification
     end
-    
+
     def configure(rootmodel)
         #####################################################################
         ## LIME Go custom fields.
@@ -334,5 +334,22 @@ class Converter
         #     deal.add_status( {:label => '4. Deal lost', :assessment => GoImport::DealState::NegativeEndState })
         # end
     end
+
+    # HOOKS
+    #
+    # Sometimes you need to add exra information to the rootmodel, this can be done
+    # with hooks, below is an example of an organization hook that adds a note to
+    # an organization if a field has a specific value
+    #def organization_hook(row, organization, rootmodel)
+    #    if not row['fieldname'].empty?
+    #        note = GoImport::Note.new
+    #        note.text = row['fieldname']
+    #        note.organization = organization
+    #        note.created_by = rootmodel.import_coworker
+    #        rootmodel.add_note(note)
+    #    end
+    #end
+    
+
 end
 

data/sources/lime-pro-basic/converter.rb

@@ -376,5 +376,21 @@ class Converter
         #     deal.add_status( {:label => '4. Deal lost', :assessment => GoImport::DealState::NegativeEndState })
         # end
     end
+
+    # HOOKS
+    #
+    # Sometimes you need to add exra information to the rootmodel, this can be done
+    # with hooks, below is an example of an organization hook that adds a note to
+    # an organization if a field has a specific value
+    #def organization_hook(row, organization, rootmodel)
+    #    if not row['fieldname'].empty?
+    #        note = GoImport::Note.new
+    #        note.text = row['fieldname']
+    #        note.organization = organization
+    #        note.created_by = rootmodel.import_coworker
+    #        rootmodel.add_note(note)
+    #    end
+    #end
+
 end
 

data/sources/salesforce/converter.rb

@@ -94,4 +94,20 @@ class Converter
         
         # return nil
     end
+
+    # HOOKS
+    #
+    # Sometimes you need to add exra information to the rootmodel, this can be done
+    # with hooks, below is an example of an organization hook that adds a note to
+    # an organization if a field has a specific value
+    #def organization_hook(row, organization, rootmodel)
+    #    if not row['fieldname'].empty?
+    #        note = GoImport::Note.new
+    #        note.text = row['fieldname']
+    #        note.organization = organization
+    #        note.created_by = rootmodel.import_coworker
+    #        rootmodel.add_note(note)
+    #    end
+    #end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: go_import
 version: !ruby/object:Gem::Version
-  version: 3.0.26
+  version: 3.0.27
 platform: ruby
 authors:
 - Petter Sandholdt
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-06 00:00:00.000000000 Z
+date: 2015-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iso_country_codes