move-to-go 5.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 30c626193cc7bca95c36653b34caaf2bf9e7b593
+  data.tar.gz: 0ffa9010615f179ed02e9c0a51b7f9723eee3cfd
+SHA512:
+  metadata.gz: 6338cf52b8472ffd1d4ccc2a494d2191549b27aa290e8feed5971e9716f7d9eac84662dc87a472c40b7830cbda310b1390209763fdaa03817116c22e11c42119
+  data.tar.gz: d961df52d8fa809abd86d355420da3eaf626b60a04a8bcac207829d911943cae79fe9b5c8eaa2ad01bba18232a60308bb00e03bc46c95a9182e7f2c3bc25ee43

data/bin/move-to-go

@@ -0,0 +1,210 @@
+#!/usr/bin/env ruby
+
+require "thor"
+require_relative '../lib/move-to-go'
+require 'progress'
+
+
+RUNNER_DIR = ".move-to-go"
+
+class MoveToGoCommandLine < Thor
+
+    desc "about", "About move-to-go"
+    def about()
+        puts "move-to-go is an migration tool for LIME Go. It can take virtually any input source and create import data files that LIME Go likes." 
+        puts "move-to-go has some predefined sources that will make it easy for you to migrate your data."
+        puts ""
+    end
+
+    desc "list-sources", "Lists the available sources"
+    def list_sources()
+        puts "The following sources are available:"
+        puts
+
+        sources = MoveToGo::Sources.new(source_path)
+        sources.list().each do |s|
+            puts "\t#{s}"
+        end
+
+        puts "\nCreate a new project with 'move-to-go new <PROJECT> <SOURCE>' with one of these sources."
+        puts "Use 'move-to-go about <SOURCE>' for more information about a specific source."
+    end
+
+    desc "about <SOURCE>", "Prints information about the specifed source"
+    def about(source)
+        sources = MoveToGo::Sources.new(source_path)
+
+        sources.about_source(source)
+    end
+
+    desc "new <PROJECT> <SOURCE>", "Creates a new migration project with a specifed name and source"
+    def new(project, source)
+        sources = MoveToGo::Sources.new(source_path)
+
+        if sources.create_project_from_source(project, source)
+            puts "\nProject '#{project}' created from source '#{source}'."
+            puts "Modify the #{project}/converter.rb script to suit your source."
+            puts "Use 'move-to-go run' from the project directory to create the zip file for LIME Go."
+        end
+    end
+
+    desc "run", "Executes the current project and create a go.zip file with data and files. Existing go.zip will be overwritten, use --output to specify a different filename."
+    option(:output,
+           :desc => "Name of the file where the converted source will be saved. This file should be sent to LIME Go. If the file already exist it will be replaced.",
+           :type => :string,
+           :required => false)
+    option(:ignore_invalid_files,
+           :desc => "Output will be created even if the import contains missing or invalid files",
+           :type => :boolean,
+           :required => false)
+    option(:log_to_file,
+           :desc => "Console output will be redirected to file",
+           :type => :string,
+           :required => false)
+    option(:max_file_size,
+           :desc => "Maximum size in bytes of documents included in zip",
+           :type => :numeric,
+           :required => false)
+    option(:output_documents,
+           :desc => "Name of the file to put imported documents in (default in same as --output)",
+           :type => :string,
+           :required => false)
+    option(:shard_size,
+           :desc => "Large imports are sharded into several zip-files. This property sets how many objects each zip-file should contain. Default is 25 000",
+           :type => :numeric,
+           :required => false)
+    def run_import()
+        if !options.log_to_file.nil?
+            $stdout = File.new(options.log_to_file == "log_to_file" ? "move-to-go.log" : options.log_to_file, 'w')
+            $stdout.sync = true
+        end
+        max_file_size = options.max_file_size.nil? ? MoveToGo::File::DEFAULT_MAX_FILE_SIZE : options.max_file_size
+
+        if !is_valid_project?
+            return
+        end
+
+        runner_file = ::File.expand_path("./#{RUNNER_DIR}/runner.rb", Dir.pwd)
+        require(runner_file)
+        model = convert_source()
+
+        if model.documents.files.length > 0 && (!defined?(FILES_FOLDER) || FILES_FOLDER.empty?())
+            puts "WARNING: It looks like you are importing files but FILES_FOLDER has not been set in your converter.rb."
+            puts "WARNING: FILES_FOLDER should be set unless you are only importing files with absolute paths."
+        end
+
+        if model.documents.files.length > 0 && (!defined?(FILES_FOLDER_AT_CUSTOMER) || FILES_FOLDER_AT_CUSTOMER.empty?())
+            puts "WARNING: It looks like you are importing files but FILES_FOLDER_AT_CUSTOMER has not been set in your converter.rb"
+            puts "WARNING: This means that files with an absolute path will be imported with their original path. Set this constant if you want to get files from the FILES_FOLDER directory."
+        end
+
+        is_ok, error_msg, warnings_msg = can_be_serialized?(model, options.ignore_invalid_files, max_file_size)
+        if is_ok
+
+            if options.ignore_invalid_files && model.documents.files.length > 0
+                log_and_remove_invalid_files model, max_file_size
+            end
+            model.report_rootmodel_status()
+
+            puts "Starting sharding of model..."
+            sharder = MoveToGo::ShardHelper.new(options.shard_size)
+            models_to_serialize = sharder.shard_model(model)
+
+            if models_to_serialize.length > 1
+                puts "Import is large and will be broken into #{models_to_serialize.length} files"
+            end
+
+            models_to_serialize.each_with_index do |model, i|
+                go_data_zip = options.output.nil? == true ? "go" : options.output
+                go_data_zip += "_#{i}.zip"
+                go_files = options.output_documents.nil? == true ? nil : ::File.basename(options.output_documents,File.extname(options.output_documents))
+                model.save_to_zip(go_data_zip, go_files)
+                puts "Source has been been converted into '#{go_data_zip}'."
+                puts "  - and files into '#{go_files}.zip'." if !go_files.nil?
+                if !warnings_msg.empty?
+                    puts "WARNINGS: "
+                    puts warnings_msg
+                end
+            end
+        else
+            puts "ERROR: Source could not be converted due to:"
+            puts error_msg
+
+            if !options.ignore_invalid_files &&
+                    model.documents.files.any? {|file| file.validate.empty?}
+                puts "move-to-go detected invalid files (see above), you can ignore these with the option --ignore-invalid-files."
+            end
+        end
+    end
+
+    private
+    def log_and_remove_invalid_files(model, max_file_size)
+        if model.documents.files.length > 0
+            file_log_header = "name;integration_id;path;organization.integrationid;organization.name;deal.integrationid;deal.name;file.size"
+            file_log = ""
+            files_to_remove = []
+            model.documents.files.with_progress(" - Trying to log files that can't be found...").each do |file|
+                if !::File.exists?(file.path_for_project)
+                    file_log = "#{file_log}#{file.name};#{file.integration_id};#{file.path};#{file.organization.nil? ? '' : file.organization.integration_id};#{file.organization.nil? ? '' : file.organization.name};#{file.deal.nil? ? '' : file.deal.integration_id};#{file.deal.nil? ? '' : file.deal.name};0\n"
+                    files_to_remove.push file
+                elsif ::File.size(file.path_for_project) > max_file_size
+                    file_log = "#{file_log}#{file.name};#{file.integration_id};#{file.path};#{file.organization.nil? ? '' : file.organization.integration_id};#{file.organization.nil? ? '' : file.organization.name};#{file.deal.nil? ? '' : file.deal.integration_id};#{file.deal.nil? ? '' : file.deal.name};#{::File.size(file.path_for_project)}\n"
+                    files_to_remove.push file
+                end
+            end
+
+            files_to_remove.each do |file|
+                model.documents.files.delete file
+            end
+
+            if file_log.length > 0
+                log_filename = 'move-to-go-invalid-files.csv'
+                ::File.open(log_filename, 'w') { |f|
+                    f.puts file_log_header
+                    f.puts file_log
+                }
+                puts "WARNING: move-to-go has invalid files (#{files_to_remove.length} of #{model.documents.files.length}). Filenames of all ignored files has been written to '#{log_filename}'."
+            else
+                puts "All files are OK."
+            end
+        end
+    end
+
+    private
+    def can_be_serialized?(rootmodel, ignore_invalid_files, max_file_size)
+        is_ok = false
+        error = rootmodel.sanity_check
+        if error.empty?
+            error, warnings = rootmodel.validate(ignore_invalid_files, max_file_size)
+
+            if error.empty?
+                is_ok = true
+            end
+        end
+
+        return [is_ok, error, warnings]
+    end
+
+    private
+    def is_valid_project?()
+        if Dir.exists?(RUNNER_DIR) == false
+            puts "This doesnt look like a move-to-go project. Are you in the right directory or did you mess with the '#{RUNNER_DIR}' folder?"
+            return false
+        end
+
+        runner_file = File.expand_path("./#{RUNNER_DIR}/runner.rb", Dir.pwd)
+        if File.exists?(runner_file) == false
+            puts "I can't run this project. Did you mess with the '#{RUNNER_DIR}' folder?"
+            return false
+        end
+
+        return true
+    end
+
+    private
+    def source_path()
+        File.expand_path("../sources", File.dirname(__FILE__))
+    end
+end
+
+MoveToGoCommandLine.start(ARGV)

data/lib/move-to-go/can_become_immutable.rb

@@ -0,0 +1,29 @@
+
+module MoveToGo
+    class CanBecomeImmutable
+        def self.immutable_accessor(name)
+            define_method(name) do
+                return instance_variable_get("@#{name}")
+            end
+            
+            define_method("#{name}=") do |value|
+                raise_if_immutable
+                instance_variable_set("@#{name}", value)
+            end
+        end
+        
+        def raise_if_immutable
+            if @is_immutable
+                raise ObjectIsImmutableError
+            end
+        end
+        
+        def set_is_immutable()
+            @is_immutable = true
+        end
+
+        def is_immutable()
+            @is_immutable
+        end
+    end
+end

data/lib/move-to-go/csv_helper.rb

@@ -0,0 +1,47 @@
+require "csv"
+module MoveToGo
+    module CsvHelper
+
+        # @example Detect column separator and transform to hashes
+        #     hashes = MoveToGo::CsvHelper.text_to_hashes(text)
+        #
+        # @example Use specific column separator and transform to hashes
+        #     column_separator = ','
+        #     hashes = MoveToGo::CsvHelper.text_to_hashes(text, column_separator)
+        def self.text_to_hashes(text, column_separator = nil, row_separator = :auto, quote_char = '"')
+            if !text
+                raise "Missing text"
+            end
+
+            if !column_separator
+                column_separator = self.detect_col_sep text
+            end
+
+            rows = CSV.parse(text.strip,{:col_sep => column_separator,
+                :row_sep => row_separator, :quote_char => quote_char})
+            map = {}
+            first = rows.first
+            (0 .. first.length-1).each do |i|
+                map[i] = first[i]
+            end
+            rs = []
+            (1 .. rows.length-1).each do |i|
+                r={}
+                (0 .. map.length-1).each do |j|
+                    r[map[j]] = rows[i][j]
+                end
+                rs.push(r)
+            end
+            return rs
+        end
+
+        private
+        def self.detect_col_sep(text)
+            firstline = text.split('\n').first
+            col_seps = [';','\t',',']
+            return col_seps.find do |c|
+                firstline.include? c
+            end
+        end
+    end
+end

data/lib/move-to-go/email_helper.rb

@@ -0,0 +1,14 @@
+require "sixarm_ruby_email_address_validation"
+
+module MoveToGo
+    # The EmailHelper helps you validate email addresses.
+    class EmailHelper
+        def self.is_valid?(email)
+            begin
+                return (email =~ EmailAddressValidation::Pattern) ? true : false
+            rescue
+                return false
+            end
+        end
+    end
+end

data/lib/move-to-go/errors.rb

@@ -0,0 +1,31 @@
+module MoveToGo
+    class AlreadyAddedError < StandardError
+    end
+
+    class IntegrationIdIsRequiredError < StandardError
+    end
+
+    class InvalidCustomFieldError < StandardError
+    end
+
+    class InvalidRelationError < StandardError
+    end
+
+    class InvalidValueError < StandardError
+        def initalize(value)
+            super("#{value} is not a valid value.")
+        end
+    end
+
+    class InvalidDealStatusError < StandardError
+    end
+
+    class InvalidHistoryClassificationError < StandardError
+        def initalize(classification)
+            super("#{classification} is not a valid history classification")
+        end
+    end
+
+    class ObjectIsImmutableError < StandardError
+    end
+end

data/lib/move-to-go/excel_helper.rb

@@ -0,0 +1,10 @@
+module MoveToGo
+    # The ExcelHelper just makes it a little bit easier to open an
+    # excel file in the imports. With ExcelHelper you don't need to
+    # know anything about Roo and RooHelper.
+    class ExcelHelper
+        def self.Open(excel_filename)
+            return RooHelper.new(Roo::Excelx.new(excel_filename))
+        end
+    end
+end