sf_migrate 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+--- 
+SHA1: 
+  metadata.gz: e862896e87413a3c8447dcb85efa73fea0f1c761
+  data.tar.gz: 4ee2a6e915659d3e257cb9f0dc13222835839439
+SHA512: 
+  metadata.gz: 6bc6c9a71e565aad931468ab34c143d700a3113b084d57128dc0f608db0567a0f3915c35ab57e27db14cf25a2587486b630de28a5d500d9e2ce084119f77c187
+  data.tar.gz: 90b2079df5066ae955500f2ab3b2ce04effc6a84362decf2e96d60d021e847800a5c9a8db82a84145fdf46ee970144a11a604e60fd004a1995e87e467e390b0d

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://www.rubygems.org"
+ 
+gemspec

data/README.md

@@ -0,0 +1,2 @@
+Some documentation
+==================

data/bin/sf_migrate

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+# $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+require "pathname"
+bin_file = Pathname.new(__FILE__).realpath
+
+$LOAD_PATH.unshift File.expand_path("../../lib", bin_file)
+require 'sf_migrate'
+
+SalesforceMigration::Runner.start

data/config/credentials.yaml.example

@@ -0,0 +1,13 @@
+sugar_url: <url>
+sugar_username: <username>
+sugar_password: <password>
+
+salesforce_consumer_key: <SF_consumer_key>
+salesforce_consumer_secret: <SF_consumer_secret>
+salesforce_username: <SF_username>
+salesforce_password: <SF_password>
+
+db_type: mysql2
+db_user: <db_user>
+db_password: <db_password>
+db_database: <schema_name>

data/lib/export.rb

@@ -0,0 +1,146 @@
+$LOAD_PATH.unshift File.dirname(__FILE__)
+
+module SalesforceMigration
+  class Export
+    # Initilize migration export object, authenticates to SalesForce and start the export
+    #
+    # @param [Hash] options hash representing passed command line options
+    def initialize(options)
+      credentials     = YAML.load_file(options[:config_file])
+      consumer_key    = credentials['salesforce_consumer_key']
+      consumer_secret = credentials['salesforce_consumer_secret']
+      username        = credentials['salesforce_username']
+      password        = credentials['salesforce_password']
+      @csv_dir        = options[:csv_dir]
+      @action         = options[:action]
+      @logger = SalesforceMigration::Runner::create_logger
+      @logger.info("Export action started")
+      @client = Databasedotcom::Client.new :client_id => consumer_key, :client_secret => consumer_secret
+      @client.authenticate :username => username, :password => password
+      
+      @logger.info("Authentication to SalesForce successfull")
+      start
+      @logger.info("Export action ended successfully")
+    end
+    
+    private
+    def start
+      %w(ISOs__c Agent__c Account Payment_Methods__c Banks__c MerchantToAPM__c ccrmbasic__Email__c Email_Association__c).each do |type|
+        @logger.info "Writing CSV for #{type}"
+        write_to_csv type
+      end
+    end
+    
+    # Generate and save CSV file for given type
+    #
+    # @param [String] type export type name
+    def write_to_csv(type)
+      fields    = return_fields(type)
+      records   = get_records(type)
+      file_name = generate_name(type)
+
+      CSV.open(file_name, "wb:UTF-8") do |file|
+        @logger.info "Opened #{file_name} for export"
+        file << fields
+        records.each do |record|
+          arr = []
+          fields.each do |field|
+            arr << record.send(field)
+            arr.map!(&method(:remove_quotes))
+          end
+          file << arr
+        end
+      end
+    end
+    
+    # In order to fix the Invalid Session Id in SugarCRM we need to remove all
+    # quotes from text fields, because they mess up the JSON object which comunicate with
+    # our SugarCRM instance.
+    # Note that on Mac we don`t need this method.
+    def remove_quotes(field)
+      field.gsub(/'/, "").strip if field.is_a? String
+    end
+    
+
+    # Return the header fields
+    def return_fields(type)
+      return constantize "SalesforceMigration::Fields::#{type.slice(0..-4).upcase}_FIELDS" if type.end_with?("__c")
+      return constantize "SalesforceMigration::Fields::#{type.upcase}_FIELDS"
+    end
+
+    # Get all records for specific type of object
+    #
+    # @param [String] type export type name
+    #
+    # @return [Databasedotcom::Collection] the requested records
+    def get_records(type)
+      if @action == 'initial_run'
+        @logger.info "Getting all records from SalesForce for #{type}"
+        records = get_all_sobjects(type)
+      else
+        @logger.info "Getting records for yesterday from SalesForce for #{type}"
+        records  = @client.materialize(type)
+        
+        datetime = DateTime.now
+        datetime = datetime -= 1
+        records.query("lastmodifieddate >= #{datetime}")
+      end
+    end
+
+    # Generates a name for CSV file depending on the action
+    #
+    # @param [String] type export type name
+    #
+    # @return [String] generated filename
+    #
+    # @example
+    #   @action  = 'update'
+    #   @csv_dir = '/tmp'
+    #   generate_name('Account') #=> '/tmp/update/2012-07-07/Account_export.csv'
+    def generate_name(type)
+      if @action == 'initial_run'
+        FileUtils.mkdir_p("#{@csv_dir}/initial/") unless Dir.exists? "#{@csv_dir}/initial/"
+        "#{@csv_dir}/initial/#{type}_export.csv"
+      else
+        today = lambda { Date.today.to_s }
+        dir = "#{@csv_dir}/update/#{today.call}"
+        FileUtils.mkdir_p(dir) unless Dir.exists? dir
+        "#{dir}/#{type}_export.csv"
+      end
+    end
+
+    # Borrowed from activesupport/lib/active_support/inflector/methods.rb, line 212
+    def constantize(camel_cased_word)
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+      constant = Object
+      names.each do |name|
+        constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+      end
+      constant
+    end
+
+    # Get all objects of given type. Databasedotcom pulls only 250 records by query
+    #
+    # @param [String] type type of sobject
+    #
+    # @return [Array<Databasedotcom::Sobject::Sobject>] All available sobjects
+    # Some objects, like Merchants and Emails are exported VIA a certain criteria
+    def get_all_sobjects(type)
+      case type
+      when 'Account'
+        records = @client.materialize(type).query("Agents__c != ''")
+      when 'ccrmbasic__Email__c'
+        records = @client.materialize(type).query("ccrmbasic__Contact__c != ''")
+      else
+        records = @client.materialize(type).all
+      end
+      sobjects = records.dup.to_a
+      while records.next_page?
+        sobjects += records.next_page
+        records = records.next_page
+      end
+      sobjects
+    end
+  end
+end

data/lib/fields.rb

@@ -0,0 +1,14 @@
+module SalesforceMigration
+  class Fields
+    # Salesforce objects and the fields for import
+    ISOS_FIELDS             = %w{Active__c Address__c Adult_Chargeback_Fee__c Adult_Commission__c Adult_Transaction_Fee__c Agreement_Received__c Background_Check_Complete__c Business_Phone_Number__c Cell_Phone_Number__c Comments__c CreatedBy CreatedById CreatedDate Curren__c CurrencyIsoCode Email_Address__c Fax_Number__c Gambling_Chargeback_Fee__c Gambling_Commission_Fee__c Gambling_Transaction_Fee__c General_Conventional_Chargeback_Fee__c General_Conventional_Commission_Fee__c General_Conventional_Transaction_Fee__c General_Non_Conventional_Chargeback_Fee__c General_Non_Conventional_Commission_Fee__c General_Non_Conventional_Transaction_Fee__c ISO_ID_Assigned__c ISO_ID_Number__c ISO_Owner_Name__c Id KYC_Gathered__c LastActivityDate LastModifiedBy LastModifiedById LastModifiedDate Name New_Hire_Package_Received__c Owner OwnerId Pharmacy_Chargeback_Fee__c Pharmacy_Commission_Fee__c Pharmacy_Transaction_Fee__c Projected_Deals_Count__c Recruiter__c Region__c System_Access_Granted__c Training_Completed__c Welcome_Call_Completed__c Welcome_Package_Sent__c eMerchantPay_ISO_ID__c}
+    AGENT_FIELDS            = %w{Id Name LastActivityDate CreatedBy CreatedDate CreatedById LastModifiedDate LastModifiedById LastModifiedBy Owner OwnerId Agent_Billing_Address__c Agent_Cell_Phone__c Agent_Id__c Agent_Phone__c Agent_Skype__c Agreement_Received__c Agreement_Sent__c Background_Check_Complete__c Business_Phone_Number__c Comments__c Commission_Fee__c Email_Address__c eMerchantPay_Agent_ID__c Fee_Comments__c ISO_Company__c New_Hire_Package_Received__c One_Time_Commission_Currency__c Recruiter__c Region__c Split_percentage__c System_Access_Granted__c Title__c Transaction_Fee__c}
+    ACCOUNT_FIELDS          = %w{Additional_Url__c Agents__c AnnualRevenue Approval_Citeria__c Approve_1__c Approve_2__c Approve_3__c BillingCity BillingCountry BillingPostalCode BillingState BillingStreet City__c Comment__c CreatedBy CreatedById CreatedDate Credit_Card__c Curren__c CurrencyIsoCode Current_Volume__c Description Descriptor__c FD_Merchant_Number__c Fax ISO_Name__c Id Industry Info_Capture__c Integration__c Iovation__c IsPartner Jigsaw LastModifiedBy LastModifiedById LastModifiedDate Merchant_Email__c Merchant_Type__c Name NumberOfEmployees Overlying_Company__c Owner OwnerId Phone Projected_Monthly_Volume__c Recruiter__c Secondary_FD_Merchant_Number__c ShippingCity ShippingCountry ShippingPostalCode ShippingState ShippingStreet Status_Comments__c Status__c Threat_Metrix__c Type__c Url__c VBV_MC3D__c Vacation1__c Website}
+    PAYMENT_METHODS_FIELDS  = %w{Accounts__c Business_Contact__c Business_Type_Coments__c Business_type__c Buy_Rates__c Chargeback_Comments__c Chargeback__c Client_Integration__c Countries2__c Countries__c CreatedBy CreatedById CreatedDate CurrencyIsoCode Id Integration__c LastModifiedBy LastModifiedById LastModifiedDate Name Our_Buy_Rates__c Owner OwnerId Proccessing_Currency__c Processing_Currency__c Rebilling_Comments__c Rebilling__c Refund_Comments__c Refund__c Release_Date__c Settlement_Currency__c Settlement_Cycle__c Settlement__c Tech_Contact__c Type__c V_terminal_Comments__c V_terminal__c}
+    BANKS_FIELDS            = %w{Account_Number__c BIC_Code__c Merchant_2__c Bank_Address__c Bank_Name__c Contract__c CreatedBy CreatedById CreatedDate CurrencyIsoCode Holder_Address__c IBAN__c Id LastActivityDate LastModifiedBy LastModifiedById LastModifiedDate Name Owner OwnerId Processing_Currency__c Routing_Number__c SWIFT__c Settlement_Currency__c}
+    MERCHANTTOAPM_FIELDS    = %w{Merchant__c Payment_Methods__c}
+    CCRMBASIC__EMAIL_FIELDS = %w{ccrmbasic__To__c ccrmbasic__Cc__c ccrmbasic__Body__c Id Name}
+    EMAIL_ASSOCIATION_FIELDS= %w{Merchant_Name__c Email__c}
+    SYSTEM_FIELDS           = %w{id owner ownerid createddate createdby createdbyid lastactivitydate lastmodifieddate lastmodifiedby lastmodifiedbyid recruiter region}
+  end
+end

data/lib/import.rb

@@ -0,0 +1,553 @@
+$LOAD_PATH.unshift File.dirname(__FILE__)
+
+module SalesforceMigration
+  # Workflow for import of specific object
+  # 
+  # import_agents -> load_file_for_import -> transform_csv_file -> prefix_sf_attribute_names
+  #               -> transform_agents
+  #               -> populate_sugar -> get_sugarcrm_module_type
+  #                                 -> convert_string_to_datetime
+  #                                 -> create_association -> find_sugarcrm_object
+  #                                 -> create_user
+  class Import
+
+    # Initialize migration import object and start the import
+    #
+    # @param [Hash] options hash representing passed command line options
+    def initialize(options)
+      credentials = YAML.load_file(options[:config_file])
+      url      = credentials['sugar_url']
+      username = credentials['sugar_username']
+      password = credentials['sugar_password']
+      @csv_dir = options[:csv_dir]
+      SugarCRM.connect(url, username, password)
+      @action = options[:action]
+      @isos, @agents, @merchants, @bank_accounts, @agent_users, @iso_users, @emails  = [], [], [], [], [], [], []
+      @payment_to_merchants, @emails_to_merchants = [], []
+      @logger = SalesforceMigration::Runner::create_logger
+      @logger.info("Import action started")
+      start
+      @logger.info("Import action ended successfully")
+    end
+    
+    private
+    
+    def start
+      import_emails
+      import_payment_methods
+      import_isos
+      import_agents
+      import_merchants
+      import_settlement_bank_accounts
+      
+      associate_records_to_groups
+    end
+
+    # Import methods. They load the csv file, for the given type
+    # get rid of unnecessary information, transforms other fields, so
+    # they can match the one in SugarCRM and calls for the populate_sugar method
+    # which start the actual import. Keep in mind, that the order of the imports is important
+    def import_isos
+      records = load_file_for_import 'ISOs__c'
+      records = transform_isos(records)
+      populate_sugar(records, "iso")
+    end
+
+    def import_agents
+      records = load_file_for_import 'Agent__c'
+      records = transform_agents(records)
+      populate_sugar(records, "agent")
+    end
+
+    def import_merchants
+      records = load_file_for_import 'Account'
+      records = transform_merchants(records)
+      populate_sugar(records, "merchant")
+    end
+
+    def import_settlement_bank_accounts
+      records = load_file_for_import 'Banks__c'
+      records = transform_settlement_bank_accounts(records)
+      populate_sugar(records, "settlement_bank_account")
+    end
+
+    def import_payment_methods
+      records = load_file_for_import 'Payment_Methods__c'
+      junction_records = load_file_for_import 'MerchantToAPM__c'
+      # We don`t need db table to save the many-to-many associations between Merchant and Payment Method,
+      # so we just store them in an array and use it later on
+      junction_records.each do |jrecord|
+        @payment_to_merchants << jrecord
+      end
+      populate_sugar(records, "payment_method")
+    end
+    
+    def import_emails
+      records = load_file_for_import 'ccrmbasic__Email__c'
+      records = transform_emails(records)
+      junction_records = load_file_for_import 'Email_Association__c'
+      # We don`t need db table to save the many-to-many associations between Merchant and Payment Method,
+      # so we just store them in an array and use it later on
+      junction_records.each do |jrecord|
+        @emails_to_merchants << jrecord
+      end
+      populate_sugar(records, "email")
+    end
+    
+    # Load CSV for import
+    #
+    # @param [String] module_name name of the module
+    #
+    # @return [Array] parsed CSV file
+    def load_file_for_import(module_name)
+      if @action == 'initial_run'
+        filename = "#{@csv_dir}/initial/#{module_name}_export.csv"
+      else
+        today = lambda { Date.today.to_s }
+        dir = "#{@csv_dir}/update/#{today.call}"
+        filename = "#{dir}/#{module_name}_export.csv"
+      end
+      @logger.error("Could not create or find a csv filename for module #{module_name}") unless defined? filename
+      @logger.info("Loading CSV file #{filename} for import")
+      csv_file = ::CSV.read(filename, :encoding => 'utf-8')
+      transform_csv_file(csv_file)
+    end
+
+    # Load CSV file, removes headers and create a hash with the headers as keys
+    #
+    # @param [Array] file parsed CSV file
+    #
+    # @return [Array] transformed CSV file 
+    def transform_csv_file(csv_file)
+      @logger.info("Transforming CSV file")
+      transformed = []
+      headers = csv_file.shift
+      headers.map!(&method(:prepare_custom_headers))
+
+      csv_file.each do |row|
+        transformed << Hash[headers.zip row]
+      end
+      prefix_sf_attribute_names transformed
+    end
+    
+    #Remove the __c and lowercase the custom header fields, exported from Salesforce.
+    #
+    # @param [String] header
+    def prepare_custom_headers(header)
+      header.end_with?("__c") ? header.slice(0..-4).downcase : header.downcase
+    end
+
+    # Add sf_ prefix to all auto-generated attribute names from Salesforce
+    #
+    # @param [Array<SugarCRM::Namespace::Object>] records records to be prefixed
+    #
+    # @return [Array<SugarCRM::Namespace::Object>] returns the records
+    def prefix_sf_attribute_names(records)
+      sf_attribute_names = SalesforceMigration::Fields::SYSTEM_FIELDS
+      records.each do |r|
+        sf_attribute_names.each do |field|
+          r["sf_#{field.downcase}"] = r[field]
+          r.delete(field)
+        end
+      end
+      records
+    end
+
+    def transform_isos(records)
+      @logger.info("Transforming ISO fields")
+      records.each do |record|
+        record['general_c_chargeback_fee']   = record['general_conventional_chargeback_fee']
+        record['general_c_commission_fee']   = record['general_conventional_commission_fee']
+        record['general_c_transaction_fee']  = record['general_conventional_transaction_fee']
+        record['general_nc_chargeback_fee']  = record['general_non_conventional_chargeback_fee']
+        record['general_nc_commission_fee']  = record['general_non_conventional_commission_fee']
+        record['general_nc_transaction_fee'] = record['general_non_conventional_transaction_fee']
+        record.delete 'general_conventional_chargeback_fee'
+        record.delete 'general_conventional_commission_fee'
+        record.delete 'general_conventional_transaction_fee'
+        record.delete 'general_non_conventional_chargeback_fee'
+        record.delete 'general_non_conventional_commission_fee'
+        record.delete 'general_non_conventional_transaction_fee'
+      end
+      records
+    end
+
+    def transform_agents(records)
+      @logger.info("Transforming fields")
+      records.each do |record|
+        record['sf_iso'] = record['iso_company']
+        record.delete 'iso_company'
+      end
+      records
+    end
+
+    def transform_merchants(records)
+      @logger.info("Transforming Merchant fields")
+      records.each do |record|
+        record['sf_agent'] = record['agents']
+        record['url'] = record['url'].to_s.gsub(/<[^>]*>|^[\n\r]*/, '')
+        record['additional_url'] = record['additional_url'].to_s.gsub(/(<[^>]*>|^[\n\r]*)/, '')
+        record.delete 'agents'
+        record.delete 'sf_lastactivitydate'
+        record.delete 'sf_lastmodifieddate'
+        record.delete 'sf_region'
+      end
+      records
+    end
+
+    def transform_settlement_bank_accounts(records)
+      @logger.info("Transforming Settlement Bank Account fields")
+      records.each do |record|
+        record['sf_iso'] = record['iso']
+        record['sf_contract'] = record['contract']
+        record['sf_merchant'] = record['merchant_2']
+        record.delete 'merchant_2'
+        record.delete 'iso'
+        record.delete 'contract'
+        record.delete 'sf_recruiter'
+        record.delete 'sf_region'
+      end
+      records
+    end
+    
+    def transform_emails(records)
+      @logger.info("Transforming Email fields")
+      records.each do |record|
+        record['receiver'] = record['ccrmbasic__to']
+        record['cc'] = record['ccrmbasic__cc']
+        record['body'] = convert_characters(record['ccrmbasic__body']) unless record['ccrmbasic__body'].nil?
+        
+        record.delete 'ccrmbasic__to'
+        record.delete 'ccrmbasic__cc'
+        record.delete 'ccrmbasic__body'
+      end
+      records
+    end
+
+    # Convert all string datetime attributes to DateTime objects
+    #
+    # @param [SugarCRM::Namespace::Object] record the record which attributes must be converted
+    #
+    # @return [SugarCRM::Namespace::Object] the record
+    def convert_string_to_datetime(record)
+      record['sf_lastmodifieddate'] = record['sf_lastmodifieddate'].to_datetime if record['sf_lastmodifieddate']
+      record['sf_createddate']      = record['sf_createddate'].to_datetime      if record['sf_createddate']
+      record['sf_lastactivitydate'] = record['sf_lastactivitydate'].to_date     if record['sf_lastactivitydate']
+      record
+    end
+    
+    def convert_characters(string)
+        string.gsub!(/\'/, '&apos;')
+        string.gsub!(/'/, '')
+        string.gsub!(/"/, '')
+        string.gsub!(/\"/, '&quot;')
+        string
+      end
+
+    # Populate SugarCRM with records
+    #  If it's initial run, it will create all records from the CSV file
+    #  If it's update, it will update existing records and create new if necessary
+    #
+    # @param [Array] records records to be inserted in SugarCRM
+    # @param [String] type type of the object
+    def populate_sugar(records, type)
+      module_type = get_sugarcrm_module_type(type)
+      case @action
+      when 'initial_run'
+          @logger.info("Creating new records for #{type} type")
+          records.each do |record|
+            create_sugar_record(module_type, record, type)
+          end
+      when 'update'
+        records.each do |record|
+          record = convert_string_to_datetime(record)
+          existing_record = find_sugarcrm_object(type, 'sf_id', record['sf_id'])
+          existing_record = existing_record.first if existing_record.is_a?(Array)
+          if existing_record
+            #TODO Nil values are not handled properly by SugarCRM in update_attributes, so we must transform them into blank values
+            #remove this loop and use the main one!!
+            record.each do |key, val|
+              record[key] = "" if val.nil?
+            end
+            @logger.info("Updating record for #{type} #{record['name']}")
+            existing_record.update_attributes!(record)
+          else
+            @logger.info("Creating new record for #{type} #{record['name']}")
+            create_sugar_record(module_type, record, type)
+          end
+        end
+      end
+    end
+    
+    # Create the actual records and users in SugarCRM. Populates the var_pool
+    def create_sugar_record(module_type, record, type)
+      record = convert_string_to_datetime(record)
+
+      obj = module_type.new(record)
+      obj.save!
+      obj = create_association(obj, type) unless %(email payment_method).include? type
+      create_security_group_iso obj if type == 'iso'
+      
+      create_user(obj, type) if %(iso agent).include? type
+      populate_var_pool(obj, type)
+    end
+    
+    # Populates variables with SugarCRM objects.
+    # We use them later on, when associating objects with Security Groups
+    # Actually we don`t use @bank_accounts & @emails for now, but it`s probably a good idea to store the objects
+    # @param [SugarCRM::Namespace::Object] obj object for which a user will be created
+    # @param [String] type type of the object
+    def populate_var_pool(obj, type)
+      case type
+      when 'merchant'
+        @merchants << obj
+      when 'settlement_bank_account'
+        @bank_accounts << obj
+      when 'iso'
+        @isos << obj
+      when 'agent'
+        @agents << obj
+      when 'email'
+        @emails << obj
+      end  
+    end
+
+    # Create association for agent, merchant, settlement bank Account, Email, Payment Method
+    #   If it is agent, it will find the ISO by id and create the association
+    #   If it is merchant, it will find the Agent, Payment Method, User and create the associations
+    #   If it is Settlement Bank Account it will find the Merchant and create the associations
+    # @param [SugarCRM::Namespace::Object] obj the object for which an association will be created
+    # @param [String] type type of the object
+    #
+    # @return [SugarCRM::Namespace::Object] the object
+    def create_association(obj, type)
+      @logger.info("Creating association for #{type} #{obj.name}")
+      case type
+      when "agent"
+        iso = find_sugarcrm_object('iso', 'sf_id', obj.sf_iso)
+        obj.associate! iso if iso
+      when "merchant"
+        
+        payment_method_id = find_payment_method_id(obj.sf_id)
+        if payment_method_id
+            payment_method = find_sugarcrm_object('payment_method', 'sf_id', payment_method_id)
+            obj.associate! payment_method
+        end
+       
+        email_id = find_email_id(obj.sf_id)
+        if email_id
+          email = find_sugarcrm_object('email', 'sf_id', email_id)
+          obj.associate! email
+        end
+        
+        agent = find_sugarcrm_object('agent', 'sf_id', obj.sf_agent)
+        if agent
+          obj.associate! agent
+          obj.assigned_user_id = agent.assigned_user_id
+        end
+        
+      when "settlement_bank_account"
+        merchant = find_sugarcrm_object('merchant', 'sf_id', obj.sf_merchant)
+        obj.associate! merchant if merchant
+      end
+      obj
+    end
+
+    # Create user associated with SugarCRM object
+    # Default email: mail@example.com, default password: 123456
+    # 
+    # @param [SugarCRM::Namespace::Object] obj object for which a user will be created
+    # @param [String] type type of the object 
+    def create_user(obj, type)
+        @logger.info("Creating user for #{type} #{obj.name}")
+        user = SugarCRM::User.new
+        user.user_name = (type == 'agent') ? obj.emerchantpay_agent_id : obj.emerchantpay_iso_id
+        user.user_name ||= "EMP"
+        user.last_name = obj.name
+        user.type_c = type
+        #user.email1 = obj.email_address || "mail@example.com"
+        user.email1 = 'stefan@emerchantpay.com'
+        user.status = 'Inactive'
+        user.system_generated_password = false
+        user.save!
+        obj.assigned_user_id = user.id
+        obj.save!
+        
+        populate_user_pool(user, type)
+    end
+    
+    # Populates Users as SugarCRM objects.
+    # We use them later on, when associating objects with Security Groups
+    # @param [SugarCRM::Namespace::Object] Already created user object
+    # @param [String] type type of the object
+    def populate_user_pool(user, type)
+      case type
+      when 'iso'
+        @iso_users << user
+      when 'agent'
+        @agent_users << user
+      end
+    end
+    
+    # Find records(payment methods, emails) in the junction object arrays, given the merchant_id
+    # Both payment methods and emails have a many-to-many relationship with the merchant in Salesforce
+    def find_payment_method_id(merchant_id)
+      @payment_to_merchants.each do |record|
+        return record['payment_methods'] if record['merchant'] == merchant_id.to_s
+      end
+      false
+    end
+    
+    def find_email_id(merchant_id)
+      @emails_to_merchants.each do |record|
+        return record['email'] if record['merchant_name'] == merchant_id.to_s
+      end
+      false
+    end
+    
+    #Creates the Security Group object in SugarCRM
+    # @param [SugarCRM::Namespace::EmpIso] Iso object
+    def create_security_group_iso(iso)
+      @logger.info("Creating SecurityGroup #{iso.name}")
+      sg = SugarCRM::SecurityGroup.new(:name => iso.name) unless find_sugarcrm_object('security_group','name', iso.name)
+      sg.save! if sg
+    end
+    
+    # Assign all records, that need to be in a Security Group, to the Group
+    def associate_records_to_groups
+      put_isos_into_iso_group
+      put_agents_into_iso_group
+      put_merchants_into_iso_group
+    end
+
+    def put_isos_into_iso_group
+      if @isos
+        @logger.info("Puting ISOs into ISO groups")
+        role = SugarCRM::ACLRole.find_by_name("isos")
+        @isos.each do |iso|
+          sg = find_sugarcrm_object('security_group','name', iso.name)
+          user   = SugarCRM::User.find_by_last_name(iso.name)
+          iso.associate! sg
+          user.associate! sg
+          role.associate! sg
+        end
+      end
+    end
+
+    def put_agents_into_iso_group
+      if @agents
+        @logger.info("Putting agent records, agent users and agent role into ISO groups")
+        @agents.each do |agent|
+          sg = find_sugarcrm_object('security_group','name', agent.emp_iso.first.name) unless agent.emp_iso.empty?
+          role = SugarCRM::ACLRole.find_by_name('agents')
+            if sg
+              user = SugarCRM::User.find_by_last_name(agent.name)
+              @logger.info("Puting Agent #{agent.name} in Security Group #{sg.name}")
+              agent.associate! sg
+              user.associate! sg
+              role.associate! sg 
+            end  
+        end
+      end
+    end
+    
+    # Bank accounts, Emails and Payment Methods must be inserted from this method, not separately,
+    # because we need to use the merchant object loop, which provides us with the ready merchant id, with which 
+    # we can check if a merchant has one or more bank acounts, emails, pm, etc.. Otherwise we have to
+    # make a separate merchant select in every other method.
+    def put_merchants_into_iso_group
+      if @merchants
+        @logger.info("Puting merchants into ISO groups")
+        @merchants.each do |merchant|
+          unless (merchant.emp_agent.empty? || merchant.emp_agent.first.emp_iso.empty?)
+            sg = find_sugarcrm_object('security_group','name', merchant.emp_agent.first.emp_iso.first.name)          
+            @logger.info("Puting merchant #{merchant.name} into ISO group")
+            
+            if sg
+              merchant.associate! sg
+              put_email_objects_into_iso_group(sg, merchant.sf_id)
+              put_payment_methods_objects_into_iso_group(sg, merchant.sf_id)
+            end
+            
+            bank  = find_sugarcrm_object('settlement_bank_account', 'sf_merchant', merchant.sf_id)
+            if (bank)
+              @logger.info("Puting Bank Account for #{merchant.name} into ISO group")
+               put_bank_accounts_into_iso_group(bank, sg)
+            end
+          end
+        end
+      end
+    end
+    
+    def put_bank_accounts_into_iso_group(banks, sg)
+      if banks.is_a?(Array)
+        banks.each do |bank|
+          bank.associate! sg
+        end
+      else
+        banks.associate! sg
+      end
+    end
+    
+    def put_email_objects_into_iso_group(sg, merchant_id)
+      @logger.info("Puting Emails into ISO group")
+      email_id = find_email_id(merchant_id)
+      if email_id
+        email = find_sugarcrm_object('email', 'sf_id', email_id)
+        if email
+          if email.is_a?(Array)
+            email.each do |e|
+              e.associate! sg
+            end
+          else
+            email.associate! sg
+          end
+        end
+      end
+    end
+    
+    def put_payment_methods_objects_into_iso_group(sg, merchant_id)
+      @logger.info("Puting Payment Methods into ISO group")
+      payment_method_id = find_payment_method_id(merchant_id)
+      if payment_method_id
+        payment_method = find_sugarcrm_object('payment_method', 'sf_id', payment_method_id)
+        if payment_method
+          if payment_method.is_a?(Array)
+            payment_method.each do |pm|
+              pm.associate! sg
+            end
+          else
+            payment_method.associate! sg
+          end
+        end
+      end
+    end
+    
+    # Returns the SugarCRM module namespace
+    # @param [String] type type of the module object
+    def get_sugarcrm_module_type(type)
+      modules = {
+        "iso"                     => SugarCRM::EmpIso,
+        "agent"                   => SugarCRM::EmpAgent,
+        "merchant"                => SugarCRM::EmpMerchant,
+        "payment_method"          => SugarCRM::EmpPaymentmethod,
+        "settlement_bank_account" => SugarCRM::EmpSettlementBankAccount,
+        "security_group"          => SugarCRM::SecurityGroup,
+        "email"                   => SugarCRM::EmpEmail
+      }
+      modules[type]
+    end
+
+    # Find a SugarCRM object
+    #
+    # @param [String] type type of the module object
+    # @param [String] attribute name of the attribute to search by
+    # @param [String] search_string string used in WHERE clause
+    #   find_sugarcrm_object('emp_ISO', 'sf_id', 'a073000000SSTEYAA5') # => #<SugarCRM::Namespace0::EmpIso ... >
+    def find_sugarcrm_object(type, attribute, search_string)
+        module_type = get_sugarcrm_module_type(type)._module.name
+        SugarCRM.connection.get_entry_list(module_type, "#{module_type.downcase}.#{attribute} = '#{search_string}'")
+    end
+
+  end
+end

data/lib/mailer.rb

@@ -0,0 +1,117 @@
+$LOAD_PATH.unshift File.dirname(__FILE__)
+
+module SalesforceMigration
+  
+  class Mailer
+    
+    def initialize(options)
+      @@config = YAML.load_file(options[:config_file])
+      @logger = SalesforceMigration::Runner::create_logger
+      
+      # Ugly way to fix the bug with user_hash update
+      # We set up a connection and update it manually.update_attributes does not work.
+      # The issue is forwarded to sugarcrm gem crew
+      ActiveRecord::Base.establish_connection(
+        :adapter  => @@config['db_type'],
+        :host     => "localhost",
+        :username => @@config['db_user'],
+        :password => @@config['db_password'],
+        :database => @@config['db_database']
+      )
+      
+      @logger.info("Starting the mailer")
+      start
+      @logger.info("Mailer finished")
+    end
+    
+    private
+    
+    #Search the agent portal database for all users, who are inactive
+    #This is the script, that activates them
+    def start
+      inactive_users = SugarCRM::User.find_all_by_status('Inactive')
+
+      if inactive_users
+        inactive_users.each do |user|
+          @logger.info("Sending email to #{user.last_name}")
+
+          plain_password = generate_password
+          hash_password = Digest::MD5.hexdigest plain_password
+          
+          sm = send_mail(user.email1, user.user_name, plain_password)
+          if sm
+            query = "UPDATE users SET user_hash='#{hash_password}', status='Active' WHERE id='#{user.id}'"
+            ActiveRecord::Base.connection.execute(query);
+
+            @logger.info("Updated user #{user.last_name} status to active")
+          end
+        end
+      end
+    end
+    
+    #Send the welcoming email to the user
+    #We need the welcoming text
+    #@param [String] the email address
+    def send_mail(email, username, password)
+      mail = Mail.new do
+        from    'agentportal@emerchantpay.com'
+        to      email
+        subject 'Welcome to Emerchantpay Agent Portal'
+        html_part do
+          body  Mailer::create_template(username, password)
+        end
+      end
+      mail.delivery_method :smtp, {:address        => "emp-ldn-exch01.emp.internal.com",
+                                   :port           => 25,
+                                   :domain         => "emp.internal.com",
+                                   :authentication => nil,
+                                   :enable_starttls_auto => true}
+
+      if mail.deliver!
+        true
+      else
+        @logger.error("Email for user #{last_name} failed!")  
+      end
+    end
+    
+    # Generates the plain text password that is going to be
+    # send to the user in the email
+    # ActiveSupport::SecureRandom is deprecated in Rails  > 3.1
+    def generate_password
+      str = SecureRandom.hex(6)
+      str
+    end
+    
+    def self.create_template(username, password)
+      email_template = <<-TEMPLATE
+        Dear valued partner,<br/><br/>
+
+        We have created an account for you to access the newly created eMerchantPay <a href="#{@@config["sugar_url"]}">Agent Portal</a><br/><br/>
+
+        Your account data is:<br/><br/>
+        
+        Login: <b>#{username}</b><br/>
+        Password: <b>#{password}</b><br/><br/>
+          
+        Features:<br/>
+        
+          <ul>
+            <li>Based on the acclaimed SugarCRM, check details <a href="http://www.sugarcrm.com">here</a></li>
+            <li>Exports of merchants and various data items in different formats for further processing</li>
+            <li>Viewing information for your Merchants and the associated resources - bank accounts, emails, payment methods, etc.</li>
+            <li>Homepage customization for personalization and access</li>
+            <li>Can be accessed by mobile phone</li>
+            <li>And many others...</li>
+          </ul>
+        
+        <br/>
+        If you have any questions, send an email to support@emerchantpay.com (also available by clicking the Support link from the main menu in the <a href="#{@@config["sugar_url"]}">Agent Portal</a>)
+        <br/><br/>
+        ---<br/><br/>
+        
+        Your <a href="http://www.emerchantpay.com">eMerchantPay</a> team<br/><br/>
+        TEMPLATE
+      email_template
+    end
+  end
+end

data/lib/sf_migrate.rb

@@ -0,0 +1,77 @@
+$LOAD_PATH.unshift File.dirname(__FILE__)
+
+require 'optparse'
+require 'csv'
+require 'yaml'
+require 'logger'
+require 'fileutils'
+require 'databasedotcom'
+require 'sugarcrm'
+require 'mail'
+require 'active_support/core_ext'
+require 'active_record'
+require 'fields'
+require 'export'
+require 'import'
+require 'mailer'
+
+module SalesforceMigration
+	module Runner
+		extend self
+
+		# Start the migration dependening on command line argument: `initial_run` or `update`
+		def start
+			options = {}
+			# Where do we need to store the csvs?
+			options[:csv_dir] = "/var/sugarcrm/csv"
+			options[:log_dir] = "/var/log/sugarcrm"
+			options[:config_file] = File.join(File.dirname(__FILE__), "../config/credentials.yaml")
+			@log_dir = options[:log_dir]
+			create_logger
+			optparse = OptionParser.new do |opts|
+				opts.banner = "Usage: sf_migrate [options]"
+				opts.on("-a", "--action NAME", "Start the initial_run or the daily update") do |action|
+					options[:action] = action
+				end
+				opts.on("-f", "--config_file CONFIG_FILE", "Supply an optional config file") do |f|
+					options[:config_file] = File.expand_path(f) if File.exists? f
+				end 
+				opts.on("-c", "--csv_dir CSV_DIR", "Set the directory in which exported csv from Salesforce will be kept") do |c|
+					options[:csv_dir] = c
+				end
+				opts.on("-l", "--log_dir LOG_DIR", "Set the directory in which the logger will reside") do |l|
+					options[:log_dir] = l
+				end
+				opts.on("-m", "--send_mail", "Send activation email to every new user") do |m|
+					options[:send_mail] = m
+				end
+				opts.on( '-h', '--help', 'Display this screen' ) do
+             puts opts
+             exit
+        end
+			end
+			optparse.parse!
+				begin
+					SalesforceMigration::Export.new(options)
+					SalesforceMigration::Import.new(options)
+					SalesforceMigration::Mailer.new(options) if options[:send_mail]
+				rescue => e
+	      	@logger.error(e)
+				end
+		end
+		
+		def create_logger
+      original_formatter = Logger::Formatter.new
+      today = lambda { Date.today.to_s }
+      dir = "#{@log_dir}/#{today.call}"
+      file = "#{dir}/migration.log"
+      FileUtils.mkdir_p(dir) unless Dir.exists? dir
+      @logger = Logger.new(file)
+      @logger.formatter = proc { |severity, datetime, progname, msg|
+        original_formatter.call("IMPORT", datetime, progname, msg)
+      }
+      @logger
+    end
+	  
+	end
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification 
+name: sf_migrate
+version: !ruby/object:Gem::Version 
+  version: 1.0.2
+platform: ruby
+authors: 
+- Dimitar KostovStefan Slaveykov
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-05-30 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sugarcrm_emp
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.10.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: databasedotcom_emp
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.3.1
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 3.2.6
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 3.2.13
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: mail
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 2.5.3
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: roo
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.10.2
+  type: :runtime
+  version_requirements: *id006
+description: SalesForce to SugarCRM migration tool
+email: 
+- stefan@emerchantpay.com
+executables: 
+- sf_migrate
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.md
+- Gemfile
+- bin/sf_migrate
+- lib/fields.rb
+- lib/export.rb
+- lib/import.rb
+- lib/mailer.rb
+- lib/sf_migrate.rb
+- config/credentials.yaml.example
+homepage: 
+licenses: []
+
+metadata: {}
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - &id007 
+    - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - *id007
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Makes a bridge between Salesforce and SugarCRM
+test_files: []
+