maestrano-connector-rails 0.4.4 → 1.0.0

data/app/models/maestrano/connector/rails/concerns/entity.rb

@@ -1,14 +1,14 @@
 module Maestrano::Connector::Rails::Concerns::Entity
   extend ActiveSupport::Concern
 
-  module ClassMethods
-    # Return an array of all the entities that the connector can synchronize
-    # If you add new entities, you need to generate
-    # a migration to add them to existing organizations
-    def entities_list
-      raise "Not implemented"
-    end
+  def initialize(organization, connec_client, external_client, opts={})
+    @organization = organization
+    @connec_client = connec_client
+    @external_client = external_client
+    @opts = opts
+  end
 
+  module ClassMethods
     # ----------------------------------------------
     #                 IdMap methods
     # ----------------------------------------------
@@ -19,34 +19,18 @@ module Maestrano::Connector::Rails::Concerns::Entity
       }
     end
 
-    def find_or_create_idmap(organization_and_id)
-      Maestrano::Connector::Rails::IdMap.find_or_create_by(names_hash.merge(organization_and_id))
-    end
-
     # organization_and_id can be either:
     # * {connec_id: 'id', organization_id: 'id'}
     # * {external_id: 'id', organization_id: 'id'}
     # Needs to include either connec_entity or external_entity for complex entities
+    def find_or_create_idmap(organization_and_id)
+      Maestrano::Connector::Rails::IdMap.find_or_create_by(names_hash.merge(organization_and_id))
+    end
     def find_idmap(organization_and_id)
       Maestrano::Connector::Rails::IdMap.find_by(names_hash.merge(organization_and_id))
     end
-
-    def create_idmap_from_external_entity(entity, organization)
-      h = names_hash.merge({
-        external_id: id_from_external_entity_hash(entity),
-        name: object_name_from_external_entity_hash(entity),
-        organization_id: organization.id
-      })
-      Maestrano::Connector::Rails::IdMap.create(h)
-    end
-
-    def create_idmap_from_connec_entity(entity, organization)
-      h = names_hash.merge({
-        connec_id: entity['id'],
-        name: object_name_from_connec_entity_hash(entity),
-        organization_id: organization.id
-      })
-      Maestrano::Connector::Rails::IdMap.create(h)
+    def create_idmap(organization_and_id)
+      Maestrano::Connector::Rails::IdMap.create(names_hash.merge(organization_and_id))
     end
 
     # ----------------------------------------------
@@ -114,11 +98,16 @@ module Maestrano::Connector::Rails::Concerns::Entity
       raise "Not implemented"
     end
 
-    # [{reference_class: Entities::.., connec_field: 'account_id', external_field: 'account/something/id'}]
+    # An array of connec fields that are references
     def references
       []
     end
 
+    # An array of fields for smart merging. See connec! documentation
+    def connec_matching_fields
+      nil
+    end
+
     def can_read_connec?
       can_write_external?
     end
@@ -135,10 +124,6 @@ module Maestrano::Connector::Rails::Concerns::Entity
       true
     end
 
-    def can_update_connec?
-      true
-    end
-
     def can_update_external?
       true
     end
@@ -147,24 +132,19 @@ module Maestrano::Connector::Rails::Concerns::Entity
   # ----------------------------------------------
   #                 Mapper methods
   # ----------------------------------------------
-  # Map a Connec! entity to the external format
-  def map_to_external(entity, organization)
-    ref_hash = {}
-    self.class.references.each do |ref|
-      ref_hash.merge! ref[:external_field].split('/').reverse.inject(self.class.id_from_ref(entity, ref, false, organization)) { |a, n| { n.to_sym => a } }
-    end
-
-    self.class.mapper_class.normalize(entity).merge(ref_hash)
+  # Map a Connec! entity to the external model
+  def map_to_external(entity)
+    connec_id = entity[:__connec_id]
+    mapped_entity = self.class.mapper_class.normalize(entity)
+    (connec_id ? mapped_entity.merge(__connec_id: connec_id) : mapped_entity).with_indifferent_access
   end
 
-  # Map an external entity to Connec! format
-  def map_to_connec(entity, organization)
-    ref_hash = {}
-    self.class.references.each do |ref|
-      ref_hash.merge! ref[:connec_field].split('/').reverse.inject(self.class.id_from_ref(entity, ref, true, organization)) { |a, n| { n.to_sym => a } }
-    end
-
-    self.class.mapper_class.denormalize(entity).merge(ref_hash)
+  # Map an external entity to Connec! model
+  def map_to_connec(entity)
+    mapped_entity = self.class.mapper_class.denormalize(entity).merge(id: self.class.id_from_external_entity_hash(entity))
+    folded_entity = Maestrano::Connector::Rails::ConnecHelper.fold_references(mapped_entity, self.class.references, @organization)
+    folded_entity.merge!(opts: (mapped_entity[:opts] || {}).merge(matching_fields: self.class.connec_matching_fields)) if self.class.connec_matching_fields
+    folded_entity
   end
 
   # ----------------------------------------------
@@ -174,324 +154,346 @@ module Maestrano::Connector::Rails::Concerns::Entity
   # * full_sync
   # * $filter (see Connec! documentation)
   # * $orderby (see Connec! documentation)
-  def get_connec_entities(client, last_synchronization, organization, opts={})
-    return [] unless self.class.can_read_connec?
+  def get_connec_entities(last_synchronization)
+    return [] if @opts[:skip_connec] || !self.class.can_read_connec?
 
-    Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Fetching Connec! #{self.class.connec_entity_name}")
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching Connec! #{self.class.connec_entity_name}")
 
     entities = []
     query_params = {}
-    query_params[:$orderby] = opts[:$orderby] if opts[:$orderby]
+    query_params[:$orderby] = @opts[:$orderby] if @opts[:$orderby]
 
     # Fetch first page
-    if last_synchronization.blank? || opts[:full_sync]
-      Maestrano::Connector::Rails::ConnectorLogger.log('debug', organization, "entity=#{self.class.connec_entity_name}, fetching all data")
-      query_params[:$filter] = opts[:$filter] if opts[:$filter]
+    page_number = 0
+    if last_synchronization.blank? || @opts[:full_sync]
+      Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "entity=#{self.class.connec_entity_name}, fetching all data")
+      query_params[:$filter] = @opts[:$filter] if @opts[:$filter]
     else
-      Maestrano::Connector::Rails::ConnectorLogger.log('debug', organization, "entity=#{self.class.connec_entity_name}, fetching data since #{last_synchronization.updated_at.iso8601}")
-      filter = "updated_at gt '#{last_synchronization.updated_at.iso8601}'"
-      filter += " and #{opts[:$filter]}" if opts[:$filter]
-      query_params[:$filter] = filter
+      Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "entity=#{self.class.connec_entity_name}, fetching data since #{last_synchronization.updated_at.iso8601}")
+      query_params[:$filter] = "updated_at gt '#{last_synchronization.updated_at.iso8601}'" + (@opts[:$filter] ? " and #{@opts[:$filter]}" : '')
     end
-    response = client.get("/#{self.class.normalized_connec_entity_name}?#{query_params.to_query}")
-    raise "No data received from Connec! when trying to fetch #{self.class.normalized_connec_entity_name}" unless response && !response.body.blank?
 
-    response_hash = JSON.parse(response.body)
-    Maestrano::Connector::Rails::ConnectorLogger.log('debug', organization, "received first page entity=#{self.class.connec_entity_name}, response=#{response.body}")
-    if response_hash["#{self.class.normalized_connec_entity_name}"]
-      entities << response_hash["#{self.class.normalized_connec_entity_name}"]
-    else
-      raise "Received unrecognized Connec! data when trying to fetch #{self.class.normalized_connec_entity_name}"
-    end
+    uri = "/#{self.class.normalized_connec_entity_name}?#{query_params.to_query}"
+    response_hash = fetch_connec(uri, 0)
+    entities = response_hash["#{self.class.normalized_connec_entity_name}"]
 
     # Fetch subsequent pages
     while response_hash['pagination'] && response_hash['pagination']['next']
+      page_number += 1
       # ugly way to convert https://api-connec/api/v2/group_id/organizations?next_page_params to /organizations?next_page_params
       next_page = response_hash['pagination']['next'].gsub(/^(.*)\/#{self.class.normalized_connec_entity_name}/, self.class.normalized_connec_entity_name)
-      response = client.get(next_page)
 
-      raise "No data received from Connec! when trying to fetch subsequent page of #{self.class.connec_entity_name.pluralize}" unless response && !response.body.blank?
-      Maestrano::Connector::Rails::ConnectorLogger.log('debug', organization, "received next page entity=#{self.class.connec_entity_name}, response=#{response.body}")
-
-      response_hash = JSON.parse(response.body)
-      if response_hash["#{self.class.normalized_connec_entity_name}"]
-        entities << response_hash["#{self.class.normalized_connec_entity_name}"]
-      else
-        raise "Received unrecognized Connec! data when trying to fetch subsequent page of #{self.class.connec_entity_name.pluralize}"
-      end
+      response_hash = fetch_connec(uri, page_number)
+      entities << response_hash["#{self.class.normalized_connec_entity_name}"]
     end
 
-    entities = entities.flatten
-    Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Received data: Source=Connec!, Entity=#{self.class.connec_entity_name}, Data=#{entities}")
+    entities.flatten!
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Received data: Source=Connec!, Entity=#{self.class.connec_entity_name}, Data=#{entities}")
     entities
   end
 
-  def push_entities_to_connec(connec_client, mapped_external_entities_with_idmaps, organization)
-    push_entities_to_connec_to(connec_client, mapped_external_entities_with_idmaps, self.class.connec_entity_name, organization)
+  def push_entities_to_connec(mapped_external_entities_with_idmaps)
+    push_entities_to_connec_to(mapped_external_entities_with_idmaps, self.class.connec_entity_name)
   end
 
-  def push_entities_to_connec_to(connec_client, mapped_external_entities_with_idmaps, connec_entity_name, organization)
+  def push_entities_to_connec_to(mapped_external_entities_with_idmaps, connec_entity_name)
     return unless self.class.can_write_connec?
 
-    Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize} to Connec! #{connec_entity_name.pluralize}")
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize} to Connec! #{connec_entity_name.pluralize}")
     
-    request_per_call = 100
-    start = 0
-    while start < mapped_external_entities_with_idmaps.size
-      # Prepare batch request
-      batch_entities = mapped_external_entities_with_idmaps.slice(start, request_per_call)
-      batch_request = {sequential: true, ops: []}
-      batch_entities.each do |mapped_external_entity_with_idmap|
-        external_entity = mapped_external_entity_with_idmap[:entity]
-        idmap = mapped_external_entity_with_idmap[:idmap]
-        if idmap.connec_id.blank?
-          batch_request[:ops] << batch_op('post', external_entity, nil, self.class.normalize_connec_entity_name(connec_entity_name), organization)
-        else
-          next unless self.class.can_update_connec?
-          batch_request[:ops] << batch_op('put', external_entity, idmap.connec_id, self.class.normalize_connec_entity_name(connec_entity_name), organization)
-        end
-      end
-
-      # Batch call
-      Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending batch request to Connec! for #{self.class.normalize_connec_entity_name(connec_entity_name)}. Batch_request_size: #{batch_request[:ops].size}. Call_number: #{(start/request_per_call) + 1}")
-      response = connec_client.batch(batch_request)
-      Maestrano::Connector::Rails::ConnectorLogger.log('debug', organization, "Received batch response from Connec! for #{self.class.normalize_connec_entity_name(connec_entity_name)}: #{response}")
-      raise "No data received from Connec! when trying to send batch request for #{self.class.connec_entity_name.pluralize}" unless response && !response.body.blank?
-      response = JSON.parse(response.body)
-
-      # Parse barch response
-      response['results'].each_with_index do |result, index|
-        if result['status'] == 200
-          batch_entities[index][:idmap].update_attributes(last_push_to_connec: Time.now, message: nil)
-        elsif result['status'] == 201
-          batch_entities[index][:idmap].update_attributes(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'], last_push_to_connec: Time.now, message: nil)
-        else
-          Maestrano::Connector::Rails::ConnectorLogger.log('error', organization, "Error while pushing to Connec!: #{result['body']}")
-          batch_entities[index][:idmap].update_attributes(message: result['body'].truncate(255))
-        end
-      end
-      start += request_per_call
-    end
+    proc = lambda{|mapped_external_entity_with_idmap| batch_op('post', mapped_external_entity_with_idmap[:entity], nil, self.class.normalize_connec_entity_name(connec_entity_name))}
+    batch_calls(mapped_external_entities_with_idmaps, proc, connec_entity_name)
   end
 
-  def batch_op(method, mapped_external_entity, id, connec_entity_name, organization)
-    Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending #{method.upcase} #{connec_entity_name}: #{mapped_external_entity} to Connec! (Preparing batch request)")
+  def batch_op(method, mapped_external_entity, id, connec_entity_name)
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending #{method.upcase} #{connec_entity_name}: #{mapped_external_entity} to Connec! (Preparing batch request)")
     {
       method: method,
-      url: "/api/v2/#{organization.uid}/#{connec_entity_name}" + (id.nil? ? '' : "/#{id}"),
+      url: "/api/v2/#{@organization.uid}/#{connec_entity_name}/#{id}", # id should be nil for POST
       params: {
         "#{connec_entity_name}".to_sym => mapped_external_entity
       }
     }
   end
 
-  def map_to_external_with_idmap(entity, organization)
-    idmap = self.class.find_idmap({connec_id: entity['id'], organization_id: organization.id})
-
-    if idmap
-      return nil if idmap.external_inactive || !idmap.to_external
-
-      if idmap.last_push_to_external && idmap.last_push_to_external > entity['updated_at']
-        Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Discard Connec! #{self.class.connec_entity_name} : #{entity}")
-        nil
-      else
-        idmap.update(name: self.class.object_name_from_connec_entity_hash(entity))
-        {entity: map_to_external(entity, organization), idmap: idmap}
-      end
-    else
-      {entity: map_to_external(entity, organization), idmap: self.class.create_idmap_from_connec_entity(entity, organization)}
-    end
-  end
-
   # ----------------------------------------------
   #                 External methods
   # ----------------------------------------------
-  def get_external_entities(client, last_synchronization, organization, opts={})
-    return [] unless self.class.can_read_external?
-    Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Fetching #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize}")
+  def get_external_entities_wrapper(last_synchronization)
+    return [] if @opts[:skip_external] || !self.class.can_read_external?
+    get_external_entities(last_synchronization)
+  end
+  
+  def get_external_entities(last_synchronization)
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize}")
     raise "Not implemented"
   end
 
-  def push_entities_to_external(external_client, mapped_connec_entities_with_idmaps, organization)
-    push_entities_to_external_to(external_client, mapped_connec_entities_with_idmaps, self.class.external_entity_name, organization)
+  def push_entities_to_external(mapped_connec_entities_with_idmaps)
+    push_entities_to_external_to(mapped_connec_entities_with_idmaps, self.class.external_entity_name)
   end
 
-  def push_entities_to_external_to(external_client, mapped_connec_entities_with_idmaps, external_entity_name, organization)
+  def push_entities_to_external_to(mapped_connec_entities_with_idmaps, external_entity_name)
     return unless self.class.can_write_external?
-    Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending Connec! #{self.class.connec_entity_name.pluralize} to #{Maestrano::Connector::Rails::External.external_name} #{external_entity_name.pluralize}")
-    mapped_connec_entities_with_idmaps.each do |mapped_connec_entity_with_idmap|
-      push_entity_to_external(external_client, mapped_connec_entity_with_idmap, external_entity_name, organization)
+
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending Connec! #{self.class.connec_entity_name.pluralize} to #{Maestrano::Connector::Rails::External.external_name} #{external_entity_name.pluralize}")
+    ids_to_send_to_connec = mapped_connec_entities_with_idmaps.map{ |mapped_connec_entity_with_idmap|
+      push_entity_to_external(mapped_connec_entity_with_idmap, external_entity_name)
+    }.compact
+
+    unless ids_to_send_to_connec.empty?
+      # Send the external ids to connec if it was a creation
+      proc = lambda{|id| batch_op('put', {id: [Maestrano::Connector::Rails::ConnecHelper.id_hash(id[:external_id], @organization)]}, id[:connec_id], self.class.normalize_connec_entity_name(self.class.connec_entity_name)) }
+      batch_calls(ids_to_send_to_connec, proc, self.class.connec_entity_name, true)
     end
   end
 
-  def push_entity_to_external(external_client, mapped_connec_entity_with_idmap, external_entity_name, organization)
+
+  def push_entity_to_external(mapped_connec_entity_with_idmap, external_entity_name)
     idmap = mapped_connec_entity_with_idmap[:idmap]
-    connec_entity = mapped_connec_entity_with_idmap[:entity]
+    mapped_connec_entity = mapped_connec_entity_with_idmap[:entity]
 
     begin
+      # Create and return id to send to connec!
       if idmap.external_id.blank?
-        external_id = create_external_entity(external_client, connec_entity, external_entity_name, organization)
-        idmap.update_attributes(external_id: external_id, last_push_to_external: Time.now, message: nil)
+        connec_id = mapped_connec_entity_with_idmap[:idmap].connec_id
+        external_id = create_external_entity(mapped_connec_entity, external_entity_name)
+        idmap.update(external_id: external_id, last_push_to_external: Time.now, message: nil)
+        return {connec_id: connec_id, external_id: external_id, idmap: idmap}
+
+      # Update
       else
         return unless self.class.can_update_external?
-        update_external_entity(external_client, connec_entity, idmap.external_id, external_entity_name, organization)
-        idmap.update_attributes(last_push_to_external: Time.now, message: nil)
+        update_external_entity(mapped_connec_entity, idmap.external_id, external_entity_name)
+
+        # Return the id to send it to connec! if the first push of a singleton
+        if self.class.singleton? && idmap.last_push_to_external.nil?
+          connec_id = mapped_connec_entity_with_idmap[:idmap].connec_id
+          idmap.update(last_push_to_external: Time.now, message: nil)
+          return {connec_id: connec_id, external_id: idmap.external_id}
+        else
+          idmap.update(last_push_to_external: Time.now, message: nil)
+        end
+        
       end
     rescue => e
       # Store External error
-      Maestrano::Connector::Rails::ConnectorLogger.log('error', organization, "Error while pushing to #{Maestrano::Connector::Rails::External.external_name}: #{e}")
-      idmap.update_attributes(message: e.message.truncate(255))
+      Maestrano::Connector::Rails::ConnectorLogger.log('error', @organization, "Error while pushing to #{Maestrano::Connector::Rails::External.external_name}: #{e}")
+      idmap.update(message: e.message.truncate(255))
     end
+    nil
   end
 
-  def create_external_entity(client, mapped_connec_entity, external_entity_name, organization)
-    Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending create #{external_entity_name}: #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
+  def create_external_entity(mapped_connec_entity, external_entity_name)
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending create #{external_entity_name}: #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
     raise "Not implemented"
   end
 
-  def update_external_entity(client, mapped_connec_entity, external_id, external_entity_name, organization)
-    Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending update #{external_entity_name} (id=#{external_id}): #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
+  def update_external_entity(mapped_connec_entity, external_id, external_entity_name)
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending update #{external_entity_name} (id=#{external_id}): #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
     raise "Not implemented"
   end
 
   # This method is called during the webhook workflow only. It should return the array of filtered entities
   # The aim is to have the same filtering as with the Connec! filters on API calls in the webhooks
-  def filter_connec_entities(entities, organization, opts={})
+  def filter_connec_entities(entities)
     entities
   end
+
   # ----------------------------------------------
   #                 General methods
   # ----------------------------------------------
   # * Discards entities that do not need to be pushed because they have not been updated since their last push
   # * Discards entities from one of the two source in case of conflict
   # * Maps not discarded entities and associates them with their idmap, or create one if there isn't any
-  # * Return a hash {connec_entities: [], external_entities: []}
-  def consolidate_and_map_data(connec_entities, external_entities, organization, opts={})
-    return consolidate_and_map_singleton(connec_entities, external_entities, organization, opts) if self.class.singleton?
+  # * Returns a hash {connec_entities: [], external_entities: []}
+  def consolidate_and_map_data(connec_entities, external_entities)
+    return consolidate_and_map_singleton(connec_entities, external_entities) if self.class.singleton?
+
+    mapped_connec_entities = consolidate_and_map_connec_entities(connec_entities, external_entities, self.class.references, self.class.external_entity_name)
+    mapped_external_entities = consolidate_and_map_external_entities(external_entities, self.class.connec_entity_name)
 
-    mapped_external_entities = external_entities.map{|entity|
-      idmap = self.class.find_idmap({external_id: self.class.id_from_external_entity_hash(entity), organization_id: organization.id})
+    return {connec_entities: mapped_connec_entities, external_entities: mapped_external_entities}
+  end
+
+  def consolidate_and_map_connec_entities(connec_entities, external_entities, references, external_entity_name)
+    connec_entities.map{|entity|
+      entity = Maestrano::Connector::Rails::ConnecHelper.unfold_references(entity, references, @organization)
+      next nil unless entity
+      connec_id = entity.delete(:__connec_id)
 
-      # No idmap: creating one, nothing else to do
-      unless idmap
-        next {entity: map_to_connec(entity, organization), idmap: self.class.create_idmap_from_external_entity(entity, organization)}
+      if entity['id'].blank?
+        idmap = self.class.create_idmap(organization_id: @organization.id, name: self.class.object_name_from_connec_entity_hash(entity), external_entity: external_entity_name.downcase, connec_id: connec_id)
+        next map_connec_entity_with_idmap(entity, external_entity_name, idmap)
       end
 
+      idmap = self.class.find_or_create_idmap(external_id: entity['id'], organization_id: @organization.id, external_entity: external_entity_name.downcase, connec_id: connec_id)
+      idmap.update(name: self.class.object_name_from_connec_entity_hash(entity))
+
+      next nil if idmap.external_inactive || !idmap.to_external || (!@opts[:full_sync] && not_modified_since_last_push_to_external?(idmap, entity))
+
+      # Check for conflict with entities from external
+      solve_conflict(entity, external_entities, external_entity_name, idmap)
+    }.compact
+  end
+
+  def consolidate_and_map_external_entities(external_entities, connec_entity_name)
+    external_entities.map{|entity|
+      entity_id = self.class.id_from_external_entity_hash(entity)
+      idmap = self.class.find_or_create_idmap(external_id: entity_id, organization_id: @organization.id, connec_entity: connec_entity_name.downcase)
+
       # Not pushing entity to Connec!
       next nil unless idmap.to_connec
 
       # Not pushing to Connec! and flagging as inactive if inactive in external application
       inactive = self.class.inactive_from_external_entity_hash?(entity)
-      idmap.update(external_inactive: inactive)
+      idmap.update(external_inactive: inactive, name: self.class.object_name_from_external_entity_hash(entity))
       next nil if inactive
 
       # Entity has not been modified since its last push to connec!
-      next nil if self.class.not_modified_since_last_push_to_connec?(idmap, entity, self, organization)
+      next nil if !@opts[:full_sync] && not_modified_since_last_push_to_connec?(idmap, entity)
 
-      idmap.update(name: self.class.object_name_from_external_entity_hash(entity))
 
-      # Check for conflict with entities from connec!
-      self.class.solve_conflict(entity, self, connec_entities, self.class.connec_entity_name, idmap, organization, opts)
+      map_external_entity_with_idmap(entity, connec_entity_name, idmap)
     }.compact
-
-    mapped_connec_entities = connec_entities.map{|entity|
-      map_to_external_with_idmap(entity, organization)
-    }.compact
-
-    return {connec_entities: mapped_connec_entities, external_entities: mapped_external_entities}
   end
 
-  def consolidate_and_map_singleton(connec_entities, external_entities, organization, opts={})
+  def consolidate_and_map_singleton(connec_entities, external_entities)
     return {connec_entities: [], external_entities: []} if external_entities.empty? && connec_entities.empty?
 
-    idmap = self.class.find_or_create_idmap({organization_id: organization.id})
+    idmap = self.class.find_or_create_idmap({organization_id: @organization.id})
+    # No to_connec, to_external and inactive consideration here as we don't expect those workflow for singleton
 
     if external_entities.empty?
       keep_external = false
     elsif connec_entities.empty?
       keep_external = true
-    elsif !opts[:connec_preemption].nil?
-      keep_external = !opts[:connec_preemption]
+    elsif @opts.has_key?(:connec_preemption)
+      keep_external = !@opts[:connec_preemption]
     else
-      keep_external = self.class.is_external_more_recent?(connec_entities.first, external_entities.first, self)
+      keep_external = !is_connec_more_recent?(connec_entities.first, external_entities.first)
     end
+
     if keep_external
       idmap.update(external_id: self.class.id_from_external_entity_hash(external_entities.first), name: self.class.object_name_from_external_entity_hash(external_entities.first))
-      return {connec_entities: [], external_entities: [{entity: map_to_connec(external_entities.first, organization), idmap: idmap}]}
+      return {connec_entities: [], external_entities: [{entity: map_to_connec(external_entities.first), idmap: idmap}]}
     else
-      idmap.update(connec_id: connec_entities.first['id'], name: self.class.object_name_from_connec_entity_hash(connec_entities.first))
-      return {connec_entities: [{entity: map_to_external(connec_entities.first, organization), idmap: idmap}], external_entities: []}
+      entity = Maestrano::Connector::Rails::ConnecHelper.unfold_references(connec_entities.first, self.class.references, @organization)
+      idmap.update(name: self.class.object_name_from_connec_entity_hash(entity), connec_id: entity.delete(:__connec_id))
+      idmap.update(external_id: self.class.id_from_external_entity_hash(external_entities.first)) unless external_entities.empty?
+      return {connec_entities: [{entity: map_to_external(entity), idmap: idmap}], external_entities: []}
     end
   end
 
   # ----------------------------------------------
   #             After and before sync
   # ----------------------------------------------
-  def before_sync(connec_client, external_client, last_synchronization, organization, opts)
+  def before_sync(last_synchronization)
     # Does nothing by default
   end
 
-  def after_sync(connec_client, external_client, last_synchronization, organization, opts)
+  def after_sync(last_synchronization)
     # Does nothing by default
   end
 
+
   # ----------------------------------------------
   #             Internal helper methods
   # ----------------------------------------------
-  module ClassMethods
-    def not_modified_since_last_push_to_connec?(idmap, entity, entity_instance, organization)
-      result = idmap.last_push_to_connec && idmap.last_push_to_connec > entity_instance.class.last_update_date_from_external_entity_hash(entity)
-      Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Discard #{Maestrano::Connector::Rails::External::external_name} #{entity_instance.class.external_entity_name} : #{entity}") if result
-      result
+  private
+    # array_with_idmap must be an array of hashes with a key idmap
+    # proc is a lambda to create a batch_op from an element of the array
+    def batch_calls(array_with_idmap, proc, connec_entity_name, id_update_only=false)
+      request_per_call = @opts[:request_per_batch_call] || 100
+      start = 0
+      while start < array_with_idmap.size
+        # Prepare batch request
+        batch_entities = array_with_idmap.slice(start, request_per_call)
+        batch_request = {sequential: true, ops: []}
+
+        batch_entities.each do |id|
+          batch_request[:ops] << proc.call(id)
+        end
+
+        # Batch call
+        log_info = id_update_only ? 'with only ids' : ''
+        Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending batch request to Connec! #{log_info} for #{self.class.normalize_connec_entity_name(connec_entity_name)}. Batch_request_size: #{batch_request[:ops].size}. Call_number: #{(start/request_per_call) + 1}")
+        response = @connec_client.batch(batch_request)
+        Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "Received batch response from Connec! for #{self.class.normalize_connec_entity_name(connec_entity_name)}: #{response}")
+        raise "No data received from Connec! when trying to send batch request #{log_info} for #{self.class.connec_entity_name.pluralize}" unless response && !response.body.blank?
+        response = JSON.parse(response.body)
+
+        # Parse batch response
+        response['results'].each_with_index do |result, index|
+          if result['status'] == 200
+            batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find{|id| id['provider'] == 'connec'}['id'], last_push_to_connec: Time.now, message: nil) unless id_update_only # id_update_only only apply for 200 as it's doing PUTs
+          elsif result['status'] == 201
+            batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find{|id| id['provider'] == 'connec'}['id'], last_push_to_connec: Time.now, message: nil)
+          else
+            Maestrano::Connector::Rails::ConnectorLogger.log('error', @organization, "Error while pushing to Connec!: #{result['body']}")
+            batch_entities[index][:idmap].update(message: result['body'].to_s.truncate(255))
+          end
+        end
+        start += request_per_call
+      end
+    end
+    
+    def not_modified_since_last_push_to_connec?(idmap, entity)
+      not_modified = idmap.last_push_to_connec && idmap.last_push_to_connec > self.class.last_update_date_from_external_entity_hash(entity)
+      Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Discard #{Maestrano::Connector::Rails::External::external_name} #{self.class.external_entity_name} : #{entity}") if not_modified
+      not_modified
+    end
+
+    def not_modified_since_last_push_to_external?(idmap, entity)
+      not_modified = idmap.last_push_to_external && idmap.last_push_to_external > entity['updated_at']
+      Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Discard Connec! #{self.class.connec_entity_name} : #{entity}") if not_modified
+      not_modified
     end
 
-    def is_external_more_recent?(connec_entity, external_entity, entity_instance)
-      connec_entity['updated_at'] < entity_instance.class.last_update_date_from_external_entity_hash(external_entity)
+    def is_connec_more_recent?(connec_entity, external_entity)
+      connec_entity['updated_at'] > self.class.last_update_date_from_external_entity_hash(external_entity)
     end
 
-    def solve_conflict(external_entity, entity_instance, connec_entities, connec_entity_name, idmap, organization, opts)
-      if idmap.connec_id && connec_entity = connec_entities.detect{|connec_entity| connec_entity['id'] == idmap.connec_id}
+    def solve_conflict(connec_entity, external_entities, external_entity_name, idmap)
+      if external_entity = external_entities.find{|external_entity| connec_entity['id'] == self.class.id_from_external_entity_hash(external_entity)}
         # We keep the most recently updated entity
-        if !opts[:connec_preemption].nil?
-          keep_external = !opts[:connec_preemption]
+        if @opts.has_key?(:connec_preemption)
+          keep_connec = @opts[:connec_preemption]
         else
-          keep_external = is_external_more_recent?(connec_entity, external_entity, entity_instance)
+          keep_connec = is_connec_more_recent?(connec_entity, external_entity)
         end
 
-        if keep_external
-          Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{entity_instance.class.external_entity_name} #{external_entity} and Connec! #{connec_entity_name} #{connec_entity}. Entity from external kept")
-          connec_entities.delete(connec_entity)
-          entity_instance.map_external_entity_with_idmap(external_entity, connec_entity_name, idmap, organization)
+        if keep_connec
+          Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from external kept")
+          external_entities.delete(external_entity)
+          map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
         else
-          Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{entity_instance.class.external_entity_name} #{external_entity} and Connec! #{connec_entity_name} #{connec_entity}. Entity from Connec! kept")
+          Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from Connec! kept")
           nil
         end
 
       else
-        entity_instance.map_external_entity_with_idmap(external_entity, connec_entity_name, idmap, organization)
+        map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
       end
     end
 
-    def id_from_ref(entity, ref, is_external, organization)
-      # field can be address/billing/country_id
-      field = is_external ? ref[:external_field] : ref[:connec_field]
-      field = field.split('/')
-      id = entity
-      field.each do |f|
-        id &&= id[f]
-      end
+    def map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
+      {entity: map_to_external(connec_entity), idmap: idmap}
+    end
 
-      if is_external
-        idmap = ref[:reference_class].find_idmap({external_id: id, organization_id: organization.id})
-        idmap && idmap.connec_id
-      else
-        idmap = ref[:reference_class].find_idmap({connec_id: id, organization_id: organization.id})
-        idmap && idmap.external_id
-      end
+    def map_external_entity_with_idmap(external_entity, connec_entity_name, idmap)
+      {entity: map_to_connec(external_entity), idmap: idmap}
+    end
+
+    def fetch_connec(uri, page_number)
+      response = @connec_client.get(uri)
+      raise "No data received from Connec! when trying to fetch page #{page_number} of #{self.class.normalized_connec_entity_name}" unless response && !response.body.blank?
+
+      response_hash = JSON.parse(response.body)
+      Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "received first page entity=#{self.class.connec_entity_name}, response=#{response_hash}")
+      raise "Received unrecognized Connec! data when trying to fetch page #{page_number} of #{self.class.normalized_connec_entity_name}: #{response_hash}" unless response_hash["#{self.class.normalized_connec_entity_name}"]
+
+      response_hash
     end
-  end
 
-  def map_external_entity_with_idmap(external_entity, connec_entity_name, idmap, organization)
-    {entity: map_to_connec(external_entity, organization), idmap: idmap}
-  end
 end