maestrano-connector-rails 0.4.4 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e2b30b01e9635bb23e54086aa72a76209ff6ff90
-  data.tar.gz: 093c8366dae9dd669fd6989f95e8030c44698d1b
+  metadata.gz: 2ee22c8ca44f84ce0e4a717c80892f1765af5987
+  data.tar.gz: 65e99c4163462829142e439f8323e8e2cd6f6306
 SHA512:
-  metadata.gz: c4e022328533a25167c5c3d37b4c146edf6a9d88bec6d952209a3a52d3b156f7262800bdc7148ba8a4320272a4e39720704adb57529ccb294cfda08e5fd623a1
-  data.tar.gz: cd8609ffcf02458ea2b97419076ec7e521a28fa9fde702d2ce7c6bd9fecf773c6cc63dc9fb3a708f8e5bfb3f46132984d1ce71b2226715b31172f8c2b5c196cd
+  metadata.gz: 5a480181d0599bc16c945364ff6949c684eaa431f89b1c5af2f41f7f8999f451ab48a470abe680a1d854f57284fa6831bd73b3ecaf0b503a2053b34674f3a4b9
+  data.tar.gz: 00ec463ca23d1994baaf7873ce4a26c8c3cc76c72809e5a1b06cba7ca54899f67ed5dcc92c97f019ac4fbf6363d42862edf770c579245fb42ce30ae488fbdcc5

data/Gemfile

@@ -7,6 +7,8 @@ gem 'sidekiq'
 gem 'haml-rails'
 gem 'bootstrap-sass'
 gem 'autoprefixer-rails'
+# gem 'attr_encrypted', '~> 3.0.0'
+# gem 'config'
 
 # Add dependencies to develop your gem here.
 group :development do

data/README.md

@@ -10,4 +10,4 @@
 Maestrano Connector is a Rails Engine that bootstraps the implementation of data syncrhonization between an external application API and the Maestrano ecosystem.
 Maestrano Connector Integration is currently in closed beta. Want to know more? Send us an email to <contact@maestrano.com>.
 
-You can find the documentation [here](https://maestrano.atlassian.net/wiki/display/CONNECAPIV2/Maestrano+connector+framework+-+rails). See also some generic documentation about integration with Maestrano [here](https://maestrano.atlassian.net/wiki/display/CONNECAPIV2/Maestrano+Developers).
+You can find the documentation [here](https://maestrano.atlassian.net/wiki/display/DEV/Create+a+connector+using+our+framework+-+Ruby+on+Rails). See also some generic documentation about integration with Maestrano [here](https://maestrano.atlassian.net/wiki/display/DEV/Maestrano+Developers).

data/VERSION

@@ -1 +1 @@
-0.4.4
+1.0.0

data/app/controllers/maestrano/connec_controller.rb

@@ -6,36 +6,34 @@ class Maestrano::ConnecController < Maestrano::Rails::WebHookController
     begin
       params.except(:tenant, :controller, :action, :connec).each do |entity_name, entities|
 
-        entity_instance_hash = find_entity_instance(entity_name)
-        next Rails.logger.info "Received notification from Connec! for unknow entity: #{entity_name}" unless entity_instance_hash
-
-        entity_instance = entity_instance_hash[:instance]
+        entity_class_hash = find_entity_class(entity_name)
+        next Rails.logger.info "Received notification from Connec! for unknow entity: #{entity_name}" unless entity_class_hash
 
         entities.each do |entity|
           organization = Maestrano::Connector::Rails::Organization.find_by_uid_and_tenant(entity[:group_id], params[:tenant])
           next Rails.logger.warn "Received notification from Connec! for unknown group or group without oauth: #{entity['group_id']} (tenant: #{params[:tenant]})" unless organization && organization.oauth_uid
-          next unless organization.sync_enabled && organization.synchronized_entities[entity_instance_hash[:name].to_sym]
+          next unless organization.sync_enabled && organization.synchronized_entities[entity_class_hash[:name].to_sym]
 
 
           Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Received entity from Connec! webhook: Entity=#{entity_name}, Data=#{entity}")
-          connec_client = Maestrano::Connec::Client[organization.tenant].new(organization.uid)
+          connec_client = Maestrano::Connector::Rails::ConnecHelper.get_client(organization)
           external_client = Maestrano::Connector::Rails::External.get_client(organization)
           last_synchronization = organization.last_successful_synchronization
 
-          entity_instance.before_sync(connec_client, external_client, last_synchronization, organization, {})
+          entity_instance = entity_class_hash[:class].new(organization, connec_client, external_client, {})
+          entity_instance.before_sync(last_synchronization)
           
-
           # Build expected input for consolidate_and_map_data
-          if entity_instance_hash[:is_complex]
-            filtered_entities = entity_instance.filter_connec_entities(Hash[ *entity_instance.class.connec_entities_names.collect{|name| name.parameterize('_').pluralize == entity_name ? [name, [entity]] : [ name, []]}.flatten(1) ], organization)
-            mapped_entity = entity_instance.consolidate_and_map_data(filtered_entities, Hash[ *entity_instance.class.external_entities_names.collect{|name| [ name, []]}.flatten(1) ], organization, {})
+          if entity_class_hash[:is_complex]
+            filtered_entities = entity_instance.filter_connec_entities(Maestrano::Connector::Rails::ComplexEntity.build_hash_with_entities(entity_instance.class.connec_entities_names, entity_name, lambda{|name| name.parameterize('_').pluralize}, [entity]))
+            mapped_entity = entity_instance.consolidate_and_map_data(filtered_entities, Maestrano::Connector::Rails::ComplexEntity.build_empty_hash(entity_instance.class.external_entities_names))
           else
-            filtered_entities = entity_instance.filter_connec_entities([entity], organization)
-            mapped_entity = entity_instance.consolidate_and_map_data(filtered_entities, [], organization, {})
+            filtered_entities = entity_instance.filter_connec_entities([entity])
+            mapped_entity = entity_instance.consolidate_and_map_data(filtered_entities, [])
           end
-          entity_instance.push_entities_to_external(external_client, mapped_entity[:connec_entities], organization)
+          entity_instance.push_entities_to_external(mapped_entity[:connec_entities])
 
-          entity_instance.after_sync(connec_client, external_client, last_synchronization, organization, {})
+          entity_instance.after_sync(last_synchronization)
         end
       end
     rescue => e
@@ -47,13 +45,13 @@ class Maestrano::ConnecController < Maestrano::Rails::WebHookController
 
 
   private
-    def find_entity_instance(entity_name)
-      Maestrano::Connector::Rails::Entity.entities_list.each do |entity_name_from_list|
+    def find_entity_class(entity_name)
+      Maestrano::Connector::Rails::External.entities_list.each do |entity_name_from_list|
         clazz = "Entities::#{entity_name_from_list.singularize.titleize.split.join}".constantize
         if clazz.methods.include?('connec_entities_names'.to_sym)
-          return {instance: clazz.new, is_complex: true, name: entity_name_from_list} if clazz.connec_entities_names.map{|n| n.parameterize('_').pluralize}.include?(entity_name)
+          return {class: clazz, is_complex: true, name: entity_name_from_list} if clazz.connec_entities_names.map{|n| n.parameterize('_').pluralize}.include?(entity_name)
         elsif clazz.methods.include?('connec_entity_name'.to_sym)
-          return {instance: clazz.new, is_complex: false, name: entity_name_from_list} if clazz.normalized_connec_entity_name == entity_name
+          return {class: clazz, is_complex: false, name: entity_name_from_list} if clazz.normalized_connec_entity_name == entity_name
         end
       end
       nil

data/app/jobs/maestrano/connector/rails/all_synchronizations_job.rb

@@ -4,7 +4,7 @@ module Maestrano::Connector::Rails
 
     # Trigger synchronization of all active organizations
     def perform(name=nil, count=nil)
-      Maestrano::Connector::Rails::Organization.where("oauth_provider IS NOT NULL AND oauth_token IS NOT NULL").each do |o|
+      Maestrano::Connector::Rails::Organization.where('oauth_provider IS NOT NULL AND oauth_token IS NOT NULL').each do |o|
         next unless [true, 1].include?(o.sync_enabled)
         Maestrano::Connector::Rails::SynchronizationJob.perform_later(o, {})
       end

data/app/jobs/maestrano/connector/rails/push_to_connec_job.rb

@@ -6,26 +6,26 @@ module Maestrano::Connector::Rails
     def perform(organization, entities_hash, opts={})
       return unless organization.sync_enabled && organization.oauth_uid
 
-      connec_client = Maestrano::Connec::Client[organization.tenant].new(organization.uid)
+      connec_client = Maestrano::Connector::Rails::ConnecHelper.get_client(organization)
       external_client = Maestrano::Connector::Rails::External.get_client(organization)
       last_synchronization = organization.last_successful_synchronization
 
       entities_hash.each do |external_entity_name, entities|
-        if entity_instance_hash = find_entity_instance(external_entity_name)
+        if entity_instance_hash = find_entity_instance(external_entity_name, organization, connec_client, external_client, opts)
           next unless organization.synchronized_entities[entity_instance_hash[:name].to_sym]
 
           entity_instance = entity_instance_hash[:instance]
 
-          entity_instance.before_sync(connec_client, external_client, last_synchronization, organization, opts)
+          entity_instance.before_sync(last_synchronization)
           # Build expected input for consolidate_and_map_data
           if entity_instance_hash[:is_complex]
-            mapped_entities = entity_instance.consolidate_and_map_data(Hash[ *entity_instance.class.connec_entities_names.collect{|name| [ name, []]}.flatten(1) ], Hash[ *entity_instance.class.external_entities_names.collect{|name| name == external_entity_name ? [name, entities] : [ name, []]}.flatten(1) ], organization, {})
+            mapped_entities = entity_instance.consolidate_and_map_data(ComplexEntity.build_empty_hash(entity_instance.class.connec_entities_names), ComplexEntity.build_hash_with_entities(entity_instance.class.external_entities_names, external_entity_name, lambda{|name| name}, entities))
           else
-            mapped_entities = entity_instance.consolidate_and_map_data([], entities, organization, {})
+            mapped_entities = entity_instance.consolidate_and_map_data([], entities)
           end
-          entity_instance.push_entities_to_connec(connec_client, mapped_entities[:external_entities], organization)
+          entity_instance.push_entities_to_connec(mapped_entities[:external_entities])
 
-          entity_instance.after_sync(connec_client, external_client, last_synchronization, organization, opts)
+          entity_instance.after_sync(last_synchronization)
         else
           Rails.logger.warn "Called push to connec job with unknow entity: #{external_entity_name}"
         end
@@ -33,13 +33,13 @@ module Maestrano::Connector::Rails
     end
 
     private
-      def find_entity_instance(entity_name)
-        Maestrano::Connector::Rails::Entity.entities_list.each do |entity_name_from_list|
+      def find_entity_instance(entity_name, organization, connec_client, external_client, opts)
+        Maestrano::Connector::Rails::External.entities_list.each do |entity_name_from_list|
           clazz = "Entities::#{entity_name_from_list.singularize.titleize.split.join}".constantize
           if clazz.methods.include?('external_entities_names'.to_sym)
-            return {instance: clazz.new, is_complex: true, name: entity_name_from_list} if clazz.external_entities_names.include?(entity_name)
+            return {instance: clazz.new(organization, connec_client, external_client, opts), is_complex: true, name: entity_name_from_list} if clazz.external_entities_names.include?(entity_name)
           elsif clazz.methods.include?('external_entity_name'.to_sym)
-            return {instance: clazz.new, is_complex: false, name: entity_name_from_list} if clazz.external_entity_name == entity_name
+            return {instance: clazz.new(organization, connec_client, external_client, opts), is_complex: false, name: entity_name_from_list} if clazz.external_entity_name == entity_name
           end
         end
         nil

data/app/jobs/maestrano/connector/rails/synchronization_job.rb

@@ -7,7 +7,7 @@ module Maestrano::Connector::Rails
     #  * :only_entities => [person, tasks_list]
     #  * :full_sync => true  synchronization is performed without date filtering
     #  * :connec_preemption => true|false : preemption is always|never given to connec in case of conflict (if not set, the most recently updated entity is kept)
-    def perform(organization, opts)
+    def perform(organization, opts={})
       return unless organization.sync_enabled
 
       # Check if previous synchronization is still running
@@ -29,10 +29,19 @@ module Maestrano::Connector::Rails
 
       begin
         last_synchronization = organization.last_successful_synchronization
-        connec_client = Maestrano::Connec::Client[organization.tenant].new(organization.uid)
+        connec_client = ConnecHelper.get_client(organization)
         external_client = External.get_client(organization)
 
-        if opts[:only_entities]
+        # First synchronization should be from external to Connec! only to let the smart merging works
+        # We do a doube sync: only from external, then only from connec!
+        if last_synchronization.nil?
+          ConnectorLogger.log('info', organization, "First synchronization ever. Doing two half syncs to allow smart merging to work its magic.")
+          [{skip_connec: true}, {skip_external: true}].each do |opt|
+            organization.synchronized_entities.select{|k, v| v}.keys.each do |entity|
+              sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization, opts.merge(opt))
+            end
+          end
+        elsif opts[:only_entities]
           ConnectorLogger.log('info', organization, "Synchronization is partial and will synchronize only #{opts[:only_entities].join(' ')}")
           # The synchronization is marked as partial and will not be considered as the last-synchronization for the next sync
           current_synchronization.set_partial
@@ -54,15 +63,15 @@ module Maestrano::Connector::Rails
     end
 
     def sync_entity(entity_name, organization, connec_client, external_client, last_synchronization, opts)
-      entity_instance = "Entities::#{entity_name.titleize.split.join}".constantize.new
+      entity_instance = "Entities::#{entity_name.titleize.split.join}".constantize.new(organization, connec_client, external_client, opts)
 
-      entity_instance.before_sync(connec_client, external_client, last_synchronization, organization, opts)
-      external_entities = entity_instance.get_external_entities(external_client, last_synchronization, organization, opts)
-      connec_entities = entity_instance.get_connec_entities(connec_client, last_synchronization, organization, opts)
-      mapped_entities = entity_instance.consolidate_and_map_data(connec_entities, external_entities, organization, opts)
-      entity_instance.push_entities_to_external(external_client, mapped_entities[:connec_entities], organization)
-      entity_instance.push_entities_to_connec(connec_client, mapped_entities[:external_entities], organization)
-      entity_instance.after_sync(connec_client, external_client, last_synchronization, organization, opts)
+      entity_instance.before_sync(last_synchronization)
+      external_entities = entity_instance.get_external_entities_wrapper(last_synchronization)
+      connec_entities = entity_instance.get_connec_entities(last_synchronization)
+      mapped_entities = entity_instance.consolidate_and_map_data(connec_entities, external_entities)
+      entity_instance.push_entities_to_external(mapped_entities[:connec_entities])
+      entity_instance.push_entities_to_connec(mapped_entities[:external_entities])
+      entity_instance.after_sync(last_synchronization)
     end
   end
 end

data/app/models/maestrano/connector/rails/concerns/complex_entity.rb

@@ -1,6 +1,13 @@
 module Maestrano::Connector::Rails::Concerns::ComplexEntity
   extend ActiveSupport::Concern
 
+  def initialize(organization, connec_client, external_client, opts={})
+    @organization = organization
+    @connec_client = connec_client
+    @external_client = external_client
+    @opts = opts
+  end
+
   # -------------------------------------------------------------
   #                   Complex specific methods
   # Those methods needs to be implemented in each complex entity
@@ -28,7 +35,7 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
   #               external_entities_names[1]: [unmapped_connec_entitiy4]
   #             }
   #          }
-  def connec_model_to_external_model(connec_hash_of_entities, organization)
+  def connec_model_to_external_model(connec_hash_of_entities)
     raise "Not implemented"
   end
 
@@ -45,100 +52,65 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
   #               connec_entities_names[0]: [unmapped_external_entity3, unmapped_external_entity4]
   #             }
   #           }
-  def external_model_to_connec_model(external_hash_of_entities, organization)
+  def external_model_to_connec_model(external_hash_of_entities)
     raise "Not implemented"
   end
 
-  # -------------------------------------------------------------
-  #          General methods
-  # -------------------------------------------------------------
-  def map_to_external_with_idmap(entity, organization, external_entity_name, sub_entity_instance)
-    idmap = sub_entity_instance.class.find_idmap({connec_id: entity['id'], external_entity: external_entity_name.downcase, organization_id: organization.id})
-
-    if idmap
-      return nil if idmap.external_inactive || !idmap.to_external
-
-      if idmap.last_push_to_external && idmap.last_push_to_external > entity['updated_at']
-        Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Discard Connec! #{sub_entity_instance.class.entity_name} : #{entity}")
-        nil
-      else
-        idmap.update(name: sub_entity_instance.class.object_name_from_connec_entity_hash(entity))
-        {entity: sub_entity_instance.map_to(external_entity_name, entity, organization), idmap: idmap}
-      end
-    else
-      {entity: sub_entity_instance.map_to(external_entity_name, entity, organization), idmap: sub_entity_instance.class.create_idmap_from_connec_entity(entity, external_entity_name, organization)}
-    end
-  end
-
   # -------------------------------------------------------------
   #          Entity equivalent methods
   # -------------------------------------------------------------
-  def get_connec_entities(client, last_synchronization, organization, opts={})
+  def get_connec_entities(last_synchronization)
     entities = ActiveSupport::HashWithIndifferentAccess.new
 
     self.class.connec_entities_names.each do |connec_entity_name|
-      sub_entity_instance = "Entities::SubEntities::#{connec_entity_name.titleize.split.join}".constantize.new
-      entities[connec_entity_name] = sub_entity_instance.get_connec_entities(client, last_synchronization, organization, opts)
+      sub_entity_instance = instantiate_sub_entity_instance(connec_entity_name)
+      entities[connec_entity_name] = sub_entity_instance.get_connec_entities(last_synchronization)
     end
     entities
   end
 
-  def get_external_entities(client, last_synchronization, organization, opts={})
+  def get_external_entities_wrapper(last_synchronization)
     entities = ActiveSupport::HashWithIndifferentAccess.new
 
     self.class.external_entities_names.each do |external_entity_name|
-      sub_entity_instance = "Entities::SubEntities::#{external_entity_name.titleize.split.join}".constantize.new
-      entities[external_entity_name] = sub_entity_instance.get_external_entities(client, last_synchronization, organization, opts)
+      sub_entity_instance = instantiate_sub_entity_instance(external_entity_name)
+      entities[external_entity_name] = sub_entity_instance.get_external_entities_wrapper(last_synchronization)
     end
     entities
   end
 
-  def consolidate_and_map_data(connec_entities, external_entities, organization, opts={})
-    modeled_external_entities = external_model_to_connec_model(external_entities, organization)
-    modeled_connec_entities = connec_model_to_external_model(connec_entities, organization)
+  def consolidate_and_map_data(connec_entities, external_entities)
+    modeled_external_entities = external_model_to_connec_model(external_entities)
+    modeled_connec_entities = connec_model_to_external_model(connec_entities)
 
-    modeled_external_entities.each do |external_entity_name, entities_in_connec_model|
-      entities_in_connec_model.each do |connec_entity_name, entities|
-        sub_entity_instance = "Entities::SubEntities::#{external_entity_name.titleize.split.join}".constantize.new
-
-        entities.map!{|entity|
-          idmap = sub_entity_instance.class.find_idmap(external_id: sub_entity_instance.class.id_from_external_entity_hash(entity), connec_entity: connec_entity_name.downcase, organization_id: organization.id)
-
-          # No idmap: creating one, nothing else to do
-          unless idmap
-            next {entity: sub_entity_instance.map_to(connec_entity_name, entity, organization), idmap: sub_entity_instance.class.create_idmap_from_external_entity(entity, connec_entity_name, organization)}
-          end
-
-          # Not pushing entity to Connec!
-          next nil unless idmap.to_connec
+    mapped_connec_entities = consolidate_and_map_connec_entities(modeled_connec_entities, modeled_external_entities)
+    mapped_external_entities = consolidate_and_map_external_entities(modeled_external_entities)
 
-          # Not pushing to Connec! and flagging as inactive if inactive in external application
-          inactive = sub_entity_instance.class.inactive_from_external_entity_hash?(entity)
-          idmap.update(external_inactive: inactive)
-          next nil if inactive
+    return {connec_entities: mapped_connec_entities, external_entities: mapped_external_entities}
+  end
 
-          # Entity has not been modified since its last push to connec!
-          next nil if Maestrano::Connector::Rails::Entity.not_modified_since_last_push_to_connec?(idmap, entity, sub_entity_instance, organization)
+  def consolidate_and_map_connec_entities(modeled_connec_entities, modeled_external_entities)
+    modeled_connec_entities.each do |connec_entity_name, entities_in_external_model|
+      entities_in_external_model.each do |external_entity_name, entities|
 
-          idmap.update(name: sub_entity_instance.class.object_name_from_external_entity_hash(entity))
+        sub_entity_instance = instantiate_sub_entity_instance(connec_entity_name)
+        equivalent_external_entities = (modeled_external_entities[external_entity_name] && modeled_external_entities[external_entity_name][connec_entity_name]) || []
 
-          # Check for conflict with entities from connec!
-          equivalent_connec_entities = modeled_connec_entities[connec_entity_name][external_entity_name] || []
-          Maestrano::Connector::Rails::Entity.solve_conflict(entity, sub_entity_instance, equivalent_connec_entities, connec_entity_name, idmap, organization, opts)
-        }.compact!
+        entities_in_external_model[external_entity_name] = sub_entity_instance.consolidate_and_map_connec_entities(entities, equivalent_external_entities, sub_entity_instance.class.references[external_entity_name] || [], external_entity_name)
       end
     end
+    modeled_connec_entities
+  end
 
-    modeled_connec_entities.each do |connec_entity_name, entities_in_external_model|
-      entities_in_external_model.each do |external_entity_name, entities|
-        sub_entity_instance = "Entities::SubEntities::#{connec_entity_name.titleize.split.join}".constantize.new
-        entities.map!{|entity|
-          self.map_to_external_with_idmap(entity, organization, external_entity_name, sub_entity_instance)
-        }.compact!
+  def consolidate_and_map_external_entities(modeled_external_entities)
+    modeled_external_entities.each do |external_entity_name, entities_in_connec_model|
+      entities_in_connec_model.each do |connec_entity_name, entities|
+        sub_entity_instance = instantiate_sub_entity_instance(external_entity_name)
+
+        entities_in_connec_model[connec_entity_name] = sub_entity_instance.consolidate_and_map_external_entities(entities, connec_entity_name)
       end
     end
-
-    return {connec_entities: modeled_connec_entities, external_entities: modeled_external_entities}
+    modeled_external_entities
   end
 
   # input : {
@@ -150,30 +122,30 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
   #               connec_entities_names[0]: [mapped_external_entity3, mapped_external_entity4]
   #             }
   #          }
-  def push_entities_to_connec(connec_client, mapped_external_entities_with_idmaps, organization)
+  def push_entities_to_connec(mapped_external_entities_with_idmaps)
     mapped_external_entities_with_idmaps.each do |external_entity_name, entities_in_connec_model|
-      sub_entity_instance = "Entities::SubEntities::#{external_entity_name.titleize.split.join}".constantize.new
+      sub_entity_instance = instantiate_sub_entity_instance(external_entity_name)
       entities_in_connec_model.each do |connec_entity_name, mapped_entities_with_idmaps|
-        sub_entity_instance.push_entities_to_connec_to(connec_client, mapped_entities_with_idmaps, connec_entity_name, organization)
+        sub_entity_instance.push_entities_to_connec_to(mapped_entities_with_idmaps, connec_entity_name)
       end
     end
   end
 
 
-  def push_entities_to_external(external_client, mapped_connec_entities_with_idmaps, organization)
+  def push_entities_to_external(mapped_connec_entities_with_idmaps)
     mapped_connec_entities_with_idmaps.each do |connec_entity_name, entities_in_external_model|
-      sub_entity_instance = "Entities::SubEntities::#{connec_entity_name.titleize.split.join}".constantize.new
+      sub_entity_instance = instantiate_sub_entity_instance(connec_entity_name)
       entities_in_external_model.each do |external_entity_name, mapped_entities_with_idmaps|
-        sub_entity_instance.push_entities_to_external_to(external_client, mapped_entities_with_idmaps, external_entity_name, organization)
+        sub_entity_instance.push_entities_to_external_to(mapped_entities_with_idmaps, external_entity_name)
       end
     end
   end
 
-  def before_sync(connec_client, external_client, last_synchronization, organization, opts)
+  def before_sync(last_synchronization)
     # Does nothing by default
   end
 
-  def after_sync(connec_client, external_client, last_synchronization, organization, opts)
+  def after_sync(last_synchronization)
     # Does nothing by default
   end
 
@@ -183,7 +155,27 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
   #             external_entities_names[0]: [unmapped_external_entity1}, unmapped_external_entity2],
   #             external_entities_names[1]: [unmapped_external_entity3}, unmapped_external_entity4]
   #          }
-  def filter_connec_entities(entities, organization, opts={})
+  def filter_connec_entities(entities)
     entities
   end
+
+  def instantiate_sub_entity_instance(entity_name)
+    "Entities::SubEntities::#{entity_name.titleize.split.join}".constantize.new(@organization, @connec_client, @external_client, @opts)
+  end
+
+  # -------------------------------------------------------------
+  #                      Helper methods
+  # -------------------------------------------------------------
+  module ClassMethods
+    # output : {entities_names[0] => [], entities_names[1] => []}
+    def build_empty_hash(entities_names)
+      Hash[ *entities_names.collect{|name| [ name, []]}.flatten(1) ]
+    end
+
+    # output: {entities_name[0] => [], entities_name[1] => entities}
+    # with proc.call(entities_name[1] == entity_name)
+    def build_hash_with_entities(entities_name, entity_name, proc, entities)
+      Hash[ *entities_name.collect{|name| proc.call(name) == entity_name ? [name, entities] : [ name, []]}.flatten(1) ]
+    end
+  end
 end

data/app/models/maestrano/connector/rails/concerns/connec_helper.rb

@@ -0,0 +1,102 @@
+module Maestrano::Connector::Rails::Concerns::ConnecHelper
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+
+    def get_client(organization)
+      client = Maestrano::Connec::Client[organization.tenant].new(organization.uid)
+      client.class.headers('CONNEC-EXTERNAL-IDS' => 'true')
+      client
+    end
+    
+    # Replace the ids arrays by the external id
+    # If a reference has no id for this oauth_provider and oauth_uid but has one for connec returns nil
+    def unfold_references(connec_entity, references, organization)
+      unfolded_connec_entity = connec_entity.with_indifferent_access
+      not_nil = true
+
+      # Id
+      id = unfolded_connec_entity['id'].find{|id| id['provider'] == organization.oauth_provider && id['realm'] == organization.oauth_uid}
+      unfolded_connec_entity[:__connec_id] = unfolded_connec_entity['id'].find{|id| id['provider'] == 'connec'}['id']
+      if id
+        unfolded_connec_entity['id'] = id['id']
+      else
+        unfolded_connec_entity['id'] = nil
+      end
+
+      # Other refs
+      references.each do |reference|
+        not_nil &&= unfold_references_helper(unfolded_connec_entity, reference.split('/'), organization)
+      end
+      not_nil ? unfolded_connec_entity : nil
+    end
+
+    def fold_references(mapped_external_entity, references, organization)
+      mapped_external_entity = mapped_external_entity.with_indifferent_access
+      (references + ['id']).each do |reference|
+        fold_references_helper(mapped_external_entity, reference.split('/'), organization)
+      end
+      mapped_external_entity
+    end
+
+    def id_hash(id, organization)
+      {
+        id: id,
+        provider: organization.oauth_provider,
+        realm: organization.oauth_uid
+      }
+    end
+
+    def fold_references_helper(entity, array_of_refs, organization)
+      ref = array_of_refs.shift
+      field = entity[ref]
+
+      # Follow embedment path, remplace if it's a string
+      unless field.blank?
+        case field
+        when Array
+          field.each do |f|
+            fold_references_helper(f, array_of_refs.dup, organization)
+          end
+        when HashWithIndifferentAccess
+          fold_references_helper(entity[ref], array_of_refs, organization)
+        when String
+          id = field
+          entity[ref] = [id_hash(id, organization)]
+        end
+      end
+    end
+
+    def unfold_references_helper(entity, array_of_refs, organization)
+      ref = array_of_refs.shift
+      field = entity[ref]
+
+      # Unfold the id
+      if array_of_refs.empty?
+        if id = field && field.find{|id| id[:provider] == organization.oauth_provider && id[:realm] == organization.oauth_uid}
+          entity[ref] = id['id']
+        elsif field && field.find{|id| id[:provider] == 'connec'}
+          # We may enqueue a fetch on the endpoint of the missing association, followed by a re-fetch on this one.
+          # However it's expected to be an edge case, so for now we rely on the fact that the webhooks should be relativly in order.
+          # Worst case it'll be done on following sync
+          return nil
+        end
+
+      # Follow embedment path
+      else
+        unless field.blank?
+          case field
+          when Array
+            field.each do |f|
+              unfold_references_helper(f, array_of_refs.dup, organization)
+            end
+          when HashWithIndifferentAccess
+            unfold_references_helper(entity[ref], array_of_refs, organization)
+          end
+        end
+      end
+      true
+    end
+
+  end
+end