maestrano-connector-rails 1.1.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 252ed0e53f7dcb7b820c329f3cfac7f1a6523633
-  data.tar.gz: e3fc619f7c65568d1b4db83e918ffd17b1b26556
+  metadata.gz: 48d5cff94c3992312d69f4ae46493d883e9c752f
+  data.tar.gz: c9fb97913e6f032198ec72fc69bb2351aa220546
 SHA512:
-  metadata.gz: 0ebed3f59cbe4a425b82c2a532244b4f2e7fd3b3feba3b46135b6fc39e40fce131a7ac0837132dbae7abec21c9446e47ce1c0878bbe9cf625e9bd3f3c946f213
-  data.tar.gz: ad5443f7a03bd2c41e12945ed65d7655dffba7e4f6b2602b864cf49c24f0045d73a4f7cc46b157b76a6c79d63a11b12d80d2bda6a75e11560b0c46b5769116fe
+  metadata.gz: 9635d73732f2d8e40e545049ddf45d752b2c725d2660fc03bbbda23fc158fbab0d4445b0721736a96f4596970e9f740fd98b7f3be2efb0bc4296f3a9c68bc40b
+  data.tar.gz: 94486ead1d56581183097dfb52f08498bf0fe093b5b220bfe91b83803007baf6a01a0b80d5d18b71cb5be9ccb075a916681e3f396be09f5adf1494be6e5117b0

data/VERSION

@@ -1 +1 @@
-1.1.2
+1.2.0

data/app/controllers/maestrano/dependancies_controller.rb

@@ -0,0 +1,5 @@
+class Maestrano::DependanciesController < Maestrano::Rails::WebHookController
+  def index
+    render json: Maestrano::Connector::Rails::ConnecHelper.dependancies
+  end
+end

data/app/controllers/maestrano/synchronizations_controller.rb

@@ -0,0 +1,46 @@
+class Maestrano::SynchronizationsController < Maestrano::Rails::WebHookController
+  def show
+    uid = params[:id]
+    organization = Maestrano::Connector::Rails::Organization.find_by_uid(uid)
+    return render json: { errors: {message: "Organization not found", code: 404} }, status: :not_found unless organization
+
+    h = {
+      group_id: organization.uid,
+      sync_enabled: organization.sync_enabled
+    }
+
+    last_sync = organization.synchronizations.last
+    if last_sync
+      h.merge!(
+        {
+          status: last_sync.status,
+          message: last_sync.message,
+          updated_at: last_sync.updated_at
+        }
+      )
+    end
+
+    render json: h
+  end
+
+  def create
+    uid = params[:group_id]
+    opts = params[:opts] || {}
+    organization = Maestrano::Connector::Rails::Organization.find_by_uid(uid)
+    return render json: { errors: {message: "Organization not found", code: 404} }, status: :not_found unless organization
+
+    Maestrano::Connector::Rails::SynchronizationJob.perform_later(organization, opts.with_indifferent_access)
+    head :created
+  end
+
+  def toggle_sync
+    uid = params[:group_id]
+    organization = Maestrano::Connector::Rails::Organization.find_by_uid(uid)
+    return render json: { errors: {message: "Organization not found", code: 404} }, status: :not_found unless organization
+
+    organization.toggle(:sync_enabled)
+    organization.save
+
+    render json: {sync_enabled: organization.sync_enabled}
+  end
+end

data/app/jobs/maestrano/connector/rails/synchronization_job.rb

@@ -35,12 +35,14 @@ module Maestrano::Connector::Rails
 
         # First synchronization should be from external to Connec! only to let the smart merging works
         # We do a doube sync: only from external, then only from connec!
+        # We also do batched sync as the first one can be quite huge
         if last_synchronization.nil?
           ConnectorLogger.log('info', organization, "First synchronization ever. Doing two half syncs to allow smart merging to work its magic.")
-          [{skip_connec: true}, {skip_external: true}].each do |opt|
-            organization.synchronized_entities.select{|k, v| v}.keys.each do |entity|
-              sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization_date, opts.merge(opt))
-            end
+          organization.synchronized_entities.select{|k, v| v}.keys.each do |entity|
+            ConnectorLogger.log('info', organization, "First synchronization ever. Doing half sync from external for #{entity}.")
+            first_sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization_date, opts, true)
+            ConnectorLogger.log('info', organization, "First synchronization ever. Doing half sync from Connec! for #{entity}.")
+            first_sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization_date, opts, false)
           end
         elsif opts[:only_entities]
           ConnectorLogger.log('info', organization, "Synchronization is partial and will synchronize only #{opts[:only_entities].join(' ')}")
@@ -64,15 +66,50 @@ module Maestrano::Connector::Rails
     end
 
     def sync_entity(entity_name, organization, connec_client, external_client, last_synchronization_date, opts)
-      entity_instance = "Entities::#{entity_name.titleize.split.join}".constantize.new(organization, connec_client, external_client, opts.dup)
+      entity_instance = instanciate_entity(entity_name, organization, connec_client, external_client, opts)
+
+      perform_sync(entity_instance, last_synchronization_date)
+    end
+
+    # Does a batched sync on either external or connec!
+    def first_sync_entity(entity_name, organization, connec_client, external_client, last_synchronization_date, opts, external = true)
+      limit = Settings.first_sync_batch_size || 50
+      skip = 0
+      entities_count = limit
+
+      h = {__limit: limit}
+      external ? h.merge!(__skip_connec: true) : h.merge!(__skip_external: true)
+      entity_instance = instanciate_entity(entity_name, organization, connec_client, external_client, opts.merge(h))
 
-      entity_instance.before_sync(last_synchronization_date)
-      external_entities = entity_instance.get_external_entities_wrapper(last_synchronization_date)
-      connec_entities = entity_instance.get_connec_entities(last_synchronization_date)
-      mapped_entities = entity_instance.consolidate_and_map_data(connec_entities, external_entities)
-      entity_instance.push_entities_to_external(mapped_entities[:connec_entities])
-      entity_instance.push_entities_to_connec(mapped_entities[:external_entities])
-      entity_instance.after_sync(last_synchronization_date)
+      # IF entities_count > limit
+      # This first sync feature is probably not implemented in the connector
+      # because it fetched more than the expected number of entities
+      # No need to fetch it a second Time
+      # ELSIF entities_count < limit
+      # No more entities to fetch
+      while entities_count == limit do
+        entity_instance.opts_merge!(__skip: skip)
+        entities_count = perform_sync(entity_instance, last_synchronization_date, external)
+        skip += limit
+      end
     end
+
+    private
+      def instanciate_entity(entity_name, organization, connec_client, external_client, opts)
+        "Entities::#{entity_name.titleize.split.join}".constantize.new(organization, connec_client, external_client, opts.dup)
+      end
+
+      # Perform the sync and return the entities_count for either external or connec
+      def perform_sync(entity_instance, last_synchronization_date, external = true)
+        entity_instance.before_sync(last_synchronization_date)
+        external_entities = entity_instance.get_external_entities_wrapper(last_synchronization_date)
+        connec_entities = entity_instance.get_connec_entities(last_synchronization_date)
+        mapped_entities = entity_instance.consolidate_and_map_data(connec_entities, external_entities)
+        entity_instance.push_entities_to_external(mapped_entities[:connec_entities])
+        entity_instance.push_entities_to_connec(mapped_entities[:external_entities])
+        entity_instance.after_sync(last_synchronization_date)
+        
+        external ? entity_instance.class.count_entities(external_entities) : entity_instance.class.count_entities(connec_entities)
+      end
   end
 end

data/app/models/maestrano/connector/rails/complex_entity.rb

@@ -1,5 +1,5 @@
 module Maestrano::Connector::Rails
-  class ComplexEntity
+  class ComplexEntity < EntityBase
     include Maestrano::Connector::Rails::Concerns::ComplexEntity
   end
 end

data/app/models/maestrano/connector/rails/concerns/complex_entity.rb

@@ -1,13 +1,6 @@
 module Maestrano::Connector::Rails::Concerns::ComplexEntity
   extend ActiveSupport::Concern
 
-  def initialize(organization, connec_client, external_client, opts={})
-    @organization = organization
-    @connec_client = connec_client
-    @external_client = external_client
-    @opts = opts
-  end
-
   # -------------------------------------------------------------
   #                   Complex specific methods
   # Those methods needs to be implemented in each complex entity
@@ -20,6 +13,10 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
     def external_entities_names
       raise "Not implemented"
     end
+
+    def count_entities(entities)
+      entities.values.map(&:size).max
+    end
   end
 
   # input :  {
@@ -141,24 +138,6 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
     end
   end
 
-  def before_sync(last_synchronization_date)
-    # Does nothing by default
-  end
-
-  def after_sync(last_synchronization_date)
-    # Does nothing by default
-  end
-
-  # This method is called during the webhook workflow only. It should return the hash of arrays of filtered entities
-  # The aim is to have the same filtering as with the Connec! filters on API calls in the webhooks
-  # input :  {
-  #             external_entities_names[0]: [unmapped_external_entity1}, unmapped_external_entity2],
-  #             external_entities_names[1]: [unmapped_external_entity3}, unmapped_external_entity4]
-  #          }
-  def filter_connec_entities(entities)
-    entities
-  end
-
   def instantiate_sub_entity_instance(entity_name)
     "Entities::SubEntities::#{entity_name.titleize.split.join}".constantize.new(@organization, @connec_client, @external_client, @opts)
   end

data/app/models/maestrano/connector/rails/concerns/connec_helper.rb

@@ -2,12 +2,29 @@ module Maestrano::Connector::Rails::Concerns::ConnecHelper
   extend ActiveSupport::Concern
 
   module ClassMethods
+    def dependancies
+      # Meant to be overloaded if needed
+      {
+        connec: '1.0',
+        impac: '1.0',
+        maestrano_hub: '1.0'
+      }
+    end
 
     def get_client(organization)
       client = Maestrano::Connec::Client[organization.tenant].new(organization.uid)
       client.class.headers('CONNEC-EXTERNAL-IDS' => 'true')
       client
     end
+
+    def connec_version(organization)
+      @@connec_version = Rails.cache.fetch('connec_version', namespace: 'maestrano', expires_in: 1.day) do
+        response = get_client(organization).class.get("#{Maestrano[organization.tenant].param('connec.host')}/version")
+        response = JSON.parse(response.body)
+        @@connec_version = response['ci_branch']
+      end
+      @@connec_version
+    end
     
     # Replace the ids arrays by the external id
     # If a reference has no id for this oauth_provider and oauth_uid but has one for connec returns nil

data/app/models/maestrano/connector/rails/concerns/entity.rb

@@ -1,13 +1,6 @@
 module Maestrano::Connector::Rails::Concerns::Entity
   extend ActiveSupport::Concern
 
-  def initialize(organization, connec_client, external_client, opts={})
-    @organization = organization
-    @connec_client = connec_client
-    @external_client = external_client
-    @opts = opts
-  end
-
   module ClassMethods
     # ----------------------------------------------
     #                 IdMap methods
@@ -131,8 +124,21 @@ module Maestrano::Connector::Rails::Concerns::Entity
     def can_update_external?
       true
     end
+
+    # ----------------------------------------------
+    #                 Helper methods
+    # ----------------------------------------------
+    def count_entities(entities)
+      entities.size
+    end
   end
 
+  # ==============================================
+  # ==============================================
+  #                 Instance  methods
+  # ==============================================
+  # ==============================================
+
   # ----------------------------------------------
   #                 Mapper methods
   # ----------------------------------------------
@@ -159,7 +165,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
   # * $filter (see Connec! documentation)
   # * $orderby (see Connec! documentation)
   def get_connec_entities(last_synchronization_date=nil)
-    return [] if @opts[:skip_connec] || !self.class.can_read_connec?
+    return [] if @opts[:__skip_connec] || !self.class.can_read_connec?
 
     Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching Connec! #{self.class.connec_entity_name}")
 
@@ -169,6 +175,13 @@ module Maestrano::Connector::Rails::Concerns::Entity
 
     # Fetch first page
     page_number = 0
+
+    batched_fetch = @opts[:__limit] && @opts[:__skip]
+    if batched_fetch
+      query_params[:$top] = @opts[:__limit]
+      query_params[:$skip] = @opts[:__skip]
+    end
+
     if last_synchronization_date.blank? || @opts[:full_sync]
       query_params[:$filter] = @opts[:$filter] if @opts[:$filter]
     else
@@ -181,14 +194,17 @@ module Maestrano::Connector::Rails::Concerns::Entity
     entities = response_hash["#{self.class.normalized_connec_entity_name}"]
     entities = [entities] if self.class.singleton?
 
-    # Fetch subsequent pages
-    while response_hash['pagination'] && response_hash['pagination']['next']
-      page_number += 1
-      # ugly way to convert https://api-connec/api/v2/group_id/organizations?next_page_params to /organizations?next_page_params
-      next_page = response_hash['pagination']['next'].gsub(/^(.*)\/#{self.class.normalized_connec_entity_name}/, self.class.normalized_connec_entity_name)
+    # Only the first page if batched_fetch
+    unless batched_fetch
+      # Fetch subsequent pages
+      while response_hash['pagination'] && response_hash['pagination']['next']
+        page_number += 1
+        # ugly way to convert https://api-connec/api/v2/group_id/organizations?next_page_params to /organizations?next_page_params
+        next_page = response_hash['pagination']['next'].gsub(/^(.*)\/#{self.class.normalized_connec_entity_name}/, self.class.normalized_connec_entity_name)
 
-      response_hash = fetch_connec(next_page, page_number)
-      entities << response_hash["#{self.class.normalized_connec_entity_name}"]
+        response_hash = fetch_connec(next_page, page_number)
+        entities << response_hash["#{self.class.normalized_connec_entity_name}"]
+      end
     end
 
     entities.flatten!
@@ -224,7 +240,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
   #                 External methods
   # ----------------------------------------------
   def get_external_entities_wrapper(last_synchronization_date=nil)
-    return [] if @opts[:skip_external] || !self.class.can_read_external?
+    return [] if @opts[:__skip_external] || !self.class.can_read_external?
     get_external_entities(last_synchronization_date)
   end
   
@@ -299,12 +315,6 @@ module Maestrano::Connector::Rails::Concerns::Entity
     raise "Not implemented"
   end
 
-  # This method is called during the webhook workflow only. It should return the array of filtered entities
-  # The aim is to have the same filtering as with the Connec! filters on API calls in the webhooks
-  def filter_connec_entities(entities)
-    entities
-  end
-
   # ----------------------------------------------
   #                 General methods
   # ----------------------------------------------
@@ -313,6 +323,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
   # * Maps not discarded entities and associates them with their idmap, or create one if there isn't any
   # * Returns a hash {connec_entities: [], external_entities: []}
   def consolidate_and_map_data(connec_entities, external_entities)
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Consolidating and mapping #{self.class.external_entity_name}/#{self.class.connec_entity_name}")
     return consolidate_and_map_singleton(connec_entities, external_entities) if self.class.singleton?
 
     mapped_connec_entities = consolidate_and_map_connec_entities(connec_entities, external_entities, self.class.references, self.class.external_entity_name)
@@ -395,18 +406,6 @@ module Maestrano::Connector::Rails::Concerns::Entity
     end
   end
 
-  # ----------------------------------------------
-  #             After and before sync
-  # ----------------------------------------------
-  def before_sync(last_synchronization_date)
-    # Does nothing by default
-  end
-
-  def after_sync(last_synchronization_date)
-    # Does nothing by default
-  end
-
-
   # ----------------------------------------------
   #             Internal helper methods
   # ----------------------------------------------

data/app/models/maestrano/connector/rails/concerns/entity_base.rb

@@ -0,0 +1,34 @@
+module Maestrano::Connector::Rails::Concerns::EntityBase
+  extend ActiveSupport::Concern
+
+  def initialize(organization, connec_client, external_client, opts={})
+    @organization = organization
+    @connec_client = connec_client
+    @external_client = external_client
+    @opts = opts
+  end
+
+  def opts_merge!(opts)
+    @opts.merge!(opts)
+  end
+
+  def before_sync(last_synchronization_date)
+    # Does nothing by default
+  end
+
+  def after_sync(last_synchronization_date)
+    # Does nothing by default
+  end
+
+  # This method is called during the webhook workflow only. It should return the hash of arrays of filtered entities
+  # The aim is to have the same filtering as with the Connec! filters on API calls in the webhooks
+  # input :  
+  # For non complex entities [unmapped_external_entity1, unmapped_external_entity2]
+  # For complex entities {
+  #   external_entities_names[0]: [unmapped_external_entity1, unmapped_external_entity2],
+  #   external_entities_names[1]: [unmapped_external_entity3, unmapped_external_entity4]
+  # }
+  def filter_connec_entities(entities)
+    entities
+  end
+end

data/app/models/maestrano/connector/rails/entity.rb

@@ -1,5 +1,5 @@
 module Maestrano::Connector::Rails
-  class Entity
+  class Entity < EntityBase
     include Maestrano::Connector::Rails::Concerns::Entity
   end
-end
+end

data/app/models/maestrano/connector/rails/entity_base.rb

@@ -0,0 +1,5 @@
+module Maestrano::Connector::Rails
+  class EntityBase
+    include Maestrano::Connector::Rails::Concerns::EntityBase
+  end
+end

data/config/routes.rb

@@ -4,5 +4,12 @@ Maestrano::Connector::Rails::Engine.routes.draw do
   namespace :maestrano do
   	match 'signout', to: 'sessions#destroy', as: 'signout', via: [:get, :post]
     post 'connec/notifications/:tenant' => 'connec#notifications'
+
+    resources :synchronizations, only: [:show, :create] do
+      collection do
+        put :toggle_sync
+      end
+    end
+    resources :dependancies, only: [:index]
   end
 end

data/lib/generators/connector/templates/entity.rb

@@ -1,4 +1,4 @@
-class Maestrano::Connector::Rails::Entity
+class Maestrano::Connector::Rails::Entity < Maestrano::Connector::Rails::EntityBase
   include Maestrano::Connector::Rails::Concerns::Entity
 
   # In this class and in all entities which inherit from it, the following instance variables are available:
@@ -13,6 +13,7 @@ class Maestrano::Connector::Rails::Entity
     # TODO
     # This method should return only entities that have been updated since the last_synchronization_date
     # It should also implements an option to do a full synchronization when @opts[:full_sync] == true or when there is no last_synchronization_date
+    # It should also support [:__limit] and [:__skip] opts, meaning that if they are present, it should return only @[:__limit] entities while skipping the @opts[:__skip] firsts
     # Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Received data: Source=#{Maestrano::Connector::Rails::External.external_name}, Entity=#{self.class.external_entity_name}, Response=#{entities}")
   end
 

data/maestrano-connector-rails.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: maestrano-connector-rails 1.1.2 ruby lib
+# stub: maestrano-connector-rails 1.2.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "maestrano-connector-rails"
-  s.version = "1.1.2"
+  s.version = "1.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Pierre Berard"]
-  s.date = "2016-06-22"
+  s.date = "2016-07-01"
   s.description = "Maestrano is the next generation marketplace for SME applications. See https://maestrano.com for details."
   s.email = "pierre.berard@maestrano.com"
   s.executables = ["rails"]
@@ -32,7 +32,9 @@ Gem::Specification.new do |s|
     "app/controllers/maestrano/application_controller.rb",
     "app/controllers/maestrano/auth/saml_controller.rb",
     "app/controllers/maestrano/connec_controller.rb",
+    "app/controllers/maestrano/dependancies_controller.rb",
     "app/controllers/maestrano/sessions_controller.rb",
+    "app/controllers/maestrano/synchronizations_controller.rb",
     "app/helpers/maestrano/connector/rails/session_helper.rb",
     "app/jobs/maestrano/connector/rails/all_synchronizations_job.rb",
     "app/jobs/maestrano/connector/rails/push_to_connec_job.rb",
@@ -42,11 +44,13 @@ Gem::Specification.new do |s|
     "app/models/maestrano/connector/rails/concerns/connec_helper.rb",
     "app/models/maestrano/connector/rails/concerns/connector_logger.rb",
     "app/models/maestrano/connector/rails/concerns/entity.rb",
+    "app/models/maestrano/connector/rails/concerns/entity_base.rb",
     "app/models/maestrano/connector/rails/concerns/external.rb",
     "app/models/maestrano/connector/rails/concerns/sub_entity_base.rb",
     "app/models/maestrano/connector/rails/connec_helper.rb",
     "app/models/maestrano/connector/rails/connector_logger.rb",
     "app/models/maestrano/connector/rails/entity.rb",
+    "app/models/maestrano/connector/rails/entity_base.rb",
     "app/models/maestrano/connector/rails/external.rb",
     "app/models/maestrano/connector/rails/id_map.rb",
     "app/models/maestrano/connector/rails/organization.rb",
@@ -102,8 +106,10 @@ Gem::Specification.new do |s|
     "maestrano.png",
     "release_notes.md",
     "spec/controllers/connec_controller_spec.rb",
+    "spec/controllers/dependancies_controller_spec.rb",
     "spec/controllers/group_users_controller_spec.rb",
     "spec/controllers/groups_controller_spec.rb",
+    "spec/controllers/synchronizations_controller_spec.rb",
     "spec/dummy/README.md",
     "spec/dummy/Rakefile",
     "spec/dummy/app/assets/images/.keep",
@@ -118,9 +124,7 @@ Gem::Specification.new do |s|
     "spec/dummy/app/mailers/.keep",
     "spec/dummy/app/models/.keep",
     "spec/dummy/app/models/concerns/.keep",
-    "spec/dummy/app/models/entities/example_entitiy.rb",
-    "spec/dummy/app/models/maestrano/connector/rails/entity.rb",
-    "spec/dummy/app/models/maestrano/connector/rails/external.rb",
+    "spec/dummy/app/models/entities/.keep",
     "spec/dummy/app/views/home/index.html.erb",
     "spec/dummy/app/views/layouts/application.html.erb",
     "spec/dummy/bin/bundle",
@@ -170,6 +174,7 @@ Gem::Specification.new do |s|
     "spec/models/complex_entity_spec.rb",
     "spec/models/connec_helper_spec.rb",
     "spec/models/connector_logger_spec.rb",
+    "spec/models/entity_base_spec.rb",
     "spec/models/entity_spec.rb",
     "spec/models/external_spec.rb",
     "spec/models/id_map_spec.rb",

data/spec/controllers/dependancies_controller_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe Maestrano::DependanciesController, type: :controller do
+  routes { Maestrano::Connector::Rails::Engine.routes }
+
+  describe 'index' do
+    subject { get :index }
+
+    context 'without authentication' do
+      before {
+        controller.class.before_filter :authenticate_maestrano!
+      }
+
+      it 'respond with unauthorized' do
+        subject
+        expect(response.status).to eq(401)
+      end
+    end
+
+    context 'with authentication' do
+      before {
+        controller.class.skip_before_filter :authenticate_maestrano!
+      }
+
+      it 'renders the dependancies hash' do
+        subject
+        expect(response.body).to eql(Maestrano::Connector::Rails::ConnecHelper.dependancies.to_json)
+      end
+    end
+  end
+end

data/spec/controllers/synchronizations_controller_spec.rb

@@ -0,0 +1,179 @@
+require 'spec_helper'
+
+describe Maestrano::SynchronizationsController, type: :controller do
+  routes { Maestrano::Connector::Rails::Engine.routes }
+
+  let(:uid) { 'cld-aaaa' }
+
+  describe 'show' do
+    subject { get :show, id: uid }
+
+
+    context 'without authentication' do
+      before {
+        controller.class.before_filter :authenticate_maestrano!
+      }
+
+      it 'respond with unauthorized' do
+        subject
+        expect(response.status).to eq(401)
+      end
+    end
+
+    context 'with authentication' do
+      before {
+        controller.class.skip_before_filter :authenticate_maestrano!
+      }
+
+      context 'when organization is not found' do
+        let!(:organization) { create(:organization, uid: 'cld-bbbb') }
+
+        it 'is a 404' do
+          subject
+          expect(response.status).to eq(404)
+        end
+      end
+
+      context 'when organization is found' do
+        let!(:organization) { create(:organization, uid: uid) }
+
+        it 'is a success' do
+          subject
+          expect(response.status).to eq(200)
+        end
+
+        context 'with no last sync' do
+          it 'renders a partial json' do
+            subject
+            expect(JSON.parse(response.body)).to eql(
+              JSON.parse({
+                group_id: organization.uid,
+                sync_enabled: organization.sync_enabled,
+              }.to_json)
+            )
+          end
+        end
+
+        context 'with a last sync' do
+          let!(:sync1) { create(:synchronization, organization: organization) }
+          let!(:sync2) { create(:synchronization, organization: organization, message: 'msg') }
+
+          it 'renders a full json' do
+            subject
+            expect(JSON.parse(response.body)).to eql(
+              JSON.parse({
+                group_id: organization.uid,
+                sync_enabled: organization.sync_enabled,
+                status: sync2.status,
+                message: sync2.message,
+                updated_at: sync2.updated_at   
+              }.to_json)
+            )
+          end
+        end
+      end
+    end
+  end
+
+  describe 'create' do
+    let(:opts) { {'only_entities' => ['customer']} }
+    subject { post :create, group_id: uid, opts: opts }
+
+
+    context 'without authentication' do
+      before {
+        controller.class.before_filter :authenticate_maestrano!
+      }
+
+      it 'respond with unauthorized' do
+        subject
+        expect(response.status).to eq(401)
+      end
+    end
+
+    context 'with authentication' do
+      before {
+        controller.class.skip_before_filter :authenticate_maestrano!
+      }
+
+      context 'when organization is not found' do
+        let!(:organization) { create(:organization, uid: 'cld-bbbb') }
+
+        it 'is a 404' do
+          subject
+          expect(response.status).to eq(404)
+        end
+      end
+
+      context 'when organization is found' do
+        let!(:organization) { create(:organization, uid: uid) }
+
+        it 'is a success' do
+          subject
+          expect(response.status).to eq(201)
+        end
+
+        it 'queues a sync' do
+          expect(Maestrano::Connector::Rails::SynchronizationJob).to receive(:perform_later).with(organization, opts)
+          subject
+        end
+      end
+    end
+  end
+
+  describe 'toggle_sync' do
+    subject { put :toggle_sync, group_id: uid }
+
+
+    context 'without authentication' do
+      before {
+        controller.class.before_filter :authenticate_maestrano!
+      }
+
+      it 'respond with unauthorized' do
+        subject
+        expect(response.status).to eq(401)
+      end
+    end
+
+    context 'with authentication' do
+      before {
+        controller.class.skip_before_filter :authenticate_maestrano!
+      }
+
+      context 'when organization is not found' do
+        let!(:organization) { create(:organization, uid: 'cld-bbbb') }
+
+        it 'is a 404' do
+          subject
+          expect(response.status).to eq(404)
+        end
+      end
+
+      context 'when organization is found' do
+        let!(:organization) { create(:organization, uid: uid) }
+
+        it 'is a success' do
+          subject
+          expect(response.status).to eq(200)
+        end
+
+        context 'when sync_enabled is true' do
+          before { organization.update(sync_enabled: true) }
+          it 'disables the organizatio syncs' do
+            subject
+            expect(organization.reload.sync_enabled).to be false
+          end
+        end
+
+        context 'when  sync_enabled is false' do
+          before { organization.update(sync_enabled: false) }
+          it 'enables the organizatio syncs' do
+            subject
+            expect(organization.reload.sync_enabled).to be true
+          end
+        end
+      end
+    end
+  end
+end

data/spec/jobs/synchronization_job_spec.rb

@@ -20,7 +20,7 @@ describe Maestrano::Connector::Rails::SynchronizationJob do
     end
 
     context 'with sync_enabled set to true' do
-      before {organization.update(sync_enabled: true)}
+      before { organization.update(sync_enabled: true)} 
 
       context 'with a sync still running for less than 30 minutes' do
         let!(:running_sync) { create(:synchronization, organization: organization, status: 'RUNNING', created_at: 29.minutes.ago) }
@@ -67,17 +67,18 @@ describe Maestrano::Connector::Rails::SynchronizationJob do
         end
       end
 
-      it { performes }
 
       context 'first sync' do
         it 'does two half syncs' do
-          expect_any_instance_of(Maestrano::Connector::Rails::SynchronizationJob).to receive(:sync_entity).exactly(2 * organization.synchronized_entities.count).times
+          expect_any_instance_of(Maestrano::Connector::Rails::SynchronizationJob).to receive(:first_sync_entity).exactly(2 * organization.synchronized_entities.count).times
           subject
         end
       end
 
       context 'subsequent sync' do
         let!(:old_sync) { create(:synchronization, partial: false, status: 'SUCCESS', organization: organization) }
+        
+        it { performes }
 
         it 'calls sync entity on all the organization synchronized entities set to true' do
           organization.synchronized_entities[organization.synchronized_entities.keys.first] = false
@@ -107,42 +108,93 @@ describe Maestrano::Connector::Rails::SynchronizationJob do
     end
   end
 
-  describe 'sync_entity' do
+  describe 'other methods' do
     subject { Maestrano::Connector::Rails::SynchronizationJob.new }
 
-    context 'non complex entity' do
-      before {
-        class Entities::Person < Maestrano::Connector::Rails::Entity
+    describe 'sync_entity' do
+
+      context 'non complex entity' do
+        before {
+          class Entities::Person < Maestrano::Connector::Rails::Entity
+          end
+        }
+
+        it 'calls the seven methods' do
+          expect_any_instance_of(Entities::Person).to receive(:before_sync)
+          expect_any_instance_of(Entities::Person).to receive(:get_connec_entities).and_return([])
+          expect_any_instance_of(Entities::Person).to receive(:get_external_entities_wrapper).and_return([])
+          expect_any_instance_of(Entities::Person).to receive(:consolidate_and_map_data).and_return({})
+          expect_any_instance_of(Entities::Person).to receive(:push_entities_to_external)
+          expect_any_instance_of(Entities::Person).to receive(:push_entities_to_connec)
+          expect_any_instance_of(Entities::Person).to receive(:after_sync)
+          subject.sync_entity('person', organization, nil, nil, nil, {})
+        end
+      end
+
+      context 'complex entity' do
+        before {
+          class Entities::SomeStuff < Maestrano::Connector::Rails::ComplexEntity
+          end
+        }
+
+        it 'calls the seven methods' do
+          expect_any_instance_of(Entities::SomeStuff).to receive(:before_sync)
+          expect_any_instance_of(Entities::SomeStuff).to receive(:get_connec_entities).and_return({})
+          expect_any_instance_of(Entities::SomeStuff).to receive(:get_external_entities_wrapper).and_return({})
+          expect_any_instance_of(Entities::SomeStuff).to receive(:consolidate_and_map_data).and_return({})
+          expect_any_instance_of(Entities::SomeStuff).to receive(:push_entities_to_external)
+          expect_any_instance_of(Entities::SomeStuff).to receive(:push_entities_to_connec)
+          expect_any_instance_of(Entities::SomeStuff).to receive(:after_sync)
+          subject.sync_entity('some stuff', organization, nil, nil, nil, {})
         end
-      }
-
-      it 'calls the seven methods' do
-        expect_any_instance_of(Entities::Person).to receive(:before_sync)
-        expect_any_instance_of(Entities::Person).to receive(:get_connec_entities)
-        expect_any_instance_of(Entities::Person).to receive(:get_external_entities_wrapper)
-        expect_any_instance_of(Entities::Person).to receive(:consolidate_and_map_data).and_return({})
-        expect_any_instance_of(Entities::Person).to receive(:push_entities_to_external)
-        expect_any_instance_of(Entities::Person).to receive(:push_entities_to_connec)
-        expect_any_instance_of(Entities::Person).to receive(:after_sync)
-        subject.sync_entity('person', organization, nil, nil, nil, {})
       end
     end
 
-    context 'complex entity' do
-      before {
-        class Entities::SomeStuff < Maestrano::Connector::Rails::ComplexEntity
+    describe 'first_sync_entity' do
+      let(:batch_limit) { 50 }
+
+      context 'non complex entity' do
+        before {
+          class Entities::Person < Maestrano::Connector::Rails::Entity
+          end
+          allow_any_instance_of(Entities::Person).to receive(:get_connec_entities).and_return([])
+          allow_any_instance_of(Entities::Person).to receive(:get_external_entities_wrapper).and_return([])
+          allow_any_instance_of(Entities::Person).to receive(:consolidate_and_map_data).and_return({})
+          allow_any_instance_of(Entities::Person).to receive(:push_entities_to_external)
+          allow_any_instance_of(Entities::Person).to receive(:push_entities_to_connec)
+        }
+
+        context 'with pagination' do
+          context 'with more than 50 entities' do
+            it 'calls perform_sync several time' do
+              allow(subject).to receive(:perform_sync).and_return(batch_limit, 0)
+              expect_any_instance_of(Entities::Person).to receive(:opts_merge!).twice
+              expect(subject).to receive(:perform_sync).twice
+              
+              subject.first_sync_entity('person', organization, nil, nil, nil, {}, true)
+            end
+          end
+
+          context 'with less than 50 entities' do
+            it 'calls perform_sync once' do
+              allow(subject).to receive(:perform_sync).and_return(batch_limit - 10)
+              expect_any_instance_of(Entities::Person).to receive(:opts_merge!).once.with({__skip: 0})
+              expect(subject).to receive(:perform_sync).once
+              
+              subject.first_sync_entity('person', organization, nil, nil, nil, {}, true)
+            end
+          end
+        end
+
+        context 'without pagination' do
+          it 'calls perform_sync once' do
+            allow(subject).to receive(:perform_sync).and_return(batch_limit + 10)
+            expect_any_instance_of(Entities::Person).to receive(:opts_merge!).once.with({__skip: 0})
+            expect(subject).to receive(:perform_sync).once
+            
+            subject.first_sync_entity('person', organization, nil, nil, nil, {}, true)
+          end
         end
-      }
-
-      it 'calls the seven methods' do
-        expect_any_instance_of(Entities::SomeStuff).to receive(:before_sync)
-        expect_any_instance_of(Entities::SomeStuff).to receive(:get_connec_entities)
-        expect_any_instance_of(Entities::SomeStuff).to receive(:get_external_entities_wrapper)
-        expect_any_instance_of(Entities::SomeStuff).to receive(:consolidate_and_map_data).and_return({})
-        expect_any_instance_of(Entities::SomeStuff).to receive(:push_entities_to_external)
-        expect_any_instance_of(Entities::SomeStuff).to receive(:push_entities_to_connec)
-        expect_any_instance_of(Entities::SomeStuff).to receive(:after_sync)
-        subject.sync_entity('some stuff', organization, nil, nil, nil, {})
       end
     end
   end

data/spec/models/complex_entity_spec.rb

@@ -11,6 +11,17 @@ describe Maestrano::Connector::Rails::ComplexEntity do
       describe 'external_entities_names' do
         it { expect{ subject.external_entities_names }.to raise_error('Not implemented') }
       end
+
+      describe 'count_entities' do
+        it 'returns the biggest array size' do
+          entities = {
+            'people' => [*1..27],
+            'organizations' => [*1..39],
+            'items' => []
+          }
+          expect(subject.count_entities(entities)).to eql(39)
+        end
+      end
   end
 
   describe 'instance methods' do

data/spec/models/connec_helper_spec.rb

@@ -5,6 +5,28 @@ describe Maestrano::Connector::Rails::ConnecHelper do
 
   let!(:organization) { create(:organization) }
 
+  describe 'dependancies' do
+    it 'returns a default hash' do
+      expect(subject.dependancies).to eql({
+        connec: '1.0',
+        impac: '1.0',
+        maestrano_hub: '1.0'
+      })
+    end
+  end
+
+  describe 'connec_version' do
+    let!(:organization) { create(:organization, tenant: 'default') }
+    before {
+      allow(Maestrano::Connec::Client[organization.tenant]).to receive(:get).and_return(ActionDispatch::Response.new(200, {}, {ci_build_number: '111', ci_branch: '1.1', ci_commit: '111'}.to_json, {}))
+    }
+
+    it 'returns the connec_version' do
+      expect(subject.connec_version(organization)).to eql('1.1')
+    end
+
+  end
+
   describe 'unfold_references' do
     let(:connec_hash) {
       {

data/spec/models/entity_base_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Maestrano::Connector::Rails::EntityBase do
+  describe 'instance methods' do
+    let!(:organization) { create(:organization, uid: 'cld-123') }
+    let!(:connec_client) { Maestrano::Connec::Client[organization.tenant].new(organization.uid) }
+    let!(:external_client) { Object.new }
+    let(:opts) { {} }
+    subject { Maestrano::Connector::Rails::EntityBase.new(organization, connec_client, external_client, opts) }
+
+    describe 'opts_merge!' do
+      let(:opts) { {a: 1, opts: 2} }
+
+      it 'merges options with the instance variable' do
+        subject.opts_merge!(opts: 3, test: 'test')
+        expect(subject.instance_variable_get(:@opts)).to eql(a: 1, opts: 3, test: 'test')
+      end
+    end
+  end
+end

data/spec/models/entity_spec.rb

@@ -110,6 +110,12 @@ describe Maestrano::Connector::Rails::Entity do
     describe 'connec_matching_fields' do
       it { expect(subject.connec_matching_fields).to be_nil }
     end
+
+    describe 'count_entities' do
+      it 'returns the array size' do
+        expect(subject.count_entities([*1..27])).to eql(27)
+      end
+    end
   end
 
   describe 'instance methods' do
@@ -189,7 +195,7 @@ describe Maestrano::Connector::Rails::Entity do
         end
 
         describe 'when skip_connec' do
-          let(:opts) { {skip_connec: true} }
+          let(:opts) { {__skip_connec: true} }
           it { expect(subject.get_connec_entities(nil)).to eql([]) }
         end
 
@@ -211,6 +217,19 @@ describe Maestrano::Connector::Rails::Entity do
               allow(connec_client).to receive(:get).and_return(ActionDispatch::Response.new(200, {}, {people: []}.to_json, {}))
             }
 
+            context 'with limit and skip opts' do
+              let(:opts) { {__skip: 100, __limit: 50} }
+              before {
+                allow(connec_client).to receive(:get).and_return(ActionDispatch::Response.new(200, {}, {people: [], pagination: {next: "https://api-connec.maestrano.com/api/v2/cld-dkg601/people?%24skip=10&%24top=10"}}.to_json, {}), ActionDispatch::Response.new(200, {}, {people: []}.to_json, {}))
+              }
+
+              it 'performs a size limited date and do not paginate' do
+                uri_param = {"$filter" => "updated_at gt '#{sync.updated_at.iso8601}'", "$skip" => 100, "$top" => 50}.to_query
+                expect(connec_client).to receive(:get).once.with("#{connec_name.downcase.pluralize}?#{uri_param}")
+                subject.get_connec_entities(sync.updated_at)
+              end
+            end
+
             context 'when opts[:full_sync] is true' do
               let(:opts) { {full_sync: true} }
               it 'performs a full get' do
@@ -474,7 +493,7 @@ describe Maestrano::Connector::Rails::Entity do
         end
 
         context 'when skip external' do
-          let(:opts) { {skip_external: true} }
+          let(:opts) { {__skip_external: true} }
 
           it 'returns an empty array and does not call get_external_entities' do
             expect(subject).to_not receive(:get_connec_entities)

data/spec/models/external_spec.rb

@@ -3,17 +3,21 @@ require 'spec_helper'
 describe Maestrano::Connector::Rails::External do
   subject { Maestrano::Connector::Rails::External }
 
+  before {
+    allow(Maestrano::Connector::Rails::External).to receive(:external_name).and_call_original
+    allow(Maestrano::Connector::Rails::External).to receive(:get_client).and_call_original
+    allow(Maestrano::Connector::Rails::External).to receive(:entities_list).and_call_original
+  }
+
   describe 'external_name' do
-    it { expect(subject.external_name).to eql('Dummy app') }
+    it { expect{ subject.external_name }.to raise_error(RuntimeError) }
   end
 
   describe 'get_client' do
-    let(:organization) { create(:organization) }
-
-    it { expect(subject.get_client(organization)).to eql(nil) }
+    it { expect{ subject.get_client(nil) }.to raise_error(RuntimeError) }
   end
 
   describe 'entities_list' do
-    it { expect(subject.entities_list).to eql(%w(entity1 entity2))}
+    it { expect{ subject.entities_list }.to raise_error(RuntimeError) }
   end
 end

data/spec/models/sub_entity_base_spec.rb

@@ -76,7 +76,6 @@ describe Maestrano::Connector::Rails::SubEntityBase do
     let!(:connec_client) { Maestrano::Connec::Client[organization.tenant].new(organization.uid) }
     let!(:external_client) { Object.new }
     let(:opts) { {} }
-    subject { Maestrano::Connector::Rails::Entity.new(organization, connec_client, external_client, opts) }
     subject { Maestrano::Connector::Rails::SubEntityBase.new(organization, connec_client, external_client, opts) }
 
     describe 'map_to' do

data/spec/spec_helper.rb

@@ -20,4 +20,10 @@ RSpec.configure do |config|
   config.order = "random"
   config.include FactoryGirl::Syntax::Methods
   config.include Maestrano::Connector::Rails::Engine.routes.url_helpers
-end
+
+  config.before(:each) do
+    allow(Maestrano::Connector::Rails::External).to receive(:external_name).and_return('External app')
+    allow(Maestrano::Connector::Rails::External).to receive(:get_client).and_return(Object.new)
+    allow(Maestrano::Connector::Rails::External).to receive(:entities_list).and_return(%w(entity1 entity2))
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maestrano-connector-rails
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.0
 platform: ruby
 authors:
 - Pierre Berard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-22 00:00:00.000000000 Z
+date: 2016-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: maestrano-rails
@@ -284,7 +284,9 @@ files:
 - app/controllers/maestrano/application_controller.rb
 - app/controllers/maestrano/auth/saml_controller.rb
 - app/controllers/maestrano/connec_controller.rb
+- app/controllers/maestrano/dependancies_controller.rb
 - app/controllers/maestrano/sessions_controller.rb
+- app/controllers/maestrano/synchronizations_controller.rb
 - app/helpers/maestrano/connector/rails/session_helper.rb
 - app/jobs/maestrano/connector/rails/all_synchronizations_job.rb
 - app/jobs/maestrano/connector/rails/push_to_connec_job.rb
@@ -294,11 +296,13 @@ files:
 - app/models/maestrano/connector/rails/concerns/connec_helper.rb
 - app/models/maestrano/connector/rails/concerns/connector_logger.rb
 - app/models/maestrano/connector/rails/concerns/entity.rb
+- app/models/maestrano/connector/rails/concerns/entity_base.rb
 - app/models/maestrano/connector/rails/concerns/external.rb
 - app/models/maestrano/connector/rails/concerns/sub_entity_base.rb
 - app/models/maestrano/connector/rails/connec_helper.rb
 - app/models/maestrano/connector/rails/connector_logger.rb
 - app/models/maestrano/connector/rails/entity.rb
+- app/models/maestrano/connector/rails/entity_base.rb
 - app/models/maestrano/connector/rails/external.rb
 - app/models/maestrano/connector/rails/id_map.rb
 - app/models/maestrano/connector/rails/organization.rb
@@ -354,8 +358,10 @@ files:
 - maestrano.png
 - release_notes.md
 - spec/controllers/connec_controller_spec.rb
+- spec/controllers/dependancies_controller_spec.rb
 - spec/controllers/group_users_controller_spec.rb
 - spec/controllers/groups_controller_spec.rb
+- spec/controllers/synchronizations_controller_spec.rb
 - spec/dummy/README.md
 - spec/dummy/Rakefile
 - spec/dummy/app/assets/images/.keep
@@ -370,9 +376,7 @@ files:
 - spec/dummy/app/mailers/.keep
 - spec/dummy/app/models/.keep
 - spec/dummy/app/models/concerns/.keep
-- spec/dummy/app/models/entities/example_entitiy.rb
-- spec/dummy/app/models/maestrano/connector/rails/entity.rb
-- spec/dummy/app/models/maestrano/connector/rails/external.rb
+- spec/dummy/app/models/entities/.keep
 - spec/dummy/app/views/home/index.html.erb
 - spec/dummy/app/views/layouts/application.html.erb
 - spec/dummy/bin/bundle
@@ -422,6 +426,7 @@ files:
 - spec/models/complex_entity_spec.rb
 - spec/models/connec_helper_spec.rb
 - spec/models/connector_logger_spec.rb
+- spec/models/entity_base_spec.rb
 - spec/models/entity_spec.rb
 - spec/models/external_spec.rb
 - spec/models/id_map_spec.rb

data/spec/dummy/app/models/entities/example_entitiy.rb

@@ -1,26 +0,0 @@
-# TODO
-# This file is provided as an example and should be removed
-# One such file needs to be created for each synchronizable entity,
-# with its associated mapper
-
-# class Entities::ExampleEntity < Maestrano::Connector::Rails::Entity
-#   def connec_entity_name
-#     'ExampleEntity'
-#   end
-
-#   def external_entity_name
-#     'Contact'
-#   end
-
-#   def mapper_class
-#     ExampleEntityMapper
-#   end
-# end
-
-# class ExampleEntityMapper < Maestrano::Connector::Rails::GenericMapper
-#   extend HashMapper
-
-#   map from('/title'), to('/Salutation')
-#   map from('/first_name'), to('/FirstName')
-#   map from('address_work/billing2/city'), to('City')
-# end

data/spec/dummy/app/models/maestrano/connector/rails/entity.rb

@@ -1,3 +0,0 @@
-class Maestrano::Connector::Rails::Entity
-  include Maestrano::Connector::Rails::Concerns::Entity
-end

data/spec/dummy/app/models/maestrano/connector/rails/external.rb

@@ -1,18 +0,0 @@
-class Maestrano::Connector::Rails::External
-  include Maestrano::Connector::Rails::Concerns::External
-
-  def self.external_name
-    'Dummy app'
-  end
-
-  def self.get_client(organization)
-    nil
-  end
-
-  # Return an array of all the entities that the connector can synchronize
-  # If you add new entities, you need to generate
-  # a migration to add them to existing organizations
-  def self.entities_list
-    %w(entity1 entity2)
-  end
-end