maestrano-connector-rails 1.0.4 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4b5dba94b691f6b3af1ac41b6a7d0398ee5396af
-  data.tar.gz: dc4611f5da086cff9fc297b0d0d0eba7aa4e5c8d
+  metadata.gz: d686a6327d19c9928de390fdff97b8a49314ee12
+  data.tar.gz: 24766745312ecf5dbdc9cabd262602ecaee2883f
 SHA512:
-  metadata.gz: 4b2683f1f4283965f1f4a4c81be839aaa72d734432aa6246d21d2dbf15e90be0161d11a8e863584863531c15af823a2375f38acd667a212733793d47fc32ad57
-  data.tar.gz: d7931fb298a3b542c4a9878ccb6e5b2969efa4b67eb11c47aef601b50ff681d3539d013ed25e3934c129f55a9bb3b992a269fa81e7031fe75c8dbbda5e5f7088
+  metadata.gz: 44eeb799a05babf52afa93d910dcabb1addc680609c9ad18b5347f2dbe5055e5e9d7edd317013c032691f84570865d2d9d83b16a732273777b48e919218fd9ee
+  data.tar.gz: 5f2e00125ac1c7e482015f705fff22f689ee1b5eade8eeb3c27031dbad863ab989972851e399c97384c3b7fc5d533564405b543748c424602b3f8ce7ac77a67a

data/Gemfile

@@ -7,8 +7,8 @@ gem 'sidekiq'
 gem 'haml-rails'
 gem 'bootstrap-sass'
 gem 'autoprefixer-rails'
-# gem 'attr_encrypted', '~> 3.0.0'
-# gem 'config'
+gem 'attr_encrypted', '~> 1.4.0'
+gem 'config'
 
 # Add dependencies to develop your gem here.
 group :development do
@@ -21,4 +21,5 @@ group :development do
   gem 'factory_girl_rails'
   gem 'sqlite3'
   gem 'shoulda-matchers'
+  gem 'timecop'
 end

data/LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015 Maestrano
+Copyright (c) 2016 Maestrano
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/VERSION

@@ -1 +1 @@
-1.0.4
+1.1.0

data/app/controllers/maestrano/connec_controller.rb

@@ -18,10 +18,10 @@ class Maestrano::ConnecController < Maestrano::Rails::WebHookController
           Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Received entity from Connec! webhook: Entity=#{entity_name}, Data=#{entity}")
           connec_client = Maestrano::Connector::Rails::ConnecHelper.get_client(organization)
           external_client = Maestrano::Connector::Rails::External.get_client(organization)
-          last_synchronization = organization.last_successful_synchronization
+          last_synchronization_date = organization.last_synchronization_date
 
           entity_instance = entity_class_hash[:class].new(organization, connec_client, external_client, {})
-          entity_instance.before_sync(last_synchronization)
+          entity_instance.before_sync(last_synchronization_date)
           
           # Build expected input for consolidate_and_map_data
           if entity_class_hash[:is_complex]
@@ -33,7 +33,7 @@ class Maestrano::ConnecController < Maestrano::Rails::WebHookController
           end
           entity_instance.push_entities_to_external(mapped_entity[:connec_entities])
 
-          entity_instance.after_sync(last_synchronization)
+          entity_instance.after_sync(last_synchronization_date)
         end
       end
     rescue => e

data/app/jobs/maestrano/connector/rails/all_synchronizations_job.rb

@@ -4,7 +4,7 @@ module Maestrano::Connector::Rails
 
     # Trigger synchronization of all active organizations
     def perform(name=nil, count=nil)
-      Maestrano::Connector::Rails::Organization.where('oauth_provider IS NOT NULL AND oauth_token IS NOT NULL').each do |o|
+      Maestrano::Connector::Rails::Organization.where.not(oauth_provider: nil, encrypted_oauth_token: nil).each do |o|
         next unless [true, 1].include?(o.sync_enabled)
         Maestrano::Connector::Rails::SynchronizationJob.perform_later(o, {})
       end

data/app/jobs/maestrano/connector/rails/push_to_connec_job.rb

@@ -8,7 +8,7 @@ module Maestrano::Connector::Rails
 
       connec_client = Maestrano::Connector::Rails::ConnecHelper.get_client(organization)
       external_client = Maestrano::Connector::Rails::External.get_client(organization)
-      last_synchronization = organization.last_successful_synchronization
+      last_synchronization_date = organization.last_synchronization_date
 
       entities_hash.each do |external_entity_name, entities|
         if entity_instance_hash = find_entity_instance(external_entity_name, organization, connec_client, external_client, opts)
@@ -16,7 +16,7 @@ module Maestrano::Connector::Rails
 
           entity_instance = entity_instance_hash[:instance]
 
-          entity_instance.before_sync(last_synchronization)
+          entity_instance.before_sync(last_synchronization_date)
           # Build expected input for consolidate_and_map_data
           if entity_instance_hash[:is_complex]
             mapped_entities = entity_instance.consolidate_and_map_data(ComplexEntity.build_empty_hash(entity_instance.class.connec_entities_names), ComplexEntity.build_hash_with_entities(entity_instance.class.external_entities_names, external_entity_name, lambda{|name| name}, entities))
@@ -25,7 +25,7 @@ module Maestrano::Connector::Rails
           end
           entity_instance.push_entities_to_connec(mapped_entities[:external_entities])
 
-          entity_instance.after_sync(last_synchronization)
+          entity_instance.after_sync(last_synchronization_date)
         else
           Rails.logger.warn "Called push to connec job with unknow entity: #{external_entity_name}"
         end

data/app/jobs/maestrano/connector/rails/synchronization_job.rb

@@ -29,6 +29,7 @@ module Maestrano::Connector::Rails
 
       begin
         last_synchronization = organization.last_successful_synchronization
+        last_synchronization_date = organization.last_synchronization_date
         connec_client = ConnecHelper.get_client(organization)
         external_client = External.get_client(organization)
 
@@ -38,7 +39,7 @@ module Maestrano::Connector::Rails
           ConnectorLogger.log('info', organization, "First synchronization ever. Doing two half syncs to allow smart merging to work its magic.")
           [{skip_connec: true}, {skip_external: true}].each do |opt|
             organization.synchronized_entities.select{|k, v| v}.keys.each do |entity|
-              sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization, opts.merge(opt))
+              sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization_date, opts.merge(opt))
             end
           end
         elsif opts[:only_entities]
@@ -46,11 +47,11 @@ module Maestrano::Connector::Rails
           # The synchronization is marked as partial and will not be considered as the last-synchronization for the next sync
           current_synchronization.set_partial
           opts[:only_entities].each do |entity|
-            sync_entity(entity, organization, connec_client, external_client, last_synchronization, opts)
+            sync_entity(entity, organization, connec_client, external_client, last_synchronization_date, opts)
           end
         else
           organization.synchronized_entities.select{|k, v| v}.keys.each do |entity|
-            sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization, opts)
+            sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization_date, opts)
           end
         end
 
@@ -62,16 +63,16 @@ module Maestrano::Connector::Rails
       end
     end
 
-    def sync_entity(entity_name, organization, connec_client, external_client, last_synchronization, opts)
+    def sync_entity(entity_name, organization, connec_client, external_client, last_synchronization_date, opts)
       entity_instance = "Entities::#{entity_name.titleize.split.join}".constantize.new(organization, connec_client, external_client, opts.dup)
 
-      entity_instance.before_sync(last_synchronization)
-      external_entities = entity_instance.get_external_entities_wrapper(last_synchronization)
-      connec_entities = entity_instance.get_connec_entities(last_synchronization)
+      entity_instance.before_sync(last_synchronization_date)
+      external_entities = entity_instance.get_external_entities_wrapper(last_synchronization_date)
+      connec_entities = entity_instance.get_connec_entities(last_synchronization_date)
       mapped_entities = entity_instance.consolidate_and_map_data(connec_entities, external_entities)
       entity_instance.push_entities_to_external(mapped_entities[:connec_entities])
       entity_instance.push_entities_to_connec(mapped_entities[:external_entities])
-      entity_instance.after_sync(last_synchronization)
+      entity_instance.after_sync(last_synchronization_date)
     end
   end
 end

data/app/models/maestrano/connector/rails/concerns/complex_entity.rb

@@ -59,22 +59,22 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
   # -------------------------------------------------------------
   #          Entity equivalent methods
   # -------------------------------------------------------------
-  def get_connec_entities(last_synchronization)
+  def get_connec_entities(last_synchronization_date=nil)
     entities = ActiveSupport::HashWithIndifferentAccess.new
 
     self.class.connec_entities_names.each do |connec_entity_name|
       sub_entity_instance = instantiate_sub_entity_instance(connec_entity_name)
-      entities[connec_entity_name] = sub_entity_instance.get_connec_entities(last_synchronization)
+      entities[connec_entity_name] = sub_entity_instance.get_connec_entities(last_synchronization_date)
     end
     entities
   end
 
-  def get_external_entities_wrapper(last_synchronization)
+  def get_external_entities_wrapper(last_synchronization_date=nil)
     entities = ActiveSupport::HashWithIndifferentAccess.new
 
     self.class.external_entities_names.each do |external_entity_name|
       sub_entity_instance = instantiate_sub_entity_instance(external_entity_name)
-      entities[external_entity_name] = sub_entity_instance.get_external_entities_wrapper(last_synchronization)
+      entities[external_entity_name] = sub_entity_instance.get_external_entities_wrapper(last_synchronization_date)
     end
     entities
   end
@@ -141,11 +141,11 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
     end
   end
 
-  def before_sync(last_synchronization)
+  def before_sync(last_synchronization_date)
     # Does nothing by default
   end
 
-  def after_sync(last_synchronization)
+  def after_sync(last_synchronization_date)
     # Does nothing by default
   end
 

data/app/models/maestrano/connector/rails/concerns/entity.rb

@@ -59,6 +59,10 @@ module Maestrano::Connector::Rails::Concerns::Entity
       raise "Not implemented"
     end
 
+    def creation_date_from_external_entity_hash(entity)
+      raise "Not implemented"
+    end
+
     # Return a string representing the object from a connec! entity hash
     def object_name_from_connec_entity_hash(entity)
       raise "Not implemented"
@@ -154,7 +158,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
   # * full_sync
   # * $filter (see Connec! documentation)
   # * $orderby (see Connec! documentation)
-  def get_connec_entities(last_synchronization)
+  def get_connec_entities(last_synchronization_date=nil)
     return [] if @opts[:skip_connec] || !self.class.can_read_connec?
 
     Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching Connec! #{self.class.connec_entity_name}")
@@ -165,10 +169,10 @@ module Maestrano::Connector::Rails::Concerns::Entity
 
     # Fetch first page
     page_number = 0
-    if last_synchronization.blank? || @opts[:full_sync]
+    if last_synchronization_date.blank? || @opts[:full_sync]
       query_params[:$filter] = @opts[:$filter] if @opts[:$filter]
     else
-      query_params[:$filter] = "updated_at gt '#{last_synchronization.updated_at.iso8601}'" + (@opts[:$filter] ? " and #{@opts[:$filter]}" : '')
+      query_params[:$filter] = "updated_at gt '#{last_synchronization_date.iso8601}'" + (@opts[:$filter] ? " and #{@opts[:$filter]}" : '')
     end
 
     Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "entity=#{self.class.connec_entity_name}, fetching data with #{query_params.to_query}")
@@ -219,12 +223,12 @@ module Maestrano::Connector::Rails::Concerns::Entity
   # ----------------------------------------------
   #                 External methods
   # ----------------------------------------------
-  def get_external_entities_wrapper(last_synchronization)
+  def get_external_entities_wrapper(last_synchronization_date=nil)
     return [] if @opts[:skip_external] || !self.class.can_read_external?
-    get_external_entities(last_synchronization)
+    get_external_entities(last_synchronization_date)
   end
   
-  def get_external_entities(last_synchronization)
+  def get_external_entities(last_synchronization_date=nil)
     Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize}")
     raise "Not implemented"
   end
@@ -319,6 +323,9 @@ module Maestrano::Connector::Rails::Concerns::Entity
 
   def consolidate_and_map_connec_entities(connec_entities, external_entities, references, external_entity_name)
     connec_entities.map{|entity|
+      # Entity has been created before date filtering limit
+      next nil if before_date_filtering_limit?(entity, false) && !@opts[:full_sync]
+
       entity = Maestrano::Connector::Rails::ConnecHelper.unfold_references(entity, references, @organization)
       next nil unless entity
       connec_id = entity.delete(:__connec_id)
@@ -332,7 +339,6 @@ module Maestrano::Connector::Rails::Concerns::Entity
       idmap.update(name: self.class.object_name_from_connec_entity_hash(entity))
 
       next nil if idmap.external_inactive || !idmap.to_external || (!@opts[:full_sync] && not_modified_since_last_push_to_external?(idmap, entity))
-
       # Check for conflict with entities from external
       solve_conflict(entity, external_entities, external_entity_name, idmap)
     }.compact
@@ -340,6 +346,9 @@ module Maestrano::Connector::Rails::Concerns::Entity
 
   def consolidate_and_map_external_entities(external_entities, connec_entity_name)
     external_entities.map{|entity|
+      # Entity has been created before date filtering limit
+      next nil if before_date_filtering_limit?(entity) && !@opts[:full_sync]
+
       entity_id = self.class.id_from_external_entity_hash(entity)
       idmap = self.class.find_or_create_idmap(external_id: entity_id, organization_id: @organization.id, connec_entity: connec_entity_name.downcase)
 
@@ -389,11 +398,11 @@ module Maestrano::Connector::Rails::Concerns::Entity
   # ----------------------------------------------
   #             After and before sync
   # ----------------------------------------------
-  def before_sync(last_synchronization)
+  def before_sync(last_synchronization_date)
     # Does nothing by default
   end
 
-  def after_sync(last_synchronization)
+  def after_sync(last_synchronization_date)
     # Does nothing by default
   end
 
@@ -451,6 +460,10 @@ module Maestrano::Connector::Rails::Concerns::Entity
       not_modified
     end
 
+    def before_date_filtering_limit?(entity, external=true)
+      @organization.date_filtering_limit && @organization.date_filtering_limit > (external ? self.class.creation_date_from_external_entity_hash(entity) : entity['created_at'])
+    end
+
     def is_connec_more_recent?(connec_entity, external_entity)
       connec_entity['updated_at'] > self.class.last_update_date_from_external_entity_hash(external_entity)
     end
@@ -465,11 +478,11 @@ module Maestrano::Connector::Rails::Concerns::Entity
         end
 
         if keep_connec
-          Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from external kept")
+          Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from Connec! kept")
           external_entities.delete(external_entity)
           map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
         else
-          Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from Connec! kept")
+          Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from external kept")
           nil
         end
 

data/app/models/maestrano/connector/rails/organization.rb

@@ -20,8 +20,9 @@ module Maestrano::Connector::Rails
     #===================================
     # Encryptions
     #===================================
-    # attr_encrypted :oauth_token, key: ::Settings.encryption_key
-    # attr_encrypted :refresh_token, key: ::Settings.encryption_key
+    attr_encrypted_options.merge!(:mode => :per_attribute_iv_and_salt)
+    attr_encrypted :oauth_token, key: ::Settings.encryption_key1
+    attr_encrypted :refresh_token, key: ::Settings.encryption_key2
 
     #===================================
     # Associations
@@ -74,5 +75,9 @@ module Maestrano::Connector::Rails
     def last_successful_synchronization
       self.synchronizations.where(status: 'SUCCESS', partial: false).order(updated_at: :desc).first
     end
+
+    def last_synchronization_date
+      (last_successful_synchronization && last_successful_synchronization.updated_at) || date_filtering_limit
+    end
   end
 end

data/app/models/maestrano/connector/rails/synchronization.rb

@@ -41,7 +41,7 @@ module Maestrano::Connector::Rails
     def clean_synchronizations
       count = self.organization.synchronizations.count
       if count > 100
-        self.organization.synchronizations.limit(count - 100).destroy_all
+        self.organization.synchronizations.order('id ASC').limit(count - 100).destroy_all
       end
     end
   end

data/db/migrate/20160614114401_add_date_filtering_limit_to_organization.rb

@@ -0,0 +1,11 @@
+class AddDateFilteringLimitToOrganization < ActiveRecord::Migration
+  def change
+    add_column :organizations, :date_filtering_limit, :datetime
+    add_column :organizations, :historical_data, :boolean, default: false
+
+    # Set historical data to true for organization existing before the feature
+    Maestrano::Connector::Rails::Organization.all.each do |o|
+      o.update(historical_data: true)
+    end
+  end
+end

data/db/migrate/20160614160654_add_encryption_on_oauth_keys.rb

@@ -0,0 +1,19 @@
+class AddEncryptionOnOauthKeys < ActiveRecord::Migration
+  def change
+    tokens = Maestrano::Connector::Rails::Organization.map{|o|
+      {id: o.id, oauth_token: o.oauth_token, refresh_token: o.refresh_token}
+    }
+
+    rename_column :organizations, :oauth_token, :encrypted_oauth_token
+    add_column :organizations, :encrypted_oauth_token_iv, :string
+    add_column :organizations, :encrypted_oauth_token_salt, :string
+    rename_column :organizations, :refresh_token, :encrypted_refresh_token
+    add_column :organizations, :encrypted_refresh_token_iv, :string
+    add_column :organizations, :encrypted_refresh_token_salt, :string
+
+    tokens.each do |token|
+      o = Maestrano::Connector::Rails::Organization.find(token[:id])
+      o.update(oauth_token: token[:oauth_token], refresh_token: token[:refresh_token])
+    end
+  end
+end

data/lib/generators/connector/install_generator.rb

@@ -38,6 +38,7 @@ module Connector
         copy_file 'home_controller.rb', 'app/controllers/home_controller.rb'
         copy_file 'home_controller_spec.rb', 'spec/controllers/home_controller_spec.rb'
         copy_file 'home_index.haml', 'app/views/home/index.html.haml'
+        copy_file 'home.js', 'app/assets/javascripts/home.js'
 
         copy_file 'synchronizations_controller.rb', 'app/controllers/synchronizations_controller.rb'
         copy_file 'synchronizations_controller_spec.rb', 'spec/controllers/synchronizations_controller_spec.rb'

data/lib/generators/connector/templates/complex_entity_example/person.rb

@@ -1,7 +1,7 @@
 # TODO
 # This file is provided as an example and should be removed
 # See README for explanation
-# class Enities::SubEntities::Person < Maestrano::Connector::Rails::SubEntityBase
+# class Entities::SubEntities::Person < Maestrano::Connector::Rails::SubEntityBase
 #   def self.external?
 #     false
 #   end

data/lib/generators/connector/templates/entity.rb

@@ -8,11 +8,11 @@ class Maestrano::Connector::Rails::Entity
   # * @opts
 
   # Return an array of entities from the external app
-  def get_external_entities(last_synchronization)
+  def get_external_entities(last_synchronization_date=nil)
     Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize}")
     # TODO
-    # This method should return only entities that have been updated since the last_synchronization
-    # It should also implements an option to do a full synchronization when @opts[:full_sync] == true or when there is no last_synchronization
+    # This method should return only entities that have been updated since the last_synchronization_date
+    # It should also implements an option to do a full synchronization when @opts[:full_sync] == true or when there is no last_synchronization_date
     # Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Received data: Source=#{Maestrano::Connector::Rails::External.external_name}, Entity=#{self.class.external_entity_name}, Response=#{entities}")
   end
 
@@ -40,6 +40,12 @@ class Maestrano::Connector::Rails::Entity
     # e.g entity['last_update']
   end
 
+  def self.creation_date_from_external_entity_hash(entity)
+    # TODO
+    # This method return the creation date from an external_entity_hash
+    # e.g entity['created_at']
+  end
+
   def self.inactive_from_external_entity_hash?(entity)
     # TODO
     # This method return true is entity is inactive in the external application

data/lib/generators/connector/templates/home.js

@@ -0,0 +1,11 @@
+function historicalDataDisplay()
+{
+  if (document.getElementById('historical-data').checked)
+  {
+      document.getElementById('historical-data-display-checked').style.display = 'block';
+      document.getElementById('historical-data-display-unchecked').style.display = 'none';
+  } else {
+      document.getElementById('historical-data-display-unchecked').style.display = 'block';
+      document.getElementById('historical-data-display-checked').style.display = 'none';
+  }
+}

data/lib/generators/connector/templates/home_controller.rb

@@ -13,9 +13,20 @@ class HomeController < ApplicationController
         organization.synchronized_entities[entity] = !!params["#{entity}"]
       end
       organization.sync_enabled = organization.synchronized_entities.values.any?
+
+      unless organization.historical_data
+        historical_data = !!params['historical-data']
+        if historical_data
+          organization.date_filtering_limit = nil
+          organization.historical_data = true
+        else
+          organization.date_filtering_limit ||= Time.now
+        end
+      end
+      
       organization.save
 
-      if !old_sync_state
+      if !old_sync_state && organization.sync_enabled
         Maestrano::Connector::Rails::SynchronizationJob.perform_later(organization, {})
         flash[:info] = 'Congrats, you\'re all set up! Your data are now being synced'
       end

data/lib/generators/connector/templates/home_index.haml

@@ -57,6 +57,17 @@
                       .col-md-5.text-right
                         - if v && @organization.oauth_uid && @organization.sync_enabled
                           = link_to "Force a synchronization for #{k.to_s.humanize.pluralize} only", home_synchronize_path(opts: {only_entities: [k.to_s]}), method: :post, class: "btn btn-warning btn-sm"
+            .spacer1
+            .row
+              .col-md-4
+                %label{:for => 'historical-data'} Synchronize my historical data
+              .col-md-2
+                %input{type: 'checkbox', id: 'historical-data', name: 'historical-data', checked: @organization.historical_data, onchange: 'historicalDataDisplay();', disabled: @organization.historical_data}
+              .col-md-6
+                %small#historical-data-display-unchecked{style: "display: #{@organization.historical_data ? 'none' : 'block'}"} Only data created after #{(@organization.date_filtering_limit && @organization.date_filtering_limit.utc || Time.now.utc).to_formatted_s(:long_ordinal)} will be synchronized
+                %small#historical-data-display-checked{style: "display: #{!@organization.historical_data ? 'none' : 'block'}"}
+                  Synchronizing your historical data will share all data in ApplicationName. This action is not reversible. Want to know more? Check 
+                  = link_to 'here', 'https://maestrano.atlassian.net/wiki/display/UKB/How+Connec%21+manages+Historical+Data+Sharing'
           .spacer1
           .row
             .col-md-2.col-md-offset-10.text-center.link-step-action

data/maestrano-connector-rails.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: maestrano-connector-rails 1.0.4 ruby lib
+# stub: maestrano-connector-rails 1.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "maestrano-connector-rails"
-  s.version = "1.0.4"
+  s.version = "1.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Pierre Berard"]
-  s.date = "2016-06-07"
+  s.date = "2016-06-17"
   s.description = "Maestrano is the next generation marketplace for SME applications. See https://maestrano.com for details."
   s.email = "pierre.berard@maestrano.com"
   s.executables = ["rails"]
@@ -56,7 +56,6 @@ Gem::Specification.new do |s|
     "app/models/maestrano/connector/rails/user_organization_rel.rb",
     "bin/rails",
     "config/routes.rb",
-    "db/20160524112054_add_encryption_on_oauth_keys.rb",
     "db/migrate/20151122162100_create_users.rb",
     "db/migrate/20151122162414_create_organizations.rb",
     "db/migrate/20151122162613_create_user_organization_rels.rb",
@@ -65,6 +64,8 @@ Gem::Specification.new do |s|
     "db/migrate/20160205132857_add_sync_enabled_to_organizations.rb",
     "db/migrate/20160215103120_add_name_to_id_map.rb",
     "db/migrate/20160427112250_add_inactive_to_idmaps.rb",
+    "db/migrate/20160614114401_add_date_filtering_limit_to_organization.rb",
+    "db/migrate/20160614160654_add_encryption_on_oauth_keys.rb",
     "lib/generators/connector/USAGE",
     "lib/generators/connector/complex_entity_generator.rb",
     "lib/generators/connector/install_generator.rb",
@@ -78,6 +79,7 @@ Gem::Specification.new do |s|
     "lib/generators/connector/templates/example_entity.rb",
     "lib/generators/connector/templates/example_entity_spec.rb",
     "lib/generators/connector/templates/external.rb",
+    "lib/generators/connector/templates/home.js",
     "lib/generators/connector/templates/home_controller.rb",
     "lib/generators/connector/templates/home_controller_spec.rb",
     "lib/generators/connector/templates/home_index.haml",
@@ -145,6 +147,11 @@ Gem::Specification.new do |s|
     "spec/dummy/config/locales/en.yml",
     "spec/dummy/config/routes.rb",
     "spec/dummy/config/secrets.yml",
+    "spec/dummy/config/settings.yml",
+    "spec/dummy/config/settings/development.yml",
+    "spec/dummy/config/settings/production.yml",
+    "spec/dummy/config/settings/test.yml",
+    "spec/dummy/config/settings/uat.yml",
     "spec/dummy/db/schema.rb",
     "spec/dummy/lib/assets/.keep",
     "spec/dummy/log/.keep",
@@ -174,6 +181,7 @@ Gem::Specification.new do |s|
     "spec/routing/connec_routing_spec.rb",
     "spec/spec_helper.rb",
     "template/Procfile",
+    "template/application.yml.sample",
     "template/database.yml",
     "template/factories.rb",
     "template/gitignore",
@@ -204,6 +212,8 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<haml-rails>, [">= 0"])
       s.add_runtime_dependency(%q<bootstrap-sass>, [">= 0"])
       s.add_runtime_dependency(%q<autoprefixer-rails>, [">= 0"])
+      s.add_runtime_dependency(%q<attr_encrypted>, ["~> 1.4.0"])
+      s.add_runtime_dependency(%q<config>, [">= 0"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0"])
@@ -213,6 +223,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<factory_girl_rails>, [">= 0"])
       s.add_development_dependency(%q<sqlite3>, [">= 0"])
       s.add_development_dependency(%q<shoulda-matchers>, [">= 0"])
+      s.add_development_dependency(%q<timecop>, [">= 0"])
     else
       s.add_dependency(%q<maestrano-rails>, [">= 0"])
       s.add_dependency(%q<hash_mapper>, [">= 0"])
@@ -220,6 +231,8 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<haml-rails>, [">= 0"])
       s.add_dependency(%q<bootstrap-sass>, [">= 0"])
       s.add_dependency(%q<autoprefixer-rails>, [">= 0"])
+      s.add_dependency(%q<attr_encrypted>, ["~> 1.4.0"])
+      s.add_dependency(%q<config>, [">= 0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<rdoc>, ["~> 3.12"])
       s.add_dependency(%q<bundler>, ["~> 1.0"])
@@ -229,6 +242,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<factory_girl_rails>, [">= 0"])
       s.add_dependency(%q<sqlite3>, [">= 0"])
       s.add_dependency(%q<shoulda-matchers>, [">= 0"])
+      s.add_dependency(%q<timecop>, [">= 0"])
     end
   else
     s.add_dependency(%q<maestrano-rails>, [">= 0"])
@@ -237,6 +251,8 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<haml-rails>, [">= 0"])
     s.add_dependency(%q<bootstrap-sass>, [">= 0"])
     s.add_dependency(%q<autoprefixer-rails>, [">= 0"])
+    s.add_dependency(%q<attr_encrypted>, ["~> 1.4.0"])
+    s.add_dependency(%q<config>, [">= 0"])
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<rdoc>, ["~> 3.12"])
     s.add_dependency(%q<bundler>, ["~> 1.0"])
@@ -246,6 +262,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<factory_girl_rails>, [">= 0"])
     s.add_dependency(%q<sqlite3>, [">= 0"])
     s.add_dependency(%q<shoulda-matchers>, [">= 0"])
+    s.add_dependency(%q<timecop>, [">= 0"])
   end
 end
 

data/release_notes.md

@@ -1,3 +1,9 @@
+## 1.0.4
+
+### Fixes
+* Fix uneeded creation of idmap for Connec! entities following a failure
+* Fix Connec! pagination
+
 ## 1.0.3
 
 ### Fixes

data/spec/dummy/config/settings.yml

@@ -0,0 +1,37 @@
+encryption_key1: <%= ENV['encryption_key1'] %>
+encryption_key2: <%= ENV['encryption_key2'] %>
+
+# tenant config
+default:
+  environment: 'production'
+  x509_certificate: "-----BEGIN CERTIFICATE-----\nMIIDezCCAuSgAwIBAgIJAPFpcH2rW0pyMA0GCSqGSIb3DQEBBQUAMIGGMQswCQYD\nVQQGEwJBVTEMMAoGA1UECBMDTlNXMQ8wDQYDVQQHEwZTeWRuZXkxGjAYBgNVBAoT\nEU1hZXN0cmFubyBQdHkgTHRkMRYwFAYDVQQDEw1tYWVzdHJhbm8uY29tMSQwIgYJ\nKoZIhvcNAQkBFhVzdXBwb3J0QG1hZXN0cmFuby5jb20wHhcNMTQwMTA0MDUyNDEw\nWhcNMzMxMjMwMDUyNDEwWjCBhjELMAkGA1UEBhMCQVUxDDAKBgNVBAgTA05TVzEP\nMA0GA1UEBxMGU3lkbmV5MRowGAYDVQQKExFNYWVzdHJhbm8gUHR5IEx0ZDEWMBQG\nA1UEAxMNbWFlc3RyYW5vLmNvbTEkMCIGCSqGSIb3DQEJARYVc3VwcG9ydEBtYWVz\ndHJhbm8uY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQD3feNNn2xfEz5/\nQvkBIu2keh9NNhobpre8U4r1qC7h7OeInTldmxGL4cLHw4ZAqKbJVrlFWqNevM5V\nZBkDe4mjuVkK6rYK1ZK7eVk59BicRksVKRmdhXbANk/C5sESUsQv1wLZyrF5Iq8m\na9Oy4oYrIsEF2uHzCouTKM5n+O4DkwIDAQABo4HuMIHrMB0GA1UdDgQWBBSd/X0L\n/Pq+ZkHvItMtLnxMCAMdhjCBuwYDVR0jBIGzMIGwgBSd/X0L/Pq+ZkHvItMtLnxM\nCAMdhqGBjKSBiTCBhjELMAkGA1UEBhMCQVUxDDAKBgNVBAgTA05TVzEPMA0GA1UE\nBxMGU3lkbmV5MRowGAYDVQQKExFNYWVzdHJhbm8gUHR5IEx0ZDEWMBQGA1UEAxMN\nbWFlc3RyYW5vLmNvbTEkMCIGCSqGSIb3DQEJARYVc3VwcG9ydEBtYWVzdHJhbm8u\nY29tggkA8WlwfatbSnIwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQDE\nhe/18oRh8EqIhOl0bPk6BG49AkjhZZezrRJkCFp4dZxaBjwZTddwo8O5KHwkFGdy\nyLiPV326dtvXoKa9RFJvoJiSTQLEn5mO1NzWYnBMLtrDWojOe6Ltvn3x0HVo/iHh\nJShjAn6ZYX43Tjl1YXDd1H9O+7/VgEWAQQ32v8p5lA==\n-----END CERTIFICATE-----"
+  x509_fingerprint: '2f:57:71:e4:40:19:57:37:a6:2c:f0:c5:82:52:2f:2e:41:b7:9d:7e'
+  api_id: 'connec_api_id'
+  api_key: 'connec_api_key'
+  sso_init_path: '/maestrano/auth/saml/init/default'
+  sso_consume_path: '/maestrano/auth/saml/consume/default'
+  api_host: 'https://maestrano.com'
+  connec_host: 'https://api-connec.maestrano.com'
+  webhook:
+    account:
+      groups_path: '/maestrano/account/groups/:id/default'
+      group_users_path: '/maestrano/account/groups/:group_id/users/:id/default'
+    connec:
+      notifications_path: '/maestrano/connec/notifications/default'
+
+maestrano-uat:
+  environment: 'uat'
+  x509_certificate: "-----BEGIN CERTIFICATE-----\nMIIDezCCAuSgAwIBAgIJAMzy+weDPp7qMA0GCSqGSIb3DQEBBQUAMIGGMQswCQYD\nVQQGEwJBVTEMMAoGA1UECBMDTlNXMQ8wDQYDVQQHEwZTeWRuZXkxGjAYBgNVBAoT\nEU1hZXN0cmFubyBQdHkgTHRkMRYwFAYDVQQDEw1tYWVzdHJhbm8uY29tMSQwIgYJ\nKoZIhvcNAQkBFhVzdXBwb3J0QG1hZXN0cmFuby5jb20wHhcNMTQwMTA0MDUyMzE0\nWhcNMzMxMjMwMDUyMzE0WjCBhjELMAkGA1UEBhMCQVUxDDAKBgNVBAgTA05TVzEP\nMA0GA1UEBxMGU3lkbmV5MRowGAYDVQQKExFNYWVzdHJhbm8gUHR5IEx0ZDEWMBQG\nA1UEAxMNbWFlc3RyYW5vLmNvbTEkMCIGCSqGSIb3DQEJARYVc3VwcG9ydEBtYWVz\ndHJhbm8uY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC+2uyQeAOc/iro\nhCyT33RkkWfTGeJ8E/mu9F5ORWoCZ/h2J+QDuzuc69Rf1LoO4wZVQ8LBeWOqMBYz\notYFUIPlPfIBXDNL/stHkpg28WLDpoJM+46WpTAgp89YKgwdAoYODHiUOcO/uXOO\n2i9Ekoa+kxbvBzDJf7uuR/io6GERXwIDAQABo4HuMIHrMB0GA1UdDgQWBBTGRDBT\nie5+fHkB0+SZ5g3WY/D2RTCBuwYDVR0jBIGzMIGwgBTGRDBTie5+fHkB0+SZ5g3W\nY/D2RaGBjKSBiTCBhjELMAkGA1UEBhMCQVUxDDAKBgNVBAgTA05TVzEPMA0GA1UE\nBxMGU3lkbmV5MRowGAYDVQQKExFNYWVzdHJhbm8gUHR5IEx0ZDEWMBQGA1UEAxMN\nbWFlc3RyYW5vLmNvbTEkMCIGCSqGSIb3DQEJARYVc3VwcG9ydEBtYWVzdHJhbm8u\nY29tggkAzPL7B4M+nuowDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQAw\nRxg3rZrML//xbsS3FFXguzXiiNQAvA4KrMWhGh3jVrtzAlN1/okFNy6zuN8gzdKD\nYw2n0c/u3cSpUutIVZOkwQuPCMC1hoP7Ilat6icVewNcHayLBxKgRxpBhr5Sc4av\n3HOW5Bi/eyC7IjeBTbTnpziApEC7uUsBou2rlKmTGw==\n-----END CERTIFICATE-----"
+  x509_fingerprint: '8a:1e:2e:76:c4:67:80:68:6c:81:18:f7:d3:29:5d:77:f8:79:54:2f'
+  api_id: 'maestrano_uat_connec_api_id'
+  api_key: 'maestrano_uat_connec_api_key'
+  sso_consume_path: '/maestrano/auth/saml/consume/maestrano-uat'
+  sso_init_path: '/maestrano/auth/saml/init/maestrano-uat'
+  api_host: 'https://uat.maestrano.io'
+  connec_host: 'https://api-connec-uat.maestrano.io'
+  webhook:
+    account:
+      groups_path: '/maestrano/account/groups/:id/maestrano-uat'
+      group_users_path: '/maestrano/account/groups/:group_id/users/:id/maestrano-uat'
+    connec:
+      notifications_path: '/maestrano/connec/notifications/maestrano-uat'

data/spec/dummy/config/settings/development.yml

@@ -0,0 +1,12 @@
+app_host: 'http://localhost:3001/'
+
+encryption_key1: 'This is a key that is 256 bits!!'
+encryption_key2: 'This is a key that is 256 bits!!'
+
+default:
+  api_host: 'https://maestrano.com'
+  connec_host: 'https://api-connec.maestrano.com'
+
+maestrano-uat:
+  api_host: 'https://uat.maestrano.io'
+  connec_host: 'https://api-connec-uat.maestrano.io'

data/spec/dummy/config/settings/production.yml

@@ -0,0 +1 @@
+app_host: 'https://connector-xxx.herokuapp.com/'

data/spec/dummy/config/settings/test.yml

@@ -0,0 +1,12 @@
+app_host: 'http://localhost:3001/'
+
+encryption_key1: 'This is a key that is 256 bits!!'
+encryption_key2: 'This is a key that is 256 bits!!'
+
+default:
+  environment: 'local'
+  api_host: 'http://localhost:3000'
+
+maestrano-uat:
+  environment: 'local'
+  api_host: 'http://localhost:3000'

data/spec/dummy/config/settings/uat.yml

@@ -0,0 +1 @@
+app_host: 'https://connector-xxx-uat.herokuapp.com/'

data/spec/dummy/db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160427120963) do
+ActiveRecord::Schema.define(version: 20160614152139) do
 
   create_table "id_maps", force: :cascade do |t|
     t.string   "connec_id"
@@ -42,13 +42,18 @@ ActiveRecord::Schema.define(version: 20160427120963) do
     t.string   "oauth_provider"
     t.string   "oauth_uid"
     t.string   "oauth_name"
-    t.string   "oauth_token"
-    t.string   "refresh_token"
+    t.string   "encrypted_oauth_token"
+    t.string   "encrypted_refresh_token"
     t.string   "instance_url"
     t.string   "synchronized_entities"
-    t.datetime "created_at",                            null: false
-    t.datetime "updated_at",                            null: false
-    t.boolean  "sync_enabled",          default: false
+    t.datetime "created_at",                                   null: false
+    t.datetime "updated_at",                                   null: false
+    t.boolean  "sync_enabled",                 default: false
+    t.datetime "date_filtering_limit"
+    t.string   "encrypted_oauth_token_iv"
+    t.string   "encrypted_oauth_token_salt"
+    t.string   "encrypted_refresh_token_iv"
+    t.string   "encrypted_refresh_token_salt"
   end
 
   add_index "organizations", ["uid", "tenant"], name: "orga_uid_index"

data/spec/integration/connec_to_external_spec.rb

@@ -64,7 +64,9 @@ describe 'connec to the external application' do
       ],
       "is_customer" => false,
       "is_supplier" => true,
-      "is_lead" => false
+      "is_lead" => false,
+      "updated_at" => 2.day.ago,
+      "created_at" => 2.day.ago
     }
   }
   let(:person) { person1 }
@@ -77,7 +79,7 @@ describe 'connec to the external application' do
     allow_any_instance_of(Entities::ConnecToExternal).to receive(:get_external_entities).and_return([])
   }
 
-  subject { Maestrano::Connector::Rails::SynchronizationJob.new.sync_entity('connec_to_external', organization, connec_client, external_client, nil, {}) }
+  subject { Maestrano::Connector::Rails::SynchronizationJob.new.sync_entity('connec_to_external', organization, connec_client, external_client, organization.last_synchronization_date, {}) }
 
   describe 'a new record created in connec with all references known' do
     before {
@@ -185,4 +187,24 @@ describe 'connec to the external application' do
       }.to_not change{ Maestrano::Connector::Rails::IdMap.count }
     end
   end
+
+  describe 'an entity from before the date filtering limit' do
+    let(:date_filtering_limit) { 2.minute.ago }
+    before {
+      organization.update(date_filtering_limit: date_filtering_limit)
+    }
+
+    it 'calls get_connec_entities with a date even if there is no last sync' do
+      expect_any_instance_of(Entities::ConnecToExternal).to receive(:get_connec_entities).with(date_filtering_limit).and_return([])
+      subject
+    end
+
+    it 'pushes nothing and creates no idmap' do
+      expect_any_instance_of(Entities::ConnecToExternal).to_not receive(:create_external_entity)
+      expect_any_instance_of(Entities::ConnecToExternal).to_not receive(:update_external_entity)
+      expect{
+        subject
+      }.to_not change{ Maestrano::Connector::Rails::IdMap.count }
+    end
+  end
 end

data/spec/models/entity_spec.rb

@@ -76,6 +76,10 @@ describe Maestrano::Connector::Rails::Entity do
       it { expect{ subject.last_update_date_from_external_entity_hash(nil) }.to raise_error('Not implemented') }
     end
 
+    describe 'creation_date_from_external_entity_hash' do
+      it { expect{ subject.creation_date_from_external_entity_hash(nil) }.to raise_error('Not implemented') }
+    end
+
     # Entity specific methods
     describe 'singleton?' do
       it 'is false by default' do
@@ -166,7 +170,7 @@ describe Maestrano::Connector::Rails::Entity do
 
     # Connec! methods
     describe 'connec_methods' do
-      let(:sync) { create(:synchronization) }
+      let(:sync) { create(:synchronization, organization: organization) }
 
       describe 'filter_connec_entities' do
         it 'does nothing by default' do
@@ -211,7 +215,7 @@ describe Maestrano::Connector::Rails::Entity do
               let(:opts) { {full_sync: true} }
               it 'performs a full get' do
                 expect(connec_client).to receive(:get).with("#{connec_name.downcase.pluralize}?")
-                subject.get_connec_entities(sync)
+                subject.get_connec_entities(sync.updated_at)
               end
             end
 
@@ -226,7 +230,7 @@ describe Maestrano::Connector::Rails::Entity do
               it 'performs a time limited get' do
                 uri_param = {"$filter" => "updated_at gt '#{sync.updated_at.iso8601}'"}.to_query
                 expect(connec_client).to receive(:get).with("#{connec_name.downcase.pluralize}?#{uri_param}")
-                subject.get_connec_entities(sync)
+                subject.get_connec_entities(sync.updated_at)
               end
             end
 
@@ -235,21 +239,21 @@ describe Maestrano::Connector::Rails::Entity do
                 subject.instance_variable_set(:@opts, {full_sync: true, :$filter => "code eq 'PEO12'"})
                 uri_param = {'$filter'=>'code eq \'PEO12\''}.to_query
                 expect(connec_client).to receive(:get).with("#{connec_name.downcase.pluralize}?#{uri_param}")
-                subject.get_connec_entities(sync)
+                subject.get_connec_entities(sync.updated_at)
               end
 
               it 'support filter option for time limited sync' do
                 subject.instance_variable_set(:@opts, {:$filter => "code eq 'PEO12'"})
                 uri_param = {"$filter"=>"updated_at gt '#{sync.updated_at.iso8601}' and code eq 'PEO12'"}.to_query
                 expect(connec_client).to receive(:get).with("#{connec_name.downcase.pluralize}?#{uri_param}")
-                subject.get_connec_entities(sync)
+                subject.get_connec_entities(sync.updated_at)
               end
 
               it 'support orderby option for time limited sync' do
                 subject.instance_variable_set(:@opts, {:$orderby => "name asc"})
                 uri_param = {"$orderby"=>"name asc", "$filter"=>"updated_at gt '#{sync.updated_at.iso8601}'"}.to_query
                 expect(connec_client).to receive(:get).with("#{connec_name.downcase.pluralize}?#{uri_param}")
-                subject.get_connec_entities(sync)
+                subject.get_connec_entities(sync.updated_at)
               end
             end
 
@@ -620,6 +624,7 @@ describe Maestrano::Connector::Rails::Entity do
       before {
         allow(subject.class).to receive(:id_from_external_entity_hash).and_return(id)
         allow(subject.class).to receive(:last_update_date_from_external_entity_hash).and_return(date)
+        allow(subject.class).to receive(:creation_date_from_external_entity_hash).and_return(date)
       }
       
       describe 'consolidate_and_map_data' do
@@ -726,8 +731,8 @@ describe Maestrano::Connector::Rails::Entity do
         let(:id2) { nil }
         let(:connec_id1) { 'connec-id-1' }
         let(:connec_id2) { 'connec-id-2' }
-        let(:entity1) { {'id' => id1, 'name' => 'John', 'updated_at' => date} }
-        let(:entity2) { {'id' => id2, 'name' => 'Jane', 'updated_at' => date} }
+        let(:entity1) { {'id' => id1, 'name' => 'John', 'updated_at' => date, 'created_at' => date} }
+        let(:entity2) { {'id' => id2, 'name' => 'Jane', 'updated_at' => date, 'created_at' => date} }
         let(:entity_without_refs) { {} }
         let(:entities) { [entity1, entity2] }
         before {
@@ -788,6 +793,24 @@ describe Maestrano::Connector::Rails::Entity do
             end
           end
 
+          context 'when before date_filtering_limit' do
+            before {
+              organization.update(date_filtering_limit: 5.minutes.ago)
+            }
+
+            it 'discards the entity' do
+              expect(subject.consolidate_and_map_connec_entities(entities, [], [], external_name)).to eql([])
+            end
+
+            context 'with full synchronization opts' do
+              let(:opts) { {full_sync: true} }
+
+              it 'keeps the entity' do
+                expect(subject.consolidate_and_map_connec_entities(entities, [], [], external_name)).to eql([{entity: {mapped: 'entity'}, idmap: idmap1}])
+              end
+            end
+          end
+
         end
 
         context 'when conflict' do
@@ -906,6 +929,24 @@ describe Maestrano::Connector::Rails::Entity do
             end
           end
 
+          context 'when before date_filtering_limit' do
+            before {
+              organization.update(date_filtering_limit: 5.minutes.ago)
+            }
+
+            it 'discards the entity' do
+              expect(subject.consolidate_and_map_external_entities([entity], connec_name)).to eql([])
+            end
+
+            context 'with full synchronization opts' do
+              let(:opts) { {full_sync: true} }
+
+              it 'keeps the entity' do
+                expect(subject.consolidate_and_map_external_entities([entity], connec_name)).to eql([{entity: {mapped: 'ext_entity'}, idmap: idmap}])
+              end
+            end
+          end
+
         end
       end
     end

data/spec/models/organization_spec.rb

@@ -137,12 +137,45 @@ describe Maestrano::Connector::Rails::Organization do
     describe 'last_successful_synchronization' do
       let!(:running_sync) { create(:synchronization, organization: subject, status: 'RUNNING') }
       let!(:failed_sync) { create(:synchronization, organization: subject, status: 'ERROR') }
-      let!(:success_sync) { create(:synchronization, organization: subject, status: 'SUCCESS') }
+      let!(:success_sync) { create(:synchronization, organization: subject, status: 'SUCCESS', updated_at: 1.minute.ago) }
       let!(:success_sync2) { create(:synchronization, organization: subject, status: 'SUCCESS', updated_at: 3.hours.ago) }
       let!(:partial) { create(:synchronization, organization: subject, status: 'SUCCESS', partial: true) }
 
       it { expect(subject.last_successful_synchronization).to eql(success_sync) }
     end
+
+    describe 'last_synchronization_date' do
+      let(:date) { 2.days.ago }
+      
+      context 'with date_filtering_limit' do
+        before {
+          subject.date_filtering_limit = date
+        }
+
+        it { expect(subject.last_synchronization_date).to eql(date) }
+      end
+
+      context 'with sync' do
+        let!(:success_sync) { create(:synchronization, organization: subject, status: 'SUCCESS') }
+
+        it { expect(subject.last_synchronization_date.to_date).to eql(success_sync.updated_at.to_date) }
+      end
+
+      context 'with both' do
+        let!(:success_sync) { create(:synchronization, organization: subject, status: 'SUCCESS') }
+        before {
+          subject.date_filtering_limit = date
+        }
+
+        it 'returns the sync date' do
+          expect(subject.last_synchronization_date.to_date).to eql(success_sync.updated_at.to_date)
+        end
+      end
+
+      context 'with none' do
+        it { expect(subject.last_synchronization_date).to eql(nil) }
+      end
+    end
   end
 
 

data/spec/models/synchronization_spec.rb

@@ -72,25 +72,34 @@ describe Maestrano::Connector::Rails::Synchronization do
       end
     end
 
-    describe 'clean_synchronizations' do
+    describe 'clean_synchronizations on creation' do
       let!(:organization) { create(:organization) }
-      let!(:sync) { create(:synchronization, organization: organization) }
-      let!(:sync2) { create(:synchronization, organization: organization) }
 
       context 'when less than 100 syncs' do
+        before {
+          2.times do
+            create(:synchronization, organization: organization)
+          end
+        }
+
         it 'does nothing' do
-          expect{ sync.clean_synchronizations }.to_not change{ organization.synchronizations.count }
+          expect{ organization.synchronizations.create(status: 'RUNNING') }.to change{ organization.synchronizations.count }.by(1)
         end
       end
 
       context 'when more than 100 syncs' do
         before {
-          allow_any_instance_of(ActiveRecord::Associations::CollectionProxy).to receive(:count).and_return(102)
+          100.times do
+            create(:synchronization, organization: organization)
+          end
         }
 
-        it 'destroy the idmaps' do
-          expect{ sync.clean_synchronizations }.to change{ Maestrano::Connector::Rails::Synchronization.count }.by(-2)
+        it 'destroy the right syncs' do
+          sync = organization.synchronizations.create(status: 'RUNNING')
+          expect(Maestrano::Connector::Rails::Synchronization.count).to eql(100)
+          expect(Maestrano::Connector::Rails::Synchronization.all.map(&:id)).to eql([*2..101])
         end
+
       end
     end
   end

data/spec/spec_helper.rb

@@ -5,6 +5,7 @@ require 'rspec/rails'
 require 'factory_girl_rails'
 require 'shoulda/matchers'
 require 'simplecov'
+require 'timecop'
 SimpleCov.start
 
 Rails.backtrace_cleaner.remove_silencers!

data/template/application.yml.sample

@@ -0,0 +1,5 @@
+connec_api_id: ''
+connec_api_key: ''
+
+encryption_key1: ''
+encryption_key2: ''

data/template/maestrano-connector-template.rb

@@ -47,7 +47,7 @@ gem 'uglifier', '>= 1.3.0'
 
 gem 'maestrano-connector-rails'
 gem 'config'
-# gem 'attr_encrypted', '~> 3.0.0'
+gem 'attr_encrypted', '~> 1.4.0'
 
 # Background jobs
 gem 'sinatra', :require => nil
@@ -113,6 +113,8 @@ after_bundle do
   end
   copy_file 'settings/settings.yml', 'config/settings.yml'
 
+  copy_file 'application.yml.sample', 'config/application.yml.sample'
+
   application do <<-RUBY
     config.generators do |g|
       g.test_framework :rspec, fixture: false

data/template/settings/development.yml

@@ -1,6 +1,7 @@
 app_host: 'http://localhost:3001/'
 
-# encryption_key: 'This is a key that is 256 bits!!'
+encryption_key1: 'This is a key that is 256 bits!!'
+encryption_key2: 'This is a key that is 256 bits!!'
 
 default:
   api_host: 'https://maestrano.com'

data/template/settings/settings.yml

@@ -1,4 +1,5 @@
-# encryption_key: <%= ENV['encryption_key'] %>
+encryption_key1: <%= ENV['encryption_key1'] %>
+encryption_key2: <%= ENV['encryption_key2'] %>
 
 # tenant config
 default:

data/template/settings/test.yml

@@ -1,6 +1,7 @@
 app_host: 'http://localhost:3001/'
 
-# encryption_key: 'This is a key that is 256 bits!!'
+encryption_key1: 'This is a key that is 256 bits!!'
+encryption_key2: 'This is a key that is 256 bits!!'
 
 default:
   environment: 'local'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maestrano-connector-rails
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.1.0
 platform: ruby
 authors:
 - Pierre Berard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-07 00:00:00.000000000 Z
+date: 2016-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: maestrano-rails
@@ -94,6 +94,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: attr_encrypted
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+- !ruby/object:Gem::Dependency
+  name: config
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: shoulda
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +248,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Maestrano is the next generation marketplace for SME applications. See
   https://maestrano.com for details.
 email: pierre.berard@maestrano.com
@@ -266,7 +308,6 @@ files:
 - app/models/maestrano/connector/rails/user_organization_rel.rb
 - bin/rails
 - config/routes.rb
-- db/20160524112054_add_encryption_on_oauth_keys.rb
 - db/migrate/20151122162100_create_users.rb
 - db/migrate/20151122162414_create_organizations.rb
 - db/migrate/20151122162613_create_user_organization_rels.rb
@@ -275,6 +316,8 @@ files:
 - db/migrate/20160205132857_add_sync_enabled_to_organizations.rb
 - db/migrate/20160215103120_add_name_to_id_map.rb
 - db/migrate/20160427112250_add_inactive_to_idmaps.rb
+- db/migrate/20160614114401_add_date_filtering_limit_to_organization.rb
+- db/migrate/20160614160654_add_encryption_on_oauth_keys.rb
 - lib/generators/connector/USAGE
 - lib/generators/connector/complex_entity_generator.rb
 - lib/generators/connector/install_generator.rb
@@ -288,6 +331,7 @@ files:
 - lib/generators/connector/templates/example_entity.rb
 - lib/generators/connector/templates/example_entity_spec.rb
 - lib/generators/connector/templates/external.rb
+- lib/generators/connector/templates/home.js
 - lib/generators/connector/templates/home_controller.rb
 - lib/generators/connector/templates/home_controller_spec.rb
 - lib/generators/connector/templates/home_index.haml
@@ -355,6 +399,11 @@ files:
 - spec/dummy/config/locales/en.yml
 - spec/dummy/config/routes.rb
 - spec/dummy/config/secrets.yml
+- spec/dummy/config/settings.yml
+- spec/dummy/config/settings/development.yml
+- spec/dummy/config/settings/production.yml
+- spec/dummy/config/settings/test.yml
+- spec/dummy/config/settings/uat.yml
 - spec/dummy/db/schema.rb
 - spec/dummy/lib/assets/.keep
 - spec/dummy/log/.keep
@@ -384,6 +433,7 @@ files:
 - spec/routing/connec_routing_spec.rb
 - spec/spec_helper.rb
 - template/Procfile
+- template/application.yml.sample
 - template/database.yml
 - template/factories.rb
 - template/gitignore

data/db/20160524112054_add_encryption_on_oauth_keys.rb

@@ -1,8 +0,0 @@
-class AddEncryptionOnOauthKeys < ActiveRecord::Migration
-  def change
-    rename_column :organizations, :oauth_token, :encrypted_oauth_token
-    add_column :organizations, :encrypted_oauth_token_iv, :string
-    rename_column :organizations, :refresh_token, :encrypted_refresh_token
-    add_column :organizations, :encrypted_refresh_token_iv, :string
-  end
-end