maestrano-connector-rails 0.4.4 → 1.0.0

data/app/models/maestrano/connector/rails/concerns/external.rb

@@ -9,5 +9,12 @@ module Maestrano::Connector::Rails::Concerns::External
     def external_name
       raise 'Not implemented'
     end
+
+    # Return an array of all the entities that the connector can synchronize
+    # If you add new entities, you need to generate
+    # a migration to add them to existing organizations
+    def entities_list
+      raise "Not implemented"
+    end
   end
 end

data/app/models/maestrano/connector/rails/concerns/sub_entity_base.rb

@@ -34,34 +34,6 @@ module Maestrano::Connector::Rails::Concerns::SubEntityBase
       end
     end
 
-    def create_idmap_from_external_entity(entity, connec_entity_name, organization)
-      if external?
-        h = names_hash.merge({
-          external_id: id_from_external_entity_hash(entity),
-          name: object_name_from_external_entity_hash(entity),
-          connec_entity: connec_entity_name.downcase,
-          organization_id: organization.id
-        })
-        Maestrano::Connector::Rails::IdMap.create(h)
-      else
-        raise 'Forbidden call: cannot call create_idmap_from_external_entity for a connec entity'
-      end
-    end
-
-    def create_idmap_from_connec_entity(entity, external_entity_name, organization)
-      if external?
-        raise 'Forbidden call: cannot call create_idmap_from_connec_entity for an external entity'
-      else
-        h = names_hash.merge({
-          connec_id: entity['id'],
-          name: object_name_from_connec_entity_hash(entity),
-          external_entity: external_entity_name.downcase,
-          organization_id: organization.id
-        })
-        Maestrano::Connector::Rails::IdMap.create(h)
-      end
-    end
-
     # { 'External Entity'  => LalaMapper, 'Other external entity' => LiliMapper }
     # or { 'Connec Entity'  => LalaMapper, 'Other connec entity' => LiliMapper }
     def mapper_classes
@@ -69,8 +41,8 @@ module Maestrano::Connector::Rails::Concerns::SubEntityBase
     end
 
     # {
-    #   'External Entity' => [{reference_class: Entities::.., connec_field: '', external_field: ''}],
-    #   'Other external entity' => [{reference_class: Entities::.., connec_field: '', external_field: ''}]
+    #   'External Entity' => ['organization_id'],
+    #   'Other external entity' => ['an array of the connec reference fields']
     # }
     def references
       {}
@@ -78,28 +50,27 @@ module Maestrano::Connector::Rails::Concerns::SubEntityBase
   end
 
 
-  def map_to(name, entity, organization)
+  def map_to(name, entity)
     mapper = self.class.mapper_classes[name]
     raise "Impossible mapping from #{self.class.entity_name} to #{name}" unless mapper
 
-    ref_hash = {}
-    if self.class.references[name]
-      self.class.references[name].each do |ref|
-        field = self.class.external? ? ref[:connec_field] : ref[:external_field]
-        ref_hash.merge! field.split('/').reverse.inject(Maestrano::Connector::Rails::Entity.id_from_ref(entity, ref, self.class.external?, organization)) { |a, n| { n.to_sym => a } }
-      end
-    end
-
     if self.class.external?
-      mapped_entity = mapper.denormalize(entity)
+      mapped_entity = mapper.denormalize(entity).merge(id: self.class.id_from_external_entity_hash(entity))
+      folded_entity = Maestrano::Connector::Rails::ConnecHelper.fold_references(mapped_entity, self.class.references[name] || [], @organization)
+      folded_entity.merge!(opts: (folded_entity[:opts] || {}).merge(matching_fields: self.class.connec_matching_fields)) if self.class.connec_matching_fields
+      folded_entity
     else
+      connec_id = entity[:__connec_id]
       mapped_entity = mapper.normalize(entity)
+      (connec_id ? mapped_entity.merge(__connec_id: connec_id) : mapped_entity).with_indifferent_access
     end
-    mapped_entity.merge(ref_hash)
   end
 
+  def map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
+    {entity: map_to(external_entity_name, connec_entity), idmap: idmap}
+  end
 
-  def map_external_entity_with_idmap(external_entity, connec_entity_name, idmap, organization)
-    {entity: map_to(connec_entity_name, external_entity, organization), idmap: idmap}
+  def map_external_entity_with_idmap(external_entity, connec_entity_name, idmap)
+    {entity: map_to(connec_entity_name, external_entity), idmap: idmap}
   end
 end

data/app/models/maestrano/connector/rails/connec_helper.rb

@@ -0,0 +1,5 @@
+module Maestrano::Connector::Rails
+  class ConnecHelper
+    include Maestrano::Connector::Rails::Concerns::ConnecHelper
+  end
+end

data/app/models/maestrano/connector/rails/organization.rb

@@ -1,6 +1,5 @@
 module Maestrano::Connector::Rails
   class Organization < ActiveRecord::Base
-
     # Enable Maestrano for this group
     maestrano_group_via :provider, :uid, :tenant do |group, maestrano|
       group.name = (maestrano.name.blank? ? "Default Group name" : maestrano.name)
@@ -13,11 +12,17 @@ module Maestrano::Connector::Rails
     def initialize
       super
       self.synchronized_entities = {}
-      Entity.entities_list.each do |entity|
+      External.entities_list.each do |entity|
         self.synchronized_entities[entity.to_sym] = true
       end
     end
 
+    #===================================
+    # Encryptions
+    #===================================
+    # attr_encrypted :oauth_token, key: ::Settings.encryption_key
+    # attr_encrypted :refresh_token, key: ::Settings.encryption_key
+
     #===================================
     # Associations
     #===================================
@@ -31,6 +36,7 @@ module Maestrano::Connector::Rails
     #===================================
     validates :name, presence: true
     validates :tenant, presence: true
+    validates :uid, uniqueness: true
 
     #===================================
     # Serialized field

data/db/20160524112054_add_encryption_on_oauth_keys.rb

@@ -0,0 +1,8 @@
+class AddEncryptionOnOauthKeys < ActiveRecord::Migration
+  def change
+    rename_column :organizations, :oauth_token, :encrypted_oauth_token
+    add_column :organizations, :encrypted_oauth_token_iv, :string
+    rename_column :organizations, :refresh_token, :encrypted_refresh_token
+    add_column :organizations, :encrypted_refresh_token_iv, :string
+  end
+end

data/lib/generators/connector/install_generator.rb

@@ -31,6 +31,7 @@ module Connector
 
       def copy_example_entity
         copy_file 'example_entity.rb', 'app/models/entities/example_entitiy.rb'
+        copy_file 'example_entity_spec.rb', 'spec/models/entities/example_entitiy.rb'
       end
 
       def copy_controllers_and_views

data/lib/generators/connector/templates/complex_entity_example/contact.rb

@@ -1,6 +1,6 @@
 # TODO
 # This file is provided as an example and should be removed
-# See README for explanation
+# See doc for explanation
 # class Entities::SubEntities::Contact < Maestrano::Connector::Rails::SubEntityBase
 
 #   def self.external?

data/lib/generators/connector/templates/complex_entity_example/contact_and_lead.rb

@@ -23,7 +23,7 @@
 #   #               external_entities_names[1]: [unmapped_connec_entitiy4]
 #   #             }
 #   #          }
-#   def connec_model_to_external_model(connec_hash_of_entities, organization)
+#   def connec_model_to_external_model(connec_hash_of_entities)
 #     people = connec_hash_of_entities['person']
 #     modeled_connec_entities = {'person' => { 'lead' => [], 'contact' => [] }}
 
@@ -50,7 +50,7 @@
 #   #               connec_entity_names[0]: [unmapped_external_entity3, unmapped_external_entity4]
 #   #             }
 #   #           }
-#   def external_model_to_connec_model(external_hash_of_entities, organization)
+#   def external_model_to_connec_model(external_hash_of_entities)
 #     return {'lead' => {'person' => external_hash_of_entities['lead']}, 'contact' => {'person' => external_hash_of_entities['contact']}}
 #   end
 # end

data/lib/generators/connector/templates/entity.rb

@@ -1,35 +1,29 @@
 class Maestrano::Connector::Rails::Entity
   include Maestrano::Connector::Rails::Concerns::Entity
 
-  # Return an array of all the entities that the connector can synchronize
-  # If you add new entities, you need to generate
-  # a migration to add them to existing organizations
-  def self.entities_list
-    # TODO
-    # The names in this list should match the names of your entities class
-    # e.g %w(person, tasks_list)
-    #  will synchronized Entities::Person and Entities::TasksList
-    []
-  end
+  # In this class and in all entities which inherit from it, the following instance variables are available:
+  # * @organization
+  # * @connec_client
+  # * @external_client
+  # * @opts
 
   # Return an array of entities from the external app
-  def get_external_entities(client, last_synchronization, organization, opts={})
-    return [] unless self.class.can_read_external?
-    Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Fetching #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize}")
+  def get_external_entities(last_synchronization)
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize}")
     # TODO
     # This method should return only entities that have been updated since the last_synchronization
-    # It should also implements an option to do a full synchronization when opts[:full_sync] == true or when there is no last_synchronization
-    # Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Received data: Source=#{Maestrano::Connector::Rails::External.external_name}, Entity=#{self.class.external_entity_name}, Response=#{entities}")
+    # It should also implements an option to do a full synchronization when @opts[:full_sync] == true or when there is no last_synchronization
+    # Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Received data: Source=#{Maestrano::Connector::Rails::External.external_name}, Entity=#{self.class.external_entity_name}, Response=#{entities}")
   end
 
-  def create_external_entity(client, mapped_connec_entity, external_entity_name, organization)
-    Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending create #{external_entity_name}: #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
+  def create_external_entity(mapped_connec_entity, external_entity_name)
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending create #{external_entity_name}: #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
     # TODO
     # This method creates the entity in the external app and returns the external id
   end
 
-  def update_external_entity(client, mapped_connec_entity, external_id, external_entity_name, organization)
-    Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending update #{external_entity_name} (id=#{external_id}): #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
+  def update_external_entity(mapped_connec_entity, external_id, external_entity_name)
+    Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending update #{external_entity_name} (id=#{external_id}): #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
     # TODO
     # This method updates the entity with the given id in the external app
   end

data/lib/generators/connector/templates/example_entity.rb

@@ -29,7 +29,7 @@
 # class ExampleEntityMapper
 #   extend HashMapper
 
-#   map from('/title'), to('/Salutation')
-#   map from('/first_name'), to('/FirstName')
+#   map from('title'), to('Salutation')
+#   map from('first_name'), to('FirstName')
 #   map from('address_work/billing2/city'), to('City')
 # end

data/lib/generators/connector/templates/example_entity_spec.rb

@@ -0,0 +1,73 @@
+# TODO
+# This file is provided as an example and should be removed
+# require 'spec_helper'
+
+# describe Entities::ExampleEntity do
+
+#   describe 'class methods' do
+#     subject { Entities::ExampleEntity }
+
+#     it { expect(subject.connec_entity_name).to eql('ExampleEntity') }
+#     it { expect(subject.external_entity_name).to eql('Contact') }
+#     it { expect(subject.object_name_from_connec_entity_hash({'first_name' => 'A', 'last_name' => 'contact'})).to eql('A contact') }
+#     it { expect(subject.object_name_from_external_entity_hash({'FirstName' => 'A', 'LastName' => 'contact'})).to eql('A contact') }
+#   end
+
+#   describe 'instance methods' do
+#     let(:organization) { create(:organization) }
+#     let(:connec_client) { Maestrano::Connec::Client[organization.tenant].new(organization.uid) }
+#     let(:external_client) { Maestrano::Connector::Rails::External.get_client(organization) }
+#     let(:opts) { {} }
+#     subject { Entities::ExampleEntity.new(organization, connec_client, external_client, opts) }
+
+#     describe 'external to connec!' do
+#       let(:external_hash) {
+#         {
+#           "Id" => '2345uoi'
+#           "Salutation" => 'Mr',
+#           "FirstName" => 'John',
+#           "City" => 'London'
+#         }
+#       }
+
+#       let (:mapped_external_hash) {
+#         {
+#           "id" => [{'id' => '2345uoi', 'provider' => organization.oauth_provider, 'realm' => organization.oauth_uid}],
+#           "first_name" => "John",
+#           "title" => "Mr"
+#           "address_work" => {
+#             "billing2" => {
+#               "city" => 'London'
+#             }
+#           }
+#         }.with_indifferent_access
+#       }
+
+#       it { expect(subject.map_to_connec(external_hash)).to eql(mapped_external_hash) }
+#     end
+
+#     describe 'connec to salesforce' do
+#       let(:connec_hash) {
+#         {
+#           "first_name" => "John",
+#           "title" => "Mr"
+#           "address_work" => {
+#             "billing2" => {
+#               "city" => 'London'
+#             }
+#           }
+#         }
+#       }
+
+#       let(:mapped_connec_hash) {
+#         {
+#           "Salutation" => 'Mr',
+#           "FirstName" => 'John',
+#           "City" => 'London'
+#         }.with_indifferent_access
+#       }
+
+#       it { expect(subject.map_to_external(connec_hash)).to eql(mapped_connec_hash) }
+#     end
+#   end
+# end

data/lib/generators/connector/templates/external.rb

@@ -16,4 +16,15 @@ class Maestrano::Connector::Rails::External
     # client_id: ENV[''],
     # client_secret: ENV['']
   end
+
+  # Return an array of all the entities that the connector can synchronize
+  # If you add new entities, you need to generate
+  # a migration to add them to existing organizations
+  def self.entities_list
+    # TODO
+    # The names in this list should match the names of your entities class
+    # e.g %w(person, tasks_list)
+    #  will synchronized Entities::Person and Entities::TasksList
+    []
+  end
 end

data/maestrano-connector-rails.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: maestrano-connector-rails 0.4.4 ruby lib
+# stub: maestrano-connector-rails 1.0.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "maestrano-connector-rails"
-  s.version = "0.4.4"
+  s.version = "1.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Pierre Berard"]
-  s.date = "2016-05-11"
+  s.date = "2016-05-25"
   s.description = "Maestrano is the next generation marketplace for SME applications. See https://maestrano.com for details."
   s.email = "pierre.berard@maestrano.com"
   s.executables = ["rails"]
@@ -23,7 +23,6 @@ Gem::Specification.new do |s|
     ".rspec",
     "DEVELOPER.md",
     "Gemfile",
-    "Gemfile.lock",
     "LICENSE",
     "README.md",
     "Rakefile",
@@ -40,10 +39,12 @@ Gem::Specification.new do |s|
     "app/jobs/maestrano/connector/rails/synchronization_job.rb",
     "app/models/maestrano/connector/rails/complex_entity.rb",
     "app/models/maestrano/connector/rails/concerns/complex_entity.rb",
+    "app/models/maestrano/connector/rails/concerns/connec_helper.rb",
     "app/models/maestrano/connector/rails/concerns/connector_logger.rb",
     "app/models/maestrano/connector/rails/concerns/entity.rb",
     "app/models/maestrano/connector/rails/concerns/external.rb",
     "app/models/maestrano/connector/rails/concerns/sub_entity_base.rb",
+    "app/models/maestrano/connector/rails/connec_helper.rb",
     "app/models/maestrano/connector/rails/connector_logger.rb",
     "app/models/maestrano/connector/rails/entity.rb",
     "app/models/maestrano/connector/rails/external.rb",
@@ -55,6 +56,7 @@ Gem::Specification.new do |s|
     "app/models/maestrano/connector/rails/user_organization_rel.rb",
     "bin/rails",
     "config/routes.rb",
+    "db/20160524112054_add_encryption_on_oauth_keys.rb",
     "db/migrate/20151122162100_create_users.rb",
     "db/migrate/20151122162414_create_organizations.rb",
     "db/migrate/20151122162613_create_user_organization_rels.rb",
@@ -74,6 +76,7 @@ Gem::Specification.new do |s|
     "lib/generators/connector/templates/complex_entity_example/person.rb",
     "lib/generators/connector/templates/entity.rb",
     "lib/generators/connector/templates/example_entity.rb",
+    "lib/generators/connector/templates/example_entity_spec.rb",
     "lib/generators/connector/templates/external.rb",
     "lib/generators/connector/templates/home_controller.rb",
     "lib/generators/connector/templates/home_controller_spec.rb",
@@ -95,7 +98,7 @@ Gem::Specification.new do |s|
     "lib/maestrano/connector/rails.rb",
     "maestrano-connector-rails.gemspec",
     "maestrano.png",
-    "realse_notes.md",
+    "release_notes.md",
     "spec/controllers/connec_controller_spec.rb",
     "spec/controllers/group_users_controller_spec.rb",
     "spec/controllers/groups_controller_spec.rb",
@@ -116,7 +119,6 @@ Gem::Specification.new do |s|
     "spec/dummy/app/models/entities/example_entitiy.rb",
     "spec/dummy/app/models/maestrano/connector/rails/entity.rb",
     "spec/dummy/app/models/maestrano/connector/rails/external.rb",
-    "spec/dummy/app/views/admin/index.html.erb",
     "spec/dummy/app/views/home/index.html.erb",
     "spec/dummy/app/views/layouts/application.html.erb",
     "spec/dummy/bin/bundle",
@@ -143,9 +145,7 @@ Gem::Specification.new do |s|
     "spec/dummy/config/locales/en.yml",
     "spec/dummy/config/routes.rb",
     "spec/dummy/config/secrets.yml",
-    "spec/dummy/db/development.sqlite3",
     "spec/dummy/db/schema.rb",
-    "spec/dummy/db/test.sqlite3",
     "spec/dummy/lib/assets/.keep",
     "spec/dummy/log/.keep",
     "spec/dummy/public/404.html",
@@ -153,10 +153,15 @@ Gem::Specification.new do |s|
     "spec/dummy/public/500.html",
     "spec/dummy/public/favicon.ico",
     "spec/factories.rb",
+    "spec/integration/connec_to_external_spec.rb",
+    "spec/integration/external_to_connec_spec.rb",
+    "spec/integration/integration_complex_spec.rb",
+    "spec/integration/singleton_spec.rb",
     "spec/jobs/all_synchronizations_job_spec.rb",
     "spec/jobs/push_to_connec_job_spec.rb",
     "spec/jobs/synchronization_job_spec.rb",
     "spec/models/complex_entity_spec.rb",
+    "spec/models/connec_helper_spec.rb",
     "spec/models/connector_logger_spec.rb",
     "spec/models/entity_spec.rb",
     "spec/models/external_spec.rb",

data/release_notes.md

@@ -0,0 +1,81 @@
+## 1.0.0
+
+### Features
+* Reference in mapping greatly simplified. See the documentation for an explanation and example: [here](https://maestrano.atlassian.net/wiki/display/DEV/Mapping+and+synchronization#Mappingandsynchronization-References).
+* Smart merging available. You can specifiy field on which Connec! will attempt to merge the record with an existing one. See the framework [documentation](https://maestrano.atlassian.net/wiki/display/DEV/Examples#Examples-Smartmerging) for an example, as well as the Connec! [documentation](http://maestrano.github.io/connec/#api-|-save-data-resource-creation-post)
+
+### Breaking changes
+A major refactoring as lead to some breaking changes:
+* `Entity` and `ComplexEntity` `initialize` method now take 3-4 argument instead of 0: `organization`, `connec_client`, `external_client` and `opts`
+* All the **instance** methods of those classes that previously took one of these arguments have been change to not include them anymore. Full list:
+
+**ComplexEntity & Entity:**
+**`connec_model_to_external_model`**
+**`external_model_to_connec_model`**
+`get_connec_entities`
+**`get_external_entities`**
+`consolidate_and_map_data`
+`push_entities_to_connec`
+`push_entities_to_external`
+**`before_sync`**
+**`after_sync`**
+**`filter_connec_entities`**
+
+
+**Entity:**
+**`map_to_external`**
+**`map_to_connec`**
+`push_entities_to_connec_to`
+`batch_op`
+`push_entities_to_external_to`
+`push_entity_to_external`
+**`create_external_entity`**
+**`update_external_entity`**
+`consolidate_and_map_singleton`
+`map_external_entity_with_idmap`
+
+**SubEntityBase:**
+**`map_to`**
+
+* The class method `entities_list` has been moved from `Entity` to `External`
+* The `references` method has been changed with the new reference system. The framework now expect it to be an array os strings instead of an array of hashes. Furthermore, it now supports reference fields embedded in hashes and arrays.
+* The following method from `Entity` as been changed from class methods to instance methods (and their arguments have been changed):
+`not_modified_since_last_push_to_connec?`
+`is_external_more_recent?`
+`solve_conflict`
+
+* The following methods have been removed
+`map_to_external_with_idmap`
+`create_idmap_from_external_entity`
+`create_idmap_from_connec_entity`
+`can_update_connec?`
+`id_from_ref`
+
+
+- - - -
+- - - -
+
+## 0.4.4
+
+### Features
+* Add filter_connec_entities method to allow filtering in webhook workflow
+
+### Fixes
+* Truncate idmap message to avoid database errors
+
+## 0.4.3
+
+### Fixes
+* Add `forced` option in home_controller when requesting a manual synchronization.
+
+
+## 0.4.2
+/!\ This release contains a new migration that you'll need to fetch and run
+
+### Features
+* Add call to before and after_sync methods during synchronization workflows
+* Add logic for record flagged as inactive in the external application
+
+### Fixes
+* Fix recovery mode: recovery mode was previously on as soon as the organization had three failed synchronizations, even if a synchronization had succeed afterward. Moreover, recovery mode doesn't prevent manual synchronization anymore.
+* Fix a typo in saml controller that prevented deletion of a session param.