maestrano-connector-rails 1.2.1 → 1.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +41 -0
- data/.rubocop_todo.yml +180 -0
- data/.ruby-version +1 -1
- data/CODESHIP.md +3 -1
- data/Gemfile +16 -10
- data/Rakefile +12 -12
- data/VERSION +1 -1
- data/app/controllers/maestrano/account/group_users_controller.rb +5 -7
- data/app/controllers/maestrano/account/groups_controller.rb +5 -7
- data/app/controllers/maestrano/application_controller.rb +1 -1
- data/app/controllers/maestrano/auth/saml_controller.rb +5 -10
- data/app/controllers/maestrano/connec_controller.rb +12 -11
- data/app/controllers/maestrano/sessions_controller.rb +1 -1
- data/app/controllers/maestrano/synchronizations_controller.rb +7 -9
- data/app/controllers/version_controller.rb +9 -0
- data/app/helpers/maestrano/connector/rails/session_helper.rb +0 -2
- data/app/jobs/maestrano/connector/rails/all_synchronizations_job.rb +2 -2
- data/app/jobs/maestrano/connector/rails/push_to_connec_job.rb +8 -7
- data/app/jobs/maestrano/connector/rails/push_to_connec_worker.rb +6 -4
- data/app/jobs/maestrano/connector/rails/synchronization_job.rb +10 -9
- data/app/models/maestrano/connector/rails/complex_entity.rb +1 -1
- data/app/models/maestrano/connector/rails/concerns/complex_entity.rb +10 -12
- data/app/models/maestrano/connector/rails/concerns/connec_helper.rb +20 -25
- data/app/models/maestrano/connector/rails/concerns/connector_logger.rb +1 -1
- data/app/models/maestrano/connector/rails/concerns/entity.rb +63 -69
- data/app/models/maestrano/connector/rails/concerns/entity_base.rb +3 -3
- data/app/models/maestrano/connector/rails/concerns/external.rb +2 -2
- data/app/models/maestrano/connector/rails/concerns/sub_entity_base.rb +13 -15
- data/app/models/maestrano/connector/rails/connector_logger.rb +1 -1
- data/app/models/maestrano/connector/rails/entity.rb +1 -1
- data/app/models/maestrano/connector/rails/external.rb +1 -1
- data/app/models/maestrano/connector/rails/id_map.rb +1 -2
- data/app/models/maestrano/connector/rails/organization.rb +12 -14
- data/app/models/maestrano/connector/rails/sub_entity_base.rb +1 -1
- data/app/models/maestrano/connector/rails/synchronization.rb +17 -15
- data/app/models/maestrano/connector/rails/user.rb +1 -2
- data/app/models/maestrano/connector/rails/user_organization_rel.rb +1 -2
- data/config/routes.rb +4 -2
- data/lib/maestrano/connector/rails.rb +17 -2
- data/lib/maestrano_connector_rails.rb +1 -0
- data/maestrano-connector-rails.gemspec +32 -13
- data/release_notes.md +5 -0
- data/spec/controllers/version_controller_spec.rb +17 -0
- data/spec/dummy/config/application.rb +1 -1
- data/spec/jobs/push_to_connec_worker_spec.rb +16 -5
- data/spec/models/synchronization_spec.rb +9 -9
- data/spec/models/user_organization_rel_spec.rb +1 -1
- data/template/maestrano-connector-template.rb +2 -17
- metadata +90 -16
- data/lib/maestrano-connector-rails.rb +0 -1
@@ -2,7 +2,7 @@ class Maestrano::SynchronizationsController < Maestrano::Rails::WebHookControlle
|
|
2
2
|
def show
|
3
3
|
uid = params[:id]
|
4
4
|
organization = Maestrano::Connector::Rails::Organization.find_by_uid(uid)
|
5
|
-
return render json: {
|
5
|
+
return render json: {errors: {message: "Organization not found", code: 404}}, status: :not_found unless organization
|
6
6
|
|
7
7
|
h = {
|
8
8
|
group_id: organization.uid,
|
@@ -12,11 +12,9 @@ class Maestrano::SynchronizationsController < Maestrano::Rails::WebHookControlle
|
|
12
12
|
last_sync = organization.synchronizations.last
|
13
13
|
if last_sync
|
14
14
|
h.merge!(
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
updated_at: last_sync.updated_at
|
19
|
-
}
|
15
|
+
status: last_sync.status,
|
16
|
+
message: last_sync.message,
|
17
|
+
updated_at: last_sync.updated_at
|
20
18
|
)
|
21
19
|
end
|
22
20
|
|
@@ -27,7 +25,7 @@ class Maestrano::SynchronizationsController < Maestrano::Rails::WebHookControlle
|
|
27
25
|
uid = params[:group_id]
|
28
26
|
opts = params[:opts] || {}
|
29
27
|
organization = Maestrano::Connector::Rails::Organization.find_by_uid(uid)
|
30
|
-
return render json: {
|
28
|
+
return render json: {errors: {message: "Organization not found", code: 404}}, status: :not_found unless organization
|
31
29
|
|
32
30
|
Maestrano::Connector::Rails::SynchronizationJob.perform_later(organization, opts.with_indifferent_access)
|
33
31
|
head :created
|
@@ -36,11 +34,11 @@ class Maestrano::SynchronizationsController < Maestrano::Rails::WebHookControlle
|
|
36
34
|
def toggle_sync
|
37
35
|
uid = params[:group_id]
|
38
36
|
organization = Maestrano::Connector::Rails::Organization.find_by_uid(uid)
|
39
|
-
return render json: {
|
37
|
+
return render json: {errors: {message: "Organization not found", code: 404}}, status: :not_found unless organization
|
40
38
|
|
41
39
|
organization.toggle(:sync_enabled)
|
42
40
|
organization.save
|
43
41
|
|
44
42
|
render json: {sync_enabled: organization.sync_enabled}
|
45
43
|
end
|
46
|
-
end
|
44
|
+
end
|
@@ -0,0 +1,9 @@
|
|
1
|
+
class VersionController < ApplicationController
|
2
|
+
def index
|
3
|
+
framework_version = Gem.loaded_specs['maestrano-connector-rails'].version.version
|
4
|
+
respond_to do |format|
|
5
|
+
format.html { render text: "framework_version=#{framework_version}\n" }
|
6
|
+
format.json { render json: {framework_version: framework_version} }
|
7
|
+
end
|
8
|
+
end
|
9
|
+
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
module Maestrano::Connector::Rails
|
2
2
|
module SessionHelper
|
3
|
-
|
4
3
|
def is_admin?(user, organization)
|
5
4
|
organization.member?(user) && session[:"role_#{organization.uid}"] && ['Admin', 'Super Admin'].include?(session[:"role_#{organization.uid}"])
|
6
5
|
end
|
@@ -16,6 +15,5 @@ module Maestrano::Connector::Rails
|
|
16
15
|
def is_admin
|
17
16
|
@is_admin ||= current_user && current_organization && is_admin?(current_user, current_organization)
|
18
17
|
end
|
19
|
-
|
20
18
|
end
|
21
19
|
end
|
@@ -3,11 +3,11 @@ module Maestrano::Connector::Rails
|
|
3
3
|
queue_as :default
|
4
4
|
|
5
5
|
# Trigger synchronization of all active organizations
|
6
|
-
def perform(name=nil, count=nil)
|
6
|
+
def perform(name = nil, count = nil)
|
7
7
|
Maestrano::Connector::Rails::Organization.where.not(oauth_provider: nil, encrypted_oauth_token: nil).each do |o|
|
8
8
|
next unless [true, 1].include?(o.sync_enabled)
|
9
9
|
Maestrano::Connector::Rails::SynchronizationJob.perform_later(o, {})
|
10
10
|
end
|
11
11
|
end
|
12
12
|
end
|
13
|
-
end
|
13
|
+
end
|
@@ -3,7 +3,7 @@ module Maestrano::Connector::Rails
|
|
3
3
|
queue_as :default
|
4
4
|
|
5
5
|
# expected hash: {"external_entity_name1" => [entity1, entity2], "external_entity_name2" => []}
|
6
|
-
def perform(organization, entities_hash, opts={})
|
6
|
+
def perform(organization, entities_hash, opts = {})
|
7
7
|
return unless organization.sync_enabled && organization.oauth_uid
|
8
8
|
|
9
9
|
connec_client = Maestrano::Connector::Rails::ConnecHelper.get_client(organization)
|
@@ -18,11 +18,11 @@ module Maestrano::Connector::Rails
|
|
18
18
|
|
19
19
|
entity_instance.before_sync(last_synchronization_date)
|
20
20
|
# Build expected input for consolidate_and_map_data
|
21
|
-
if entity_instance_hash[:is_complex]
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
21
|
+
mapped_entities = if entity_instance_hash[:is_complex]
|
22
|
+
entity_instance.consolidate_and_map_data(ComplexEntity.build_empty_hash(entity_instance.class.connec_entities_names), ComplexEntity.build_hash_with_entities(entity_instance.class.external_entities_names, external_entity_name, ->(name) { name }, entities))
|
23
|
+
else
|
24
|
+
entity_instance.consolidate_and_map_data([], entities)
|
25
|
+
end
|
26
26
|
entity_instance.push_entities_to_connec(mapped_entities[:external_entities])
|
27
27
|
|
28
28
|
entity_instance.after_sync(last_synchronization_date)
|
@@ -33,6 +33,7 @@ module Maestrano::Connector::Rails
|
|
33
33
|
end
|
34
34
|
|
35
35
|
private
|
36
|
+
|
36
37
|
def find_entity_instance(entity_name, organization, connec_client, external_client, opts)
|
37
38
|
Maestrano::Connector::Rails::External.entities_list.each do |entity_name_from_list|
|
38
39
|
clazz = "Entities::#{entity_name_from_list.singularize.titleize.split.join}".constantize
|
@@ -45,4 +46,4 @@ module Maestrano::Connector::Rails
|
|
45
46
|
nil
|
46
47
|
end
|
47
48
|
end
|
48
|
-
end
|
49
|
+
end
|
@@ -7,13 +7,15 @@ module Maestrano::Connector::Rails
|
|
7
7
|
sidekiq_options unique: :while_executing, unique_args: :unique_args
|
8
8
|
|
9
9
|
def self.unique_args(args)
|
10
|
-
|
10
|
+
organization_id = args[0]
|
11
11
|
entities_hash = args[1]
|
12
|
-
|
12
|
+
|
13
|
+
[organization_id, entities_hash.keys.sort]
|
13
14
|
end
|
14
15
|
|
15
|
-
def perform(
|
16
|
+
def perform(organization_id, entities_hash, opts = {})
|
17
|
+
organization = Organization.find(organization_id)
|
16
18
|
PushToConnecJob.new.perform(organization, entities_hash, opts)
|
17
19
|
end
|
18
20
|
end
|
19
|
-
end
|
21
|
+
end
|
@@ -7,19 +7,19 @@ module Maestrano::Connector::Rails
|
|
7
7
|
# * :only_entities => [person, tasks_list]
|
8
8
|
# * :full_sync => true synchronization is performed without date filtering
|
9
9
|
# * :connec_preemption => true|false : preemption is always|never given to connec in case of conflict (if not set, the most recently updated entity is kept)
|
10
|
-
def perform(organization, opts={})
|
10
|
+
def perform(organization, opts = {})
|
11
11
|
return unless organization.sync_enabled
|
12
12
|
|
13
13
|
# Check if previous synchronization is still running
|
14
14
|
if Synchronization.where(organization_id: organization.id, status: 'RUNNING').where(created_at: (30.minutes.ago..Time.now)).exists?
|
15
|
-
ConnectorLogger.log('info', organization,
|
15
|
+
ConnectorLogger.log('info', organization, 'Synchronization skipped: Previous synchronization is still running')
|
16
16
|
return
|
17
17
|
end
|
18
18
|
|
19
19
|
# Check if recovery mode: last 3 synchronizations have failed
|
20
20
|
if !opts[:forced] && organization.last_three_synchronizations_failed? \
|
21
21
|
&& organization.synchronizations.order(created_at: :desc).limit(1).first.updated_at > 1.day.ago
|
22
|
-
ConnectorLogger.log('info', organization,
|
22
|
+
ConnectorLogger.log('info', organization, 'Synchronization skipped: Recovery mode (three previous synchronizations have failed)')
|
23
23
|
return
|
24
24
|
end
|
25
25
|
|
@@ -37,8 +37,8 @@ module Maestrano::Connector::Rails
|
|
37
37
|
# We do a doube sync: only from external, then only from connec!
|
38
38
|
# We also do batched sync as the first one can be quite huge
|
39
39
|
if last_synchronization.nil?
|
40
|
-
ConnectorLogger.log('info', organization,
|
41
|
-
organization.synchronized_entities.select{|k, v| v}.keys.each do |entity|
|
40
|
+
ConnectorLogger.log('info', organization, 'First synchronization ever. Doing two half syncs to allow smart merging to work its magic.')
|
41
|
+
organization.synchronized_entities.select { |k, v| v }.keys.each do |entity|
|
42
42
|
ConnectorLogger.log('info', organization, "First synchronization ever. Doing half sync from external for #{entity}.")
|
43
43
|
first_sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization_date, opts, true)
|
44
44
|
ConnectorLogger.log('info', organization, "First synchronization ever. Doing half sync from Connec! for #{entity}.")
|
@@ -52,7 +52,7 @@ module Maestrano::Connector::Rails
|
|
52
52
|
sync_entity(entity, organization, connec_client, external_client, last_synchronization_date, opts)
|
53
53
|
end
|
54
54
|
else
|
55
|
-
organization.synchronized_entities.select{|
|
55
|
+
organization.synchronized_entities.select { |_k, v| v }.keys.each do |entity|
|
56
56
|
sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization_date, opts)
|
57
57
|
end
|
58
58
|
end
|
@@ -78,7 +78,7 @@ module Maestrano::Connector::Rails
|
|
78
78
|
entities_count = limit
|
79
79
|
|
80
80
|
h = {__limit: limit}
|
81
|
-
external ? h
|
81
|
+
external ? h[:__skip_connec] = true : h[:__skip_external] = true
|
82
82
|
entity_instance = instanciate_entity(entity_name, organization, connec_client, external_client, opts.merge(h))
|
83
83
|
|
84
84
|
# IF entities_count > limit
|
@@ -87,7 +87,7 @@ module Maestrano::Connector::Rails
|
|
87
87
|
# No need to fetch it a second Time
|
88
88
|
# ELSIF entities_count < limit
|
89
89
|
# No more entities to fetch
|
90
|
-
while entities_count == limit
|
90
|
+
while entities_count == limit
|
91
91
|
entity_instance.opts_merge!(__skip: skip)
|
92
92
|
entities_count = perform_sync(entity_instance, last_synchronization_date, external)
|
93
93
|
skip += limit
|
@@ -95,6 +95,7 @@ module Maestrano::Connector::Rails
|
|
95
95
|
end
|
96
96
|
|
97
97
|
private
|
98
|
+
|
98
99
|
def instanciate_entity(entity_name, organization, connec_client, external_client, opts)
|
99
100
|
"Entities::#{entity_name.titleize.split.join}".constantize.new(organization, connec_client, external_client, opts.dup)
|
100
101
|
end
|
@@ -112,4 +113,4 @@ module Maestrano::Connector::Rails
|
|
112
113
|
external ? entity_instance.class.count_entities(external_entities) : entity_instance.class.count_entities(connec_entities)
|
113
114
|
end
|
114
115
|
end
|
115
|
-
end
|
116
|
+
end
|
@@ -7,11 +7,11 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
7
7
|
# -------------------------------------------------------------
|
8
8
|
module ClassMethods
|
9
9
|
def connec_entities_names
|
10
|
-
raise
|
10
|
+
raise 'Not implemented'
|
11
11
|
end
|
12
12
|
|
13
13
|
def external_entities_names
|
14
|
-
raise
|
14
|
+
raise 'Not implemented'
|
15
15
|
end
|
16
16
|
|
17
17
|
def count_entities(entities)
|
@@ -33,7 +33,7 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
33
33
|
# }
|
34
34
|
# }
|
35
35
|
def connec_model_to_external_model(connec_hash_of_entities)
|
36
|
-
raise
|
36
|
+
raise 'Not implemented'
|
37
37
|
end
|
38
38
|
|
39
39
|
# input : {
|
@@ -50,13 +50,13 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
50
50
|
# }
|
51
51
|
# }
|
52
52
|
def external_model_to_connec_model(external_hash_of_entities)
|
53
|
-
raise
|
53
|
+
raise 'Not implemented'
|
54
54
|
end
|
55
55
|
|
56
56
|
# -------------------------------------------------------------
|
57
57
|
# Entity equivalent methods
|
58
58
|
# -------------------------------------------------------------
|
59
|
-
def get_connec_entities(last_synchronization_date=nil)
|
59
|
+
def get_connec_entities(last_synchronization_date = nil)
|
60
60
|
entities = ActiveSupport::HashWithIndifferentAccess.new
|
61
61
|
|
62
62
|
self.class.connec_entities_names.each do |connec_entity_name|
|
@@ -66,7 +66,7 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
66
66
|
entities
|
67
67
|
end
|
68
68
|
|
69
|
-
def get_external_entities_wrapper(last_synchronization_date=nil)
|
69
|
+
def get_external_entities_wrapper(last_synchronization_date = nil)
|
70
70
|
entities = ActiveSupport::HashWithIndifferentAccess.new
|
71
71
|
|
72
72
|
self.class.external_entities_names.each do |external_entity_name|
|
@@ -83,13 +83,12 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
83
83
|
mapped_connec_entities = consolidate_and_map_connec_entities(modeled_connec_entities, modeled_external_entities)
|
84
84
|
mapped_external_entities = consolidate_and_map_external_entities(modeled_external_entities)
|
85
85
|
|
86
|
-
|
86
|
+
{connec_entities: mapped_connec_entities, external_entities: mapped_external_entities}
|
87
87
|
end
|
88
88
|
|
89
89
|
def consolidate_and_map_connec_entities(modeled_connec_entities, modeled_external_entities)
|
90
90
|
modeled_connec_entities.each do |connec_entity_name, entities_in_external_model|
|
91
91
|
entities_in_external_model.each do |external_entity_name, entities|
|
92
|
-
|
93
92
|
sub_entity_instance = instantiate_sub_entity_instance(connec_entity_name)
|
94
93
|
equivalent_external_entities = (modeled_external_entities[external_entity_name] && modeled_external_entities[external_entity_name][connec_entity_name]) || []
|
95
94
|
|
@@ -128,7 +127,6 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
128
127
|
end
|
129
128
|
end
|
130
129
|
|
131
|
-
|
132
130
|
def push_entities_to_external(mapped_connec_entities_with_idmaps)
|
133
131
|
mapped_connec_entities_with_idmaps.each do |connec_entity_name, entities_in_external_model|
|
134
132
|
sub_entity_instance = instantiate_sub_entity_instance(connec_entity_name)
|
@@ -148,13 +146,13 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
148
146
|
module ClassMethods
|
149
147
|
# output : {entities_names[0] => [], entities_names[1] => []}
|
150
148
|
def build_empty_hash(entities_names)
|
151
|
-
Hash[
|
149
|
+
Hash[*entities_names.collect { |name| [name, []] }.flatten(1)]
|
152
150
|
end
|
153
151
|
|
154
152
|
# output: {entities_name[0] => [], entities_name[1] => entities}
|
155
153
|
# with proc.call(entities_name[1] == entity_name)
|
156
154
|
def build_hash_with_entities(entities_name, entity_name, proc, entities)
|
157
|
-
Hash[
|
155
|
+
Hash[*entities_name.collect { |name| proc.call(name) == entity_name ? [name, entities] : [name, []] }.flatten(1)]
|
158
156
|
end
|
159
157
|
end
|
160
|
-
end
|
158
|
+
end
|
@@ -25,7 +25,7 @@ module Maestrano::Connector::Rails::Concerns::ConnecHelper
|
|
25
25
|
end
|
26
26
|
@@connec_version
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
# Replace the ids arrays by the external id
|
30
30
|
# If a reference has no id for this oauth_provider and oauth_uid but has one for connec returns nil
|
31
31
|
def unfold_references(connec_entity, references, organization)
|
@@ -33,13 +33,9 @@ module Maestrano::Connector::Rails::Concerns::ConnecHelper
|
|
33
33
|
not_nil = true
|
34
34
|
|
35
35
|
# Id
|
36
|
-
|
37
|
-
unfolded_connec_entity[:__connec_id] = unfolded_connec_entity['id'].find{|id| id['provider'] == 'connec'}['id']
|
38
|
-
|
39
|
-
unfolded_connec_entity['id'] = id['id']
|
40
|
-
else
|
41
|
-
unfolded_connec_entity['id'] = nil
|
42
|
-
end
|
36
|
+
id_hash = unfolded_connec_entity['id'].find { |id| id['provider'] == organization.oauth_provider && id['realm'] == organization.oauth_uid }
|
37
|
+
unfolded_connec_entity[:__connec_id] = unfolded_connec_entity['id'].find { |id| id['provider'] == 'connec' }['id']
|
38
|
+
unfolded_connec_entity['id'] = id_hash ? id_hash['id'] : nil
|
43
39
|
|
44
40
|
# Other refs
|
45
41
|
references.each do |reference|
|
@@ -67,20 +63,19 @@ module Maestrano::Connector::Rails::Concerns::ConnecHelper
|
|
67
63
|
def fold_references_helper(entity, array_of_refs, organization)
|
68
64
|
ref = array_of_refs.shift
|
69
65
|
field = entity[ref]
|
66
|
+
return if field.blank?
|
70
67
|
|
71
68
|
# Follow embedment path, remplace if it's not an array or a hash
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
fold_references_helper(f, array_of_refs.dup, organization)
|
77
|
-
end
|
78
|
-
when HashWithIndifferentAccess
|
79
|
-
fold_references_helper(entity[ref], array_of_refs, organization)
|
80
|
-
else
|
81
|
-
id = field
|
82
|
-
entity[ref] = [id_hash(id, organization)]
|
69
|
+
case field
|
70
|
+
when Array
|
71
|
+
field.each do |f|
|
72
|
+
fold_references_helper(f, array_of_refs.dup, organization)
|
83
73
|
end
|
74
|
+
when HashWithIndifferentAccess
|
75
|
+
fold_references_helper(entity[ref], array_of_refs, organization)
|
76
|
+
else
|
77
|
+
id = field
|
78
|
+
entity[ref] = [id_hash(id, organization)]
|
84
79
|
end
|
85
80
|
end
|
86
81
|
|
@@ -89,10 +84,11 @@ module Maestrano::Connector::Rails::Concerns::ConnecHelper
|
|
89
84
|
field = entity[ref]
|
90
85
|
|
91
86
|
# Unfold the id
|
92
|
-
if array_of_refs.empty?
|
93
|
-
|
94
|
-
|
95
|
-
|
87
|
+
if array_of_refs.empty? && field
|
88
|
+
id_hash = field.find { |id| id[:provider] == organization.oauth_provider && id[:realm] == organization.oauth_uid }
|
89
|
+
if id_hash
|
90
|
+
entity[ref] = id_hash['id']
|
91
|
+
elsif field.find { |id| id[:provider] == 'connec' } # Should always be true as ids will always contain a connec id
|
96
92
|
# We may enqueue a fetch on the endpoint of the missing association, followed by a re-fetch on this one.
|
97
93
|
# However it's expected to be an edge case, so for now we rely on the fact that the webhooks should be relativly in order.
|
98
94
|
# Worst case it'll be done on following sync
|
@@ -114,6 +110,5 @@ module Maestrano::Connector::Rails::Concerns::ConnecHelper
|
|
114
110
|
end
|
115
111
|
true
|
116
112
|
end
|
117
|
-
|
118
113
|
end
|
119
|
-
end
|
114
|
+
end
|
@@ -19,9 +19,11 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
19
19
|
def find_or_create_idmap(organization_and_id)
|
20
20
|
Maestrano::Connector::Rails::IdMap.find_or_create_by(names_hash.merge(organization_and_id))
|
21
21
|
end
|
22
|
+
|
22
23
|
def find_idmap(organization_and_id)
|
23
24
|
Maestrano::Connector::Rails::IdMap.find_by(names_hash.merge(organization_and_id))
|
24
25
|
end
|
26
|
+
|
25
27
|
def create_idmap(organization_and_id)
|
26
28
|
Maestrano::Connector::Rails::IdMap.create(names_hash.merge(organization_and_id))
|
27
29
|
end
|
@@ -45,25 +47,25 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
45
47
|
# External methods
|
46
48
|
# ----------------------------------------------
|
47
49
|
def id_from_external_entity_hash(entity)
|
48
|
-
raise
|
50
|
+
raise 'Not implemented'
|
49
51
|
end
|
50
52
|
|
51
53
|
def last_update_date_from_external_entity_hash(entity)
|
52
|
-
raise
|
54
|
+
raise 'Not implemented'
|
53
55
|
end
|
54
56
|
|
55
57
|
def creation_date_from_external_entity_hash(entity)
|
56
|
-
raise
|
58
|
+
raise 'Not implemented'
|
57
59
|
end
|
58
60
|
|
59
61
|
# Return a string representing the object from a connec! entity hash
|
60
62
|
def object_name_from_connec_entity_hash(entity)
|
61
|
-
raise
|
63
|
+
raise 'Not implemented'
|
62
64
|
end
|
63
65
|
|
64
66
|
# Return a string representing the object from an external entity hash
|
65
67
|
def object_name_from_external_entity_hash(entity)
|
66
|
-
raise
|
68
|
+
raise 'Not implemented'
|
67
69
|
end
|
68
70
|
|
69
71
|
# Returns a boolean
|
@@ -71,6 +73,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
71
73
|
def inactive_from_external_entity_hash?(entity)
|
72
74
|
false
|
73
75
|
end
|
76
|
+
|
74
77
|
# ----------------------------------------------
|
75
78
|
# Entity specific methods
|
76
79
|
# Those methods need to be define in each entity
|
@@ -82,17 +85,17 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
82
85
|
|
83
86
|
# Entity name in Connec!
|
84
87
|
def connec_entity_name
|
85
|
-
raise
|
88
|
+
raise 'Not implemented'
|
86
89
|
end
|
87
90
|
|
88
91
|
# Entity name in external system
|
89
92
|
def external_entity_name
|
90
|
-
raise
|
93
|
+
raise 'Not implemented'
|
91
94
|
end
|
92
95
|
|
93
96
|
# Entity Mapper Class
|
94
97
|
def mapper_class
|
95
|
-
raise
|
98
|
+
raise 'Not implemented'
|
96
99
|
end
|
97
100
|
|
98
101
|
# An array of connec fields that are references
|
@@ -153,7 +156,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
153
156
|
def map_to_connec(entity)
|
154
157
|
mapped_entity = self.class.mapper_class.denormalize(entity).merge(id: self.class.id_from_external_entity_hash(entity))
|
155
158
|
folded_entity = Maestrano::Connector::Rails::ConnecHelper.fold_references(mapped_entity, self.class.references, @organization)
|
156
|
-
folded_entity
|
159
|
+
folded_entity[:opts] = (mapped_entity[:opts] || {}).merge(matching_fields: self.class.connec_matching_fields) if self.class.connec_matching_fields
|
157
160
|
folded_entity
|
158
161
|
end
|
159
162
|
|
@@ -164,12 +167,11 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
164
167
|
# * full_sync
|
165
168
|
# * $filter (see Connec! documentation)
|
166
169
|
# * $orderby (see Connec! documentation)
|
167
|
-
def get_connec_entities(last_synchronization_date=nil)
|
170
|
+
def get_connec_entities(last_synchronization_date = nil)
|
168
171
|
return [] if @opts[:__skip_connec] || !self.class.can_read_connec?
|
169
172
|
|
170
173
|
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching Connec! #{self.class.connec_entity_name}")
|
171
174
|
|
172
|
-
entities = []
|
173
175
|
query_params = {}
|
174
176
|
query_params[:$orderby] = @opts[:$orderby] if @opts[:$orderby]
|
175
177
|
|
@@ -191,7 +193,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
191
193
|
Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "entity=#{self.class.connec_entity_name}, fetching data with #{query_params.to_query}")
|
192
194
|
uri = "#{self.class.normalized_connec_entity_name}?#{query_params.to_query}"
|
193
195
|
response_hash = fetch_connec(uri, 0)
|
194
|
-
entities = response_hash[
|
196
|
+
entities = response_hash[self.class.normalized_connec_entity_name]
|
195
197
|
entities = [entities] if self.class.singleton?
|
196
198
|
|
197
199
|
# Only the first page if batched_fetch
|
@@ -203,7 +205,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
203
205
|
next_page = response_hash['pagination']['next'].gsub(/^(.*)\/#{self.class.normalized_connec_entity_name}/, self.class.normalized_connec_entity_name)
|
204
206
|
|
205
207
|
response_hash = fetch_connec(next_page, page_number)
|
206
|
-
entities << response_hash[
|
208
|
+
entities << response_hash[self.class.normalized_connec_entity_name]
|
207
209
|
end
|
208
210
|
end
|
209
211
|
|
@@ -220,8 +222,8 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
220
222
|
return unless self.class.can_write_connec?
|
221
223
|
|
222
224
|
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize} to Connec! #{connec_entity_name.pluralize}")
|
223
|
-
|
224
|
-
proc =
|
225
|
+
|
226
|
+
proc = ->(mapped_external_entity_with_idmap) { batch_op('post', mapped_external_entity_with_idmap[:entity], nil, self.class.normalize_connec_entity_name(connec_entity_name)) }
|
225
227
|
batch_calls(mapped_external_entities_with_idmaps, proc, connec_entity_name)
|
226
228
|
end
|
227
229
|
|
@@ -231,7 +233,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
231
233
|
method: method,
|
232
234
|
url: "/api/v2/#{@organization.uid}/#{connec_entity_name}/#{id}", # id should be nil for POST
|
233
235
|
params: {
|
234
|
-
|
236
|
+
connec_entity_name.to_sym => mapped_external_entity
|
235
237
|
}
|
236
238
|
}
|
237
239
|
end
|
@@ -239,14 +241,14 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
239
241
|
# ----------------------------------------------
|
240
242
|
# External methods
|
241
243
|
# ----------------------------------------------
|
242
|
-
def get_external_entities_wrapper(last_synchronization_date=nil)
|
244
|
+
def get_external_entities_wrapper(last_synchronization_date = nil)
|
243
245
|
return [] if @opts[:__skip_external] || !self.class.can_read_external?
|
244
246
|
get_external_entities(last_synchronization_date)
|
245
247
|
end
|
246
|
-
|
247
|
-
def get_external_entities(last_synchronization_date=nil)
|
248
|
+
|
249
|
+
def get_external_entities(last_synchronization_date = nil)
|
248
250
|
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize}")
|
249
|
-
raise
|
251
|
+
raise 'Not implemented'
|
250
252
|
end
|
251
253
|
|
252
254
|
def push_entities_to_external(mapped_connec_entities_with_idmaps)
|
@@ -262,13 +264,12 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
262
264
|
idmap ? {idmap: idmap} : nil
|
263
265
|
}.compact
|
264
266
|
|
265
|
-
|
266
|
-
# Send the external ids to connec if it was a creation
|
267
|
-
proc = lambda{|id| batch_op('put', {id: [Maestrano::Connector::Rails::ConnecHelper.id_hash(id[:idmap].external_id, @organization)]}, id[:idmap].connec_id, self.class.normalize_connec_entity_name(self.class.connec_entity_name)) }
|
268
|
-
batch_calls(ids_to_send_to_connec, proc, self.class.connec_entity_name, true)
|
269
|
-
end
|
270
|
-
end
|
267
|
+
return if ids_to_send_to_connec.empty?
|
271
268
|
|
269
|
+
# Send the external ids to connec if it was a creation
|
270
|
+
proc = ->(id) { batch_op('put', {id: [Maestrano::Connector::Rails::ConnecHelper.id_hash(id[:idmap].external_id, @organization)]}, id[:idmap].connec_id, self.class.normalize_connec_entity_name(self.class.connec_entity_name)) }
|
271
|
+
batch_calls(ids_to_send_to_connec, proc, self.class.connec_entity_name, true)
|
272
|
+
end
|
272
273
|
|
273
274
|
def push_entity_to_external(mapped_connec_entity_with_idmap, external_entity_name)
|
274
275
|
idmap = mapped_connec_entity_with_idmap[:idmap]
|
@@ -277,7 +278,6 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
277
278
|
begin
|
278
279
|
# Create and return id to send to connec!
|
279
280
|
if idmap.external_id.blank?
|
280
|
-
connec_id = idmap.connec_id
|
281
281
|
external_id = create_external_entity(mapped_connec_entity, external_entity_name)
|
282
282
|
idmap.update(external_id: external_id, last_push_to_external: Time.now, message: nil)
|
283
283
|
return idmap
|
@@ -289,13 +289,12 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
289
289
|
|
290
290
|
# Return the id to send it to connec! if the first push of a singleton
|
291
291
|
if self.class.singleton? && idmap.last_push_to_external.nil?
|
292
|
-
connec_id = mapped_connec_entity_with_idmap[:idmap].connec_id
|
293
292
|
idmap.update(last_push_to_external: Time.now, message: nil)
|
294
293
|
return idmap
|
295
294
|
else
|
296
295
|
idmap.update(last_push_to_external: Time.now, message: nil)
|
297
296
|
end
|
298
|
-
|
297
|
+
|
299
298
|
end
|
300
299
|
rescue => e
|
301
300
|
# Store External error
|
@@ -307,12 +306,12 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
307
306
|
|
308
307
|
def create_external_entity(mapped_connec_entity, external_entity_name)
|
309
308
|
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending create #{external_entity_name}: #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
|
310
|
-
raise
|
309
|
+
raise 'Not implemented'
|
311
310
|
end
|
312
311
|
|
313
312
|
def update_external_entity(mapped_connec_entity, external_id, external_entity_name)
|
314
313
|
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending update #{external_entity_name} (id=#{external_id}): #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
|
315
|
-
raise
|
314
|
+
raise 'Not implemented'
|
316
315
|
end
|
317
316
|
|
318
317
|
# ----------------------------------------------
|
@@ -329,7 +328,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
329
328
|
mapped_connec_entities = consolidate_and_map_connec_entities(connec_entities, external_entities, self.class.references, self.class.external_entity_name)
|
330
329
|
mapped_external_entities = consolidate_and_map_external_entities(external_entities, self.class.connec_entity_name)
|
331
330
|
|
332
|
-
|
331
|
+
{connec_entities: mapped_connec_entities, external_entities: mapped_external_entities}
|
333
332
|
end
|
334
333
|
|
335
334
|
def consolidate_and_map_connec_entities(connec_entities, external_entities, references, external_entity_name)
|
@@ -374,7 +373,6 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
374
373
|
# Entity has not been modified since its last push to connec!
|
375
374
|
next nil if !@opts[:full_sync] && not_modified_since_last_push_to_connec?(idmap, entity)
|
376
375
|
|
377
|
-
|
378
376
|
map_external_entity_with_idmap(entity, connec_entity_name, idmap)
|
379
377
|
}.compact
|
380
378
|
end
|
@@ -382,18 +380,18 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
382
380
|
def consolidate_and_map_singleton(connec_entities, external_entities)
|
383
381
|
return {connec_entities: [], external_entities: []} if external_entities.empty? && connec_entities.empty?
|
384
382
|
|
385
|
-
idmap = self.class.find_or_create_idmap(
|
383
|
+
idmap = self.class.find_or_create_idmap(organization_id: @organization.id)
|
386
384
|
# No to_connec, to_external and inactive consideration here as we don't expect those workflow for singleton
|
387
385
|
|
388
|
-
if external_entities.empty?
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
386
|
+
keep_external = if external_entities.empty?
|
387
|
+
false
|
388
|
+
elsif connec_entities.empty?
|
389
|
+
true
|
390
|
+
elsif @opts.key?(:connec_preemption)
|
391
|
+
!@opts[:connec_preemption]
|
392
|
+
else
|
393
|
+
!is_connec_more_recent?(connec_entities.first, external_entities.first)
|
394
|
+
end
|
397
395
|
|
398
396
|
if keep_external
|
399
397
|
idmap.update(external_id: self.class.id_from_external_entity_hash(external_entities.first), name: self.class.object_name_from_external_entity_hash(external_entities.first))
|
@@ -410,9 +408,10 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
410
408
|
# Internal helper methods
|
411
409
|
# ----------------------------------------------
|
412
410
|
private
|
411
|
+
|
413
412
|
# array_with_idmap must be an array of hashes with a key idmap
|
414
413
|
# proc is a lambda to create a batch_op from an element of the array
|
415
|
-
def batch_calls(array_with_idmap, proc, connec_entity_name, id_update_only=false)
|
414
|
+
def batch_calls(array_with_idmap, proc, connec_entity_name, id_update_only = false)
|
416
415
|
request_per_call = @opts[:request_per_batch_call] || 100
|
417
416
|
start = 0
|
418
417
|
while start < array_with_idmap.size
|
@@ -426,7 +425,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
426
425
|
|
427
426
|
# Batch call
|
428
427
|
log_info = id_update_only ? 'with only ids' : ''
|
429
|
-
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending batch request to Connec! #{log_info} for #{self.class.normalize_connec_entity_name(connec_entity_name)}. Batch_request_size: #{batch_request[:ops].size}. Call_number: #{(start/request_per_call) + 1}")
|
428
|
+
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending batch request to Connec! #{log_info} for #{self.class.normalize_connec_entity_name(connec_entity_name)}. Batch_request_size: #{batch_request[:ops].size}. Call_number: #{(start / request_per_call) + 1}")
|
430
429
|
response = @connec_client.batch(batch_request)
|
431
430
|
Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "Received batch response from Connec! for #{self.class.normalize_connec_entity_name(connec_entity_name)}: #{response}")
|
432
431
|
raise "No data received from Connec! when trying to send batch request #{log_info} for #{self.class.connec_entity_name.pluralize}" unless response && !response.body.blank?
|
@@ -435,9 +434,9 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
435
434
|
# Parse batch response
|
436
435
|
response['results'].each_with_index do |result, index|
|
437
436
|
if result['status'] == 200
|
438
|
-
batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find{|id| id['provider'] == 'connec'}['id'], last_push_to_connec: Time.now, message: nil) unless id_update_only # id_update_only only apply for 200 as it's doing PUTs
|
437
|
+
batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find { |id| id['provider'] == 'connec' }['id'], last_push_to_connec: Time.now, message: nil) unless id_update_only # id_update_only only apply for 200 as it's doing PUTs
|
439
438
|
elsif result['status'] == 201
|
440
|
-
batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find{|id| id['provider'] == 'connec'}['id'], last_push_to_connec: Time.now, message: nil)
|
439
|
+
batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find { |id| id['provider'] == 'connec' }['id'], last_push_to_connec: Time.now, message: nil)
|
441
440
|
else
|
442
441
|
Maestrano::Connector::Rails::ConnectorLogger.log('error', @organization, "Error while pushing to Connec!: #{result['body']}")
|
443
442
|
batch_entities[index][:idmap].update(message: result['body'].to_s.truncate(255))
|
@@ -446,10 +445,10 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
446
445
|
start += request_per_call
|
447
446
|
end
|
448
447
|
end
|
449
|
-
|
448
|
+
|
450
449
|
def not_modified_since_last_push_to_connec?(idmap, entity)
|
451
450
|
not_modified = idmap.last_push_to_connec && idmap.last_push_to_connec > self.class.last_update_date_from_external_entity_hash(entity)
|
452
|
-
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Discard #{Maestrano::Connector::Rails::External
|
451
|
+
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Discard #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name} : #{entity}") if not_modified
|
453
452
|
not_modified
|
454
453
|
end
|
455
454
|
|
@@ -459,7 +458,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
459
458
|
not_modified
|
460
459
|
end
|
461
460
|
|
462
|
-
def before_date_filtering_limit?(entity, external=true)
|
461
|
+
def before_date_filtering_limit?(entity, external = true)
|
463
462
|
@organization.date_filtering_limit && @organization.date_filtering_limit > (external ? self.class.creation_date_from_external_entity_hash(entity) : entity['created_at'])
|
464
463
|
end
|
465
464
|
|
@@ -468,25 +467,21 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
468
467
|
end
|
469
468
|
|
470
469
|
def solve_conflict(connec_entity, external_entities, external_entity_name, idmap)
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
keep_connec = @opts[:connec_preemption]
|
475
|
-
else
|
476
|
-
keep_connec = is_connec_more_recent?(connec_entity, external_entity)
|
477
|
-
end
|
470
|
+
external_entity = external_entities.find { |entity| connec_entity['id'] == self.class.id_from_external_entity_hash(entity) }
|
471
|
+
# No conflict
|
472
|
+
return map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap) unless external_entity
|
478
473
|
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
|
483
|
-
else
|
484
|
-
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from external kept")
|
485
|
-
nil
|
486
|
-
end
|
474
|
+
# Conflict
|
475
|
+
# We keep the most recently updated entity
|
476
|
+
keep_connec = @opts.key?(:connec_preemption) ? @opts[:connec_preemption] : is_connec_more_recent?(connec_entity, external_entity)
|
487
477
|
|
488
|
-
|
478
|
+
if keep_connec
|
479
|
+
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External.external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from Connec! kept")
|
480
|
+
external_entities.delete(external_entity)
|
489
481
|
map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
|
482
|
+
else
|
483
|
+
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External.external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from external kept")
|
484
|
+
nil
|
490
485
|
end
|
491
486
|
end
|
492
487
|
|
@@ -504,9 +499,8 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
504
499
|
|
505
500
|
response_hash = JSON.parse(response.body)
|
506
501
|
Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "Received page #{page_number} for entity=#{self.class.connec_entity_name}, response=#{response_hash}")
|
507
|
-
raise "Received unrecognized Connec! data when trying to fetch page #{page_number} of #{self.class.normalized_connec_entity_name}: #{response_hash}" unless response_hash[
|
502
|
+
raise "Received unrecognized Connec! data when trying to fetch page #{page_number} of #{self.class.normalized_connec_entity_name}: #{response_hash}" unless response_hash[self.class.normalized_connec_entity_name]
|
508
503
|
|
509
504
|
response_hash
|
510
505
|
end
|
511
|
-
|
512
|
-
end
|
506
|
+
end
|