maestrano-connector-rails 1.2.1 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +41 -0
- data/.rubocop_todo.yml +180 -0
- data/.ruby-version +1 -1
- data/CODESHIP.md +3 -1
- data/Gemfile +16 -10
- data/Rakefile +12 -12
- data/VERSION +1 -1
- data/app/controllers/maestrano/account/group_users_controller.rb +5 -7
- data/app/controllers/maestrano/account/groups_controller.rb +5 -7
- data/app/controllers/maestrano/application_controller.rb +1 -1
- data/app/controllers/maestrano/auth/saml_controller.rb +5 -10
- data/app/controllers/maestrano/connec_controller.rb +12 -11
- data/app/controllers/maestrano/sessions_controller.rb +1 -1
- data/app/controllers/maestrano/synchronizations_controller.rb +7 -9
- data/app/controllers/version_controller.rb +9 -0
- data/app/helpers/maestrano/connector/rails/session_helper.rb +0 -2
- data/app/jobs/maestrano/connector/rails/all_synchronizations_job.rb +2 -2
- data/app/jobs/maestrano/connector/rails/push_to_connec_job.rb +8 -7
- data/app/jobs/maestrano/connector/rails/push_to_connec_worker.rb +6 -4
- data/app/jobs/maestrano/connector/rails/synchronization_job.rb +10 -9
- data/app/models/maestrano/connector/rails/complex_entity.rb +1 -1
- data/app/models/maestrano/connector/rails/concerns/complex_entity.rb +10 -12
- data/app/models/maestrano/connector/rails/concerns/connec_helper.rb +20 -25
- data/app/models/maestrano/connector/rails/concerns/connector_logger.rb +1 -1
- data/app/models/maestrano/connector/rails/concerns/entity.rb +63 -69
- data/app/models/maestrano/connector/rails/concerns/entity_base.rb +3 -3
- data/app/models/maestrano/connector/rails/concerns/external.rb +2 -2
- data/app/models/maestrano/connector/rails/concerns/sub_entity_base.rb +13 -15
- data/app/models/maestrano/connector/rails/connector_logger.rb +1 -1
- data/app/models/maestrano/connector/rails/entity.rb +1 -1
- data/app/models/maestrano/connector/rails/external.rb +1 -1
- data/app/models/maestrano/connector/rails/id_map.rb +1 -2
- data/app/models/maestrano/connector/rails/organization.rb +12 -14
- data/app/models/maestrano/connector/rails/sub_entity_base.rb +1 -1
- data/app/models/maestrano/connector/rails/synchronization.rb +17 -15
- data/app/models/maestrano/connector/rails/user.rb +1 -2
- data/app/models/maestrano/connector/rails/user_organization_rel.rb +1 -2
- data/config/routes.rb +4 -2
- data/lib/maestrano/connector/rails.rb +17 -2
- data/lib/maestrano_connector_rails.rb +1 -0
- data/maestrano-connector-rails.gemspec +32 -13
- data/release_notes.md +5 -0
- data/spec/controllers/version_controller_spec.rb +17 -0
- data/spec/dummy/config/application.rb +1 -1
- data/spec/jobs/push_to_connec_worker_spec.rb +16 -5
- data/spec/models/synchronization_spec.rb +9 -9
- data/spec/models/user_organization_rel_spec.rb +1 -1
- data/template/maestrano-connector-template.rb +2 -17
- metadata +90 -16
- data/lib/maestrano-connector-rails.rb +0 -1
@@ -2,7 +2,7 @@ class Maestrano::SynchronizationsController < Maestrano::Rails::WebHookControlle
|
|
2
2
|
def show
|
3
3
|
uid = params[:id]
|
4
4
|
organization = Maestrano::Connector::Rails::Organization.find_by_uid(uid)
|
5
|
-
return render json: {
|
5
|
+
return render json: {errors: {message: "Organization not found", code: 404}}, status: :not_found unless organization
|
6
6
|
|
7
7
|
h = {
|
8
8
|
group_id: organization.uid,
|
@@ -12,11 +12,9 @@ class Maestrano::SynchronizationsController < Maestrano::Rails::WebHookControlle
|
|
12
12
|
last_sync = organization.synchronizations.last
|
13
13
|
if last_sync
|
14
14
|
h.merge!(
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
updated_at: last_sync.updated_at
|
19
|
-
}
|
15
|
+
status: last_sync.status,
|
16
|
+
message: last_sync.message,
|
17
|
+
updated_at: last_sync.updated_at
|
20
18
|
)
|
21
19
|
end
|
22
20
|
|
@@ -27,7 +25,7 @@ class Maestrano::SynchronizationsController < Maestrano::Rails::WebHookControlle
|
|
27
25
|
uid = params[:group_id]
|
28
26
|
opts = params[:opts] || {}
|
29
27
|
organization = Maestrano::Connector::Rails::Organization.find_by_uid(uid)
|
30
|
-
return render json: {
|
28
|
+
return render json: {errors: {message: "Organization not found", code: 404}}, status: :not_found unless organization
|
31
29
|
|
32
30
|
Maestrano::Connector::Rails::SynchronizationJob.perform_later(organization, opts.with_indifferent_access)
|
33
31
|
head :created
|
@@ -36,11 +34,11 @@ class Maestrano::SynchronizationsController < Maestrano::Rails::WebHookControlle
|
|
36
34
|
def toggle_sync
|
37
35
|
uid = params[:group_id]
|
38
36
|
organization = Maestrano::Connector::Rails::Organization.find_by_uid(uid)
|
39
|
-
return render json: {
|
37
|
+
return render json: {errors: {message: "Organization not found", code: 404}}, status: :not_found unless organization
|
40
38
|
|
41
39
|
organization.toggle(:sync_enabled)
|
42
40
|
organization.save
|
43
41
|
|
44
42
|
render json: {sync_enabled: organization.sync_enabled}
|
45
43
|
end
|
46
|
-
end
|
44
|
+
end
|
@@ -0,0 +1,9 @@
|
|
1
|
+
class VersionController < ApplicationController
|
2
|
+
def index
|
3
|
+
framework_version = Gem.loaded_specs['maestrano-connector-rails'].version.version
|
4
|
+
respond_to do |format|
|
5
|
+
format.html { render text: "framework_version=#{framework_version}\n" }
|
6
|
+
format.json { render json: {framework_version: framework_version} }
|
7
|
+
end
|
8
|
+
end
|
9
|
+
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
module Maestrano::Connector::Rails
|
2
2
|
module SessionHelper
|
3
|
-
|
4
3
|
def is_admin?(user, organization)
|
5
4
|
organization.member?(user) && session[:"role_#{organization.uid}"] && ['Admin', 'Super Admin'].include?(session[:"role_#{organization.uid}"])
|
6
5
|
end
|
@@ -16,6 +15,5 @@ module Maestrano::Connector::Rails
|
|
16
15
|
def is_admin
|
17
16
|
@is_admin ||= current_user && current_organization && is_admin?(current_user, current_organization)
|
18
17
|
end
|
19
|
-
|
20
18
|
end
|
21
19
|
end
|
@@ -3,11 +3,11 @@ module Maestrano::Connector::Rails
|
|
3
3
|
queue_as :default
|
4
4
|
|
5
5
|
# Trigger synchronization of all active organizations
|
6
|
-
def perform(name=nil, count=nil)
|
6
|
+
def perform(name = nil, count = nil)
|
7
7
|
Maestrano::Connector::Rails::Organization.where.not(oauth_provider: nil, encrypted_oauth_token: nil).each do |o|
|
8
8
|
next unless [true, 1].include?(o.sync_enabled)
|
9
9
|
Maestrano::Connector::Rails::SynchronizationJob.perform_later(o, {})
|
10
10
|
end
|
11
11
|
end
|
12
12
|
end
|
13
|
-
end
|
13
|
+
end
|
@@ -3,7 +3,7 @@ module Maestrano::Connector::Rails
|
|
3
3
|
queue_as :default
|
4
4
|
|
5
5
|
# expected hash: {"external_entity_name1" => [entity1, entity2], "external_entity_name2" => []}
|
6
|
-
def perform(organization, entities_hash, opts={})
|
6
|
+
def perform(organization, entities_hash, opts = {})
|
7
7
|
return unless organization.sync_enabled && organization.oauth_uid
|
8
8
|
|
9
9
|
connec_client = Maestrano::Connector::Rails::ConnecHelper.get_client(organization)
|
@@ -18,11 +18,11 @@ module Maestrano::Connector::Rails
|
|
18
18
|
|
19
19
|
entity_instance.before_sync(last_synchronization_date)
|
20
20
|
# Build expected input for consolidate_and_map_data
|
21
|
-
if entity_instance_hash[:is_complex]
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
21
|
+
mapped_entities = if entity_instance_hash[:is_complex]
|
22
|
+
entity_instance.consolidate_and_map_data(ComplexEntity.build_empty_hash(entity_instance.class.connec_entities_names), ComplexEntity.build_hash_with_entities(entity_instance.class.external_entities_names, external_entity_name, ->(name) { name }, entities))
|
23
|
+
else
|
24
|
+
entity_instance.consolidate_and_map_data([], entities)
|
25
|
+
end
|
26
26
|
entity_instance.push_entities_to_connec(mapped_entities[:external_entities])
|
27
27
|
|
28
28
|
entity_instance.after_sync(last_synchronization_date)
|
@@ -33,6 +33,7 @@ module Maestrano::Connector::Rails
|
|
33
33
|
end
|
34
34
|
|
35
35
|
private
|
36
|
+
|
36
37
|
def find_entity_instance(entity_name, organization, connec_client, external_client, opts)
|
37
38
|
Maestrano::Connector::Rails::External.entities_list.each do |entity_name_from_list|
|
38
39
|
clazz = "Entities::#{entity_name_from_list.singularize.titleize.split.join}".constantize
|
@@ -45,4 +46,4 @@ module Maestrano::Connector::Rails
|
|
45
46
|
nil
|
46
47
|
end
|
47
48
|
end
|
48
|
-
end
|
49
|
+
end
|
@@ -7,13 +7,15 @@ module Maestrano::Connector::Rails
|
|
7
7
|
sidekiq_options unique: :while_executing, unique_args: :unique_args
|
8
8
|
|
9
9
|
def self.unique_args(args)
|
10
|
-
|
10
|
+
organization_id = args[0]
|
11
11
|
entities_hash = args[1]
|
12
|
-
|
12
|
+
|
13
|
+
[organization_id, entities_hash.keys.sort]
|
13
14
|
end
|
14
15
|
|
15
|
-
def perform(
|
16
|
+
def perform(organization_id, entities_hash, opts = {})
|
17
|
+
organization = Organization.find(organization_id)
|
16
18
|
PushToConnecJob.new.perform(organization, entities_hash, opts)
|
17
19
|
end
|
18
20
|
end
|
19
|
-
end
|
21
|
+
end
|
@@ -7,19 +7,19 @@ module Maestrano::Connector::Rails
|
|
7
7
|
# * :only_entities => [person, tasks_list]
|
8
8
|
# * :full_sync => true synchronization is performed without date filtering
|
9
9
|
# * :connec_preemption => true|false : preemption is always|never given to connec in case of conflict (if not set, the most recently updated entity is kept)
|
10
|
-
def perform(organization, opts={})
|
10
|
+
def perform(organization, opts = {})
|
11
11
|
return unless organization.sync_enabled
|
12
12
|
|
13
13
|
# Check if previous synchronization is still running
|
14
14
|
if Synchronization.where(organization_id: organization.id, status: 'RUNNING').where(created_at: (30.minutes.ago..Time.now)).exists?
|
15
|
-
ConnectorLogger.log('info', organization,
|
15
|
+
ConnectorLogger.log('info', organization, 'Synchronization skipped: Previous synchronization is still running')
|
16
16
|
return
|
17
17
|
end
|
18
18
|
|
19
19
|
# Check if recovery mode: last 3 synchronizations have failed
|
20
20
|
if !opts[:forced] && organization.last_three_synchronizations_failed? \
|
21
21
|
&& organization.synchronizations.order(created_at: :desc).limit(1).first.updated_at > 1.day.ago
|
22
|
-
ConnectorLogger.log('info', organization,
|
22
|
+
ConnectorLogger.log('info', organization, 'Synchronization skipped: Recovery mode (three previous synchronizations have failed)')
|
23
23
|
return
|
24
24
|
end
|
25
25
|
|
@@ -37,8 +37,8 @@ module Maestrano::Connector::Rails
|
|
37
37
|
# We do a doube sync: only from external, then only from connec!
|
38
38
|
# We also do batched sync as the first one can be quite huge
|
39
39
|
if last_synchronization.nil?
|
40
|
-
ConnectorLogger.log('info', organization,
|
41
|
-
organization.synchronized_entities.select{|k, v| v}.keys.each do |entity|
|
40
|
+
ConnectorLogger.log('info', organization, 'First synchronization ever. Doing two half syncs to allow smart merging to work its magic.')
|
41
|
+
organization.synchronized_entities.select { |k, v| v }.keys.each do |entity|
|
42
42
|
ConnectorLogger.log('info', organization, "First synchronization ever. Doing half sync from external for #{entity}.")
|
43
43
|
first_sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization_date, opts, true)
|
44
44
|
ConnectorLogger.log('info', organization, "First synchronization ever. Doing half sync from Connec! for #{entity}.")
|
@@ -52,7 +52,7 @@ module Maestrano::Connector::Rails
|
|
52
52
|
sync_entity(entity, organization, connec_client, external_client, last_synchronization_date, opts)
|
53
53
|
end
|
54
54
|
else
|
55
|
-
organization.synchronized_entities.select{|
|
55
|
+
organization.synchronized_entities.select { |_k, v| v }.keys.each do |entity|
|
56
56
|
sync_entity(entity.to_s, organization, connec_client, external_client, last_synchronization_date, opts)
|
57
57
|
end
|
58
58
|
end
|
@@ -78,7 +78,7 @@ module Maestrano::Connector::Rails
|
|
78
78
|
entities_count = limit
|
79
79
|
|
80
80
|
h = {__limit: limit}
|
81
|
-
external ? h
|
81
|
+
external ? h[:__skip_connec] = true : h[:__skip_external] = true
|
82
82
|
entity_instance = instanciate_entity(entity_name, organization, connec_client, external_client, opts.merge(h))
|
83
83
|
|
84
84
|
# IF entities_count > limit
|
@@ -87,7 +87,7 @@ module Maestrano::Connector::Rails
|
|
87
87
|
# No need to fetch it a second Time
|
88
88
|
# ELSIF entities_count < limit
|
89
89
|
# No more entities to fetch
|
90
|
-
while entities_count == limit
|
90
|
+
while entities_count == limit
|
91
91
|
entity_instance.opts_merge!(__skip: skip)
|
92
92
|
entities_count = perform_sync(entity_instance, last_synchronization_date, external)
|
93
93
|
skip += limit
|
@@ -95,6 +95,7 @@ module Maestrano::Connector::Rails
|
|
95
95
|
end
|
96
96
|
|
97
97
|
private
|
98
|
+
|
98
99
|
def instanciate_entity(entity_name, organization, connec_client, external_client, opts)
|
99
100
|
"Entities::#{entity_name.titleize.split.join}".constantize.new(organization, connec_client, external_client, opts.dup)
|
100
101
|
end
|
@@ -112,4 +113,4 @@ module Maestrano::Connector::Rails
|
|
112
113
|
external ? entity_instance.class.count_entities(external_entities) : entity_instance.class.count_entities(connec_entities)
|
113
114
|
end
|
114
115
|
end
|
115
|
-
end
|
116
|
+
end
|
@@ -7,11 +7,11 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
7
7
|
# -------------------------------------------------------------
|
8
8
|
module ClassMethods
|
9
9
|
def connec_entities_names
|
10
|
-
raise
|
10
|
+
raise 'Not implemented'
|
11
11
|
end
|
12
12
|
|
13
13
|
def external_entities_names
|
14
|
-
raise
|
14
|
+
raise 'Not implemented'
|
15
15
|
end
|
16
16
|
|
17
17
|
def count_entities(entities)
|
@@ -33,7 +33,7 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
33
33
|
# }
|
34
34
|
# }
|
35
35
|
def connec_model_to_external_model(connec_hash_of_entities)
|
36
|
-
raise
|
36
|
+
raise 'Not implemented'
|
37
37
|
end
|
38
38
|
|
39
39
|
# input : {
|
@@ -50,13 +50,13 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
50
50
|
# }
|
51
51
|
# }
|
52
52
|
def external_model_to_connec_model(external_hash_of_entities)
|
53
|
-
raise
|
53
|
+
raise 'Not implemented'
|
54
54
|
end
|
55
55
|
|
56
56
|
# -------------------------------------------------------------
|
57
57
|
# Entity equivalent methods
|
58
58
|
# -------------------------------------------------------------
|
59
|
-
def get_connec_entities(last_synchronization_date=nil)
|
59
|
+
def get_connec_entities(last_synchronization_date = nil)
|
60
60
|
entities = ActiveSupport::HashWithIndifferentAccess.new
|
61
61
|
|
62
62
|
self.class.connec_entities_names.each do |connec_entity_name|
|
@@ -66,7 +66,7 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
66
66
|
entities
|
67
67
|
end
|
68
68
|
|
69
|
-
def get_external_entities_wrapper(last_synchronization_date=nil)
|
69
|
+
def get_external_entities_wrapper(last_synchronization_date = nil)
|
70
70
|
entities = ActiveSupport::HashWithIndifferentAccess.new
|
71
71
|
|
72
72
|
self.class.external_entities_names.each do |external_entity_name|
|
@@ -83,13 +83,12 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
83
83
|
mapped_connec_entities = consolidate_and_map_connec_entities(modeled_connec_entities, modeled_external_entities)
|
84
84
|
mapped_external_entities = consolidate_and_map_external_entities(modeled_external_entities)
|
85
85
|
|
86
|
-
|
86
|
+
{connec_entities: mapped_connec_entities, external_entities: mapped_external_entities}
|
87
87
|
end
|
88
88
|
|
89
89
|
def consolidate_and_map_connec_entities(modeled_connec_entities, modeled_external_entities)
|
90
90
|
modeled_connec_entities.each do |connec_entity_name, entities_in_external_model|
|
91
91
|
entities_in_external_model.each do |external_entity_name, entities|
|
92
|
-
|
93
92
|
sub_entity_instance = instantiate_sub_entity_instance(connec_entity_name)
|
94
93
|
equivalent_external_entities = (modeled_external_entities[external_entity_name] && modeled_external_entities[external_entity_name][connec_entity_name]) || []
|
95
94
|
|
@@ -128,7 +127,6 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
128
127
|
end
|
129
128
|
end
|
130
129
|
|
131
|
-
|
132
130
|
def push_entities_to_external(mapped_connec_entities_with_idmaps)
|
133
131
|
mapped_connec_entities_with_idmaps.each do |connec_entity_name, entities_in_external_model|
|
134
132
|
sub_entity_instance = instantiate_sub_entity_instance(connec_entity_name)
|
@@ -148,13 +146,13 @@ module Maestrano::Connector::Rails::Concerns::ComplexEntity
|
|
148
146
|
module ClassMethods
|
149
147
|
# output : {entities_names[0] => [], entities_names[1] => []}
|
150
148
|
def build_empty_hash(entities_names)
|
151
|
-
Hash[
|
149
|
+
Hash[*entities_names.collect { |name| [name, []] }.flatten(1)]
|
152
150
|
end
|
153
151
|
|
154
152
|
# output: {entities_name[0] => [], entities_name[1] => entities}
|
155
153
|
# with proc.call(entities_name[1] == entity_name)
|
156
154
|
def build_hash_with_entities(entities_name, entity_name, proc, entities)
|
157
|
-
Hash[
|
155
|
+
Hash[*entities_name.collect { |name| proc.call(name) == entity_name ? [name, entities] : [name, []] }.flatten(1)]
|
158
156
|
end
|
159
157
|
end
|
160
|
-
end
|
158
|
+
end
|
@@ -25,7 +25,7 @@ module Maestrano::Connector::Rails::Concerns::ConnecHelper
|
|
25
25
|
end
|
26
26
|
@@connec_version
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
# Replace the ids arrays by the external id
|
30
30
|
# If a reference has no id for this oauth_provider and oauth_uid but has one for connec returns nil
|
31
31
|
def unfold_references(connec_entity, references, organization)
|
@@ -33,13 +33,9 @@ module Maestrano::Connector::Rails::Concerns::ConnecHelper
|
|
33
33
|
not_nil = true
|
34
34
|
|
35
35
|
# Id
|
36
|
-
|
37
|
-
unfolded_connec_entity[:__connec_id] = unfolded_connec_entity['id'].find{|id| id['provider'] == 'connec'}['id']
|
38
|
-
|
39
|
-
unfolded_connec_entity['id'] = id['id']
|
40
|
-
else
|
41
|
-
unfolded_connec_entity['id'] = nil
|
42
|
-
end
|
36
|
+
id_hash = unfolded_connec_entity['id'].find { |id| id['provider'] == organization.oauth_provider && id['realm'] == organization.oauth_uid }
|
37
|
+
unfolded_connec_entity[:__connec_id] = unfolded_connec_entity['id'].find { |id| id['provider'] == 'connec' }['id']
|
38
|
+
unfolded_connec_entity['id'] = id_hash ? id_hash['id'] : nil
|
43
39
|
|
44
40
|
# Other refs
|
45
41
|
references.each do |reference|
|
@@ -67,20 +63,19 @@ module Maestrano::Connector::Rails::Concerns::ConnecHelper
|
|
67
63
|
def fold_references_helper(entity, array_of_refs, organization)
|
68
64
|
ref = array_of_refs.shift
|
69
65
|
field = entity[ref]
|
66
|
+
return if field.blank?
|
70
67
|
|
71
68
|
# Follow embedment path, remplace if it's not an array or a hash
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
fold_references_helper(f, array_of_refs.dup, organization)
|
77
|
-
end
|
78
|
-
when HashWithIndifferentAccess
|
79
|
-
fold_references_helper(entity[ref], array_of_refs, organization)
|
80
|
-
else
|
81
|
-
id = field
|
82
|
-
entity[ref] = [id_hash(id, organization)]
|
69
|
+
case field
|
70
|
+
when Array
|
71
|
+
field.each do |f|
|
72
|
+
fold_references_helper(f, array_of_refs.dup, organization)
|
83
73
|
end
|
74
|
+
when HashWithIndifferentAccess
|
75
|
+
fold_references_helper(entity[ref], array_of_refs, organization)
|
76
|
+
else
|
77
|
+
id = field
|
78
|
+
entity[ref] = [id_hash(id, organization)]
|
84
79
|
end
|
85
80
|
end
|
86
81
|
|
@@ -89,10 +84,11 @@ module Maestrano::Connector::Rails::Concerns::ConnecHelper
|
|
89
84
|
field = entity[ref]
|
90
85
|
|
91
86
|
# Unfold the id
|
92
|
-
if array_of_refs.empty?
|
93
|
-
|
94
|
-
|
95
|
-
|
87
|
+
if array_of_refs.empty? && field
|
88
|
+
id_hash = field.find { |id| id[:provider] == organization.oauth_provider && id[:realm] == organization.oauth_uid }
|
89
|
+
if id_hash
|
90
|
+
entity[ref] = id_hash['id']
|
91
|
+
elsif field.find { |id| id[:provider] == 'connec' } # Should always be true as ids will always contain a connec id
|
96
92
|
# We may enqueue a fetch on the endpoint of the missing association, followed by a re-fetch on this one.
|
97
93
|
# However it's expected to be an edge case, so for now we rely on the fact that the webhooks should be relativly in order.
|
98
94
|
# Worst case it'll be done on following sync
|
@@ -114,6 +110,5 @@ module Maestrano::Connector::Rails::Concerns::ConnecHelper
|
|
114
110
|
end
|
115
111
|
true
|
116
112
|
end
|
117
|
-
|
118
113
|
end
|
119
|
-
end
|
114
|
+
end
|
@@ -19,9 +19,11 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
19
19
|
def find_or_create_idmap(organization_and_id)
|
20
20
|
Maestrano::Connector::Rails::IdMap.find_or_create_by(names_hash.merge(organization_and_id))
|
21
21
|
end
|
22
|
+
|
22
23
|
def find_idmap(organization_and_id)
|
23
24
|
Maestrano::Connector::Rails::IdMap.find_by(names_hash.merge(organization_and_id))
|
24
25
|
end
|
26
|
+
|
25
27
|
def create_idmap(organization_and_id)
|
26
28
|
Maestrano::Connector::Rails::IdMap.create(names_hash.merge(organization_and_id))
|
27
29
|
end
|
@@ -45,25 +47,25 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
45
47
|
# External methods
|
46
48
|
# ----------------------------------------------
|
47
49
|
def id_from_external_entity_hash(entity)
|
48
|
-
raise
|
50
|
+
raise 'Not implemented'
|
49
51
|
end
|
50
52
|
|
51
53
|
def last_update_date_from_external_entity_hash(entity)
|
52
|
-
raise
|
54
|
+
raise 'Not implemented'
|
53
55
|
end
|
54
56
|
|
55
57
|
def creation_date_from_external_entity_hash(entity)
|
56
|
-
raise
|
58
|
+
raise 'Not implemented'
|
57
59
|
end
|
58
60
|
|
59
61
|
# Return a string representing the object from a connec! entity hash
|
60
62
|
def object_name_from_connec_entity_hash(entity)
|
61
|
-
raise
|
63
|
+
raise 'Not implemented'
|
62
64
|
end
|
63
65
|
|
64
66
|
# Return a string representing the object from an external entity hash
|
65
67
|
def object_name_from_external_entity_hash(entity)
|
66
|
-
raise
|
68
|
+
raise 'Not implemented'
|
67
69
|
end
|
68
70
|
|
69
71
|
# Returns a boolean
|
@@ -71,6 +73,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
71
73
|
def inactive_from_external_entity_hash?(entity)
|
72
74
|
false
|
73
75
|
end
|
76
|
+
|
74
77
|
# ----------------------------------------------
|
75
78
|
# Entity specific methods
|
76
79
|
# Those methods need to be define in each entity
|
@@ -82,17 +85,17 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
82
85
|
|
83
86
|
# Entity name in Connec!
|
84
87
|
def connec_entity_name
|
85
|
-
raise
|
88
|
+
raise 'Not implemented'
|
86
89
|
end
|
87
90
|
|
88
91
|
# Entity name in external system
|
89
92
|
def external_entity_name
|
90
|
-
raise
|
93
|
+
raise 'Not implemented'
|
91
94
|
end
|
92
95
|
|
93
96
|
# Entity Mapper Class
|
94
97
|
def mapper_class
|
95
|
-
raise
|
98
|
+
raise 'Not implemented'
|
96
99
|
end
|
97
100
|
|
98
101
|
# An array of connec fields that are references
|
@@ -153,7 +156,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
153
156
|
def map_to_connec(entity)
|
154
157
|
mapped_entity = self.class.mapper_class.denormalize(entity).merge(id: self.class.id_from_external_entity_hash(entity))
|
155
158
|
folded_entity = Maestrano::Connector::Rails::ConnecHelper.fold_references(mapped_entity, self.class.references, @organization)
|
156
|
-
folded_entity
|
159
|
+
folded_entity[:opts] = (mapped_entity[:opts] || {}).merge(matching_fields: self.class.connec_matching_fields) if self.class.connec_matching_fields
|
157
160
|
folded_entity
|
158
161
|
end
|
159
162
|
|
@@ -164,12 +167,11 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
164
167
|
# * full_sync
|
165
168
|
# * $filter (see Connec! documentation)
|
166
169
|
# * $orderby (see Connec! documentation)
|
167
|
-
def get_connec_entities(last_synchronization_date=nil)
|
170
|
+
def get_connec_entities(last_synchronization_date = nil)
|
168
171
|
return [] if @opts[:__skip_connec] || !self.class.can_read_connec?
|
169
172
|
|
170
173
|
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching Connec! #{self.class.connec_entity_name}")
|
171
174
|
|
172
|
-
entities = []
|
173
175
|
query_params = {}
|
174
176
|
query_params[:$orderby] = @opts[:$orderby] if @opts[:$orderby]
|
175
177
|
|
@@ -191,7 +193,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
191
193
|
Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "entity=#{self.class.connec_entity_name}, fetching data with #{query_params.to_query}")
|
192
194
|
uri = "#{self.class.normalized_connec_entity_name}?#{query_params.to_query}"
|
193
195
|
response_hash = fetch_connec(uri, 0)
|
194
|
-
entities = response_hash[
|
196
|
+
entities = response_hash[self.class.normalized_connec_entity_name]
|
195
197
|
entities = [entities] if self.class.singleton?
|
196
198
|
|
197
199
|
# Only the first page if batched_fetch
|
@@ -203,7 +205,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
203
205
|
next_page = response_hash['pagination']['next'].gsub(/^(.*)\/#{self.class.normalized_connec_entity_name}/, self.class.normalized_connec_entity_name)
|
204
206
|
|
205
207
|
response_hash = fetch_connec(next_page, page_number)
|
206
|
-
entities << response_hash[
|
208
|
+
entities << response_hash[self.class.normalized_connec_entity_name]
|
207
209
|
end
|
208
210
|
end
|
209
211
|
|
@@ -220,8 +222,8 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
220
222
|
return unless self.class.can_write_connec?
|
221
223
|
|
222
224
|
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize} to Connec! #{connec_entity_name.pluralize}")
|
223
|
-
|
224
|
-
proc =
|
225
|
+
|
226
|
+
proc = ->(mapped_external_entity_with_idmap) { batch_op('post', mapped_external_entity_with_idmap[:entity], nil, self.class.normalize_connec_entity_name(connec_entity_name)) }
|
225
227
|
batch_calls(mapped_external_entities_with_idmaps, proc, connec_entity_name)
|
226
228
|
end
|
227
229
|
|
@@ -231,7 +233,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
231
233
|
method: method,
|
232
234
|
url: "/api/v2/#{@organization.uid}/#{connec_entity_name}/#{id}", # id should be nil for POST
|
233
235
|
params: {
|
234
|
-
|
236
|
+
connec_entity_name.to_sym => mapped_external_entity
|
235
237
|
}
|
236
238
|
}
|
237
239
|
end
|
@@ -239,14 +241,14 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
239
241
|
# ----------------------------------------------
|
240
242
|
# External methods
|
241
243
|
# ----------------------------------------------
|
242
|
-
def get_external_entities_wrapper(last_synchronization_date=nil)
|
244
|
+
def get_external_entities_wrapper(last_synchronization_date = nil)
|
243
245
|
return [] if @opts[:__skip_external] || !self.class.can_read_external?
|
244
246
|
get_external_entities(last_synchronization_date)
|
245
247
|
end
|
246
|
-
|
247
|
-
def get_external_entities(last_synchronization_date=nil)
|
248
|
+
|
249
|
+
def get_external_entities(last_synchronization_date = nil)
|
248
250
|
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize}")
|
249
|
-
raise
|
251
|
+
raise 'Not implemented'
|
250
252
|
end
|
251
253
|
|
252
254
|
def push_entities_to_external(mapped_connec_entities_with_idmaps)
|
@@ -262,13 +264,12 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
262
264
|
idmap ? {idmap: idmap} : nil
|
263
265
|
}.compact
|
264
266
|
|
265
|
-
|
266
|
-
# Send the external ids to connec if it was a creation
|
267
|
-
proc = lambda{|id| batch_op('put', {id: [Maestrano::Connector::Rails::ConnecHelper.id_hash(id[:idmap].external_id, @organization)]}, id[:idmap].connec_id, self.class.normalize_connec_entity_name(self.class.connec_entity_name)) }
|
268
|
-
batch_calls(ids_to_send_to_connec, proc, self.class.connec_entity_name, true)
|
269
|
-
end
|
270
|
-
end
|
267
|
+
return if ids_to_send_to_connec.empty?
|
271
268
|
|
269
|
+
# Send the external ids to connec if it was a creation
|
270
|
+
proc = ->(id) { batch_op('put', {id: [Maestrano::Connector::Rails::ConnecHelper.id_hash(id[:idmap].external_id, @organization)]}, id[:idmap].connec_id, self.class.normalize_connec_entity_name(self.class.connec_entity_name)) }
|
271
|
+
batch_calls(ids_to_send_to_connec, proc, self.class.connec_entity_name, true)
|
272
|
+
end
|
272
273
|
|
273
274
|
def push_entity_to_external(mapped_connec_entity_with_idmap, external_entity_name)
|
274
275
|
idmap = mapped_connec_entity_with_idmap[:idmap]
|
@@ -277,7 +278,6 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
277
278
|
begin
|
278
279
|
# Create and return id to send to connec!
|
279
280
|
if idmap.external_id.blank?
|
280
|
-
connec_id = idmap.connec_id
|
281
281
|
external_id = create_external_entity(mapped_connec_entity, external_entity_name)
|
282
282
|
idmap.update(external_id: external_id, last_push_to_external: Time.now, message: nil)
|
283
283
|
return idmap
|
@@ -289,13 +289,12 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
289
289
|
|
290
290
|
# Return the id to send it to connec! if the first push of a singleton
|
291
291
|
if self.class.singleton? && idmap.last_push_to_external.nil?
|
292
|
-
connec_id = mapped_connec_entity_with_idmap[:idmap].connec_id
|
293
292
|
idmap.update(last_push_to_external: Time.now, message: nil)
|
294
293
|
return idmap
|
295
294
|
else
|
296
295
|
idmap.update(last_push_to_external: Time.now, message: nil)
|
297
296
|
end
|
298
|
-
|
297
|
+
|
299
298
|
end
|
300
299
|
rescue => e
|
301
300
|
# Store External error
|
@@ -307,12 +306,12 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
307
306
|
|
308
307
|
def create_external_entity(mapped_connec_entity, external_entity_name)
|
309
308
|
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending create #{external_entity_name}: #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
|
310
|
-
raise
|
309
|
+
raise 'Not implemented'
|
311
310
|
end
|
312
311
|
|
313
312
|
def update_external_entity(mapped_connec_entity, external_id, external_entity_name)
|
314
313
|
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending update #{external_entity_name} (id=#{external_id}): #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
|
315
|
-
raise
|
314
|
+
raise 'Not implemented'
|
316
315
|
end
|
317
316
|
|
318
317
|
# ----------------------------------------------
|
@@ -329,7 +328,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
329
328
|
mapped_connec_entities = consolidate_and_map_connec_entities(connec_entities, external_entities, self.class.references, self.class.external_entity_name)
|
330
329
|
mapped_external_entities = consolidate_and_map_external_entities(external_entities, self.class.connec_entity_name)
|
331
330
|
|
332
|
-
|
331
|
+
{connec_entities: mapped_connec_entities, external_entities: mapped_external_entities}
|
333
332
|
end
|
334
333
|
|
335
334
|
def consolidate_and_map_connec_entities(connec_entities, external_entities, references, external_entity_name)
|
@@ -374,7 +373,6 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
374
373
|
# Entity has not been modified since its last push to connec!
|
375
374
|
next nil if !@opts[:full_sync] && not_modified_since_last_push_to_connec?(idmap, entity)
|
376
375
|
|
377
|
-
|
378
376
|
map_external_entity_with_idmap(entity, connec_entity_name, idmap)
|
379
377
|
}.compact
|
380
378
|
end
|
@@ -382,18 +380,18 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
382
380
|
def consolidate_and_map_singleton(connec_entities, external_entities)
|
383
381
|
return {connec_entities: [], external_entities: []} if external_entities.empty? && connec_entities.empty?
|
384
382
|
|
385
|
-
idmap = self.class.find_or_create_idmap(
|
383
|
+
idmap = self.class.find_or_create_idmap(organization_id: @organization.id)
|
386
384
|
# No to_connec, to_external and inactive consideration here as we don't expect those workflow for singleton
|
387
385
|
|
388
|
-
if external_entities.empty?
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
386
|
+
keep_external = if external_entities.empty?
|
387
|
+
false
|
388
|
+
elsif connec_entities.empty?
|
389
|
+
true
|
390
|
+
elsif @opts.key?(:connec_preemption)
|
391
|
+
!@opts[:connec_preemption]
|
392
|
+
else
|
393
|
+
!is_connec_more_recent?(connec_entities.first, external_entities.first)
|
394
|
+
end
|
397
395
|
|
398
396
|
if keep_external
|
399
397
|
idmap.update(external_id: self.class.id_from_external_entity_hash(external_entities.first), name: self.class.object_name_from_external_entity_hash(external_entities.first))
|
@@ -410,9 +408,10 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
410
408
|
# Internal helper methods
|
411
409
|
# ----------------------------------------------
|
412
410
|
private
|
411
|
+
|
413
412
|
# array_with_idmap must be an array of hashes with a key idmap
|
414
413
|
# proc is a lambda to create a batch_op from an element of the array
|
415
|
-
def batch_calls(array_with_idmap, proc, connec_entity_name, id_update_only=false)
|
414
|
+
def batch_calls(array_with_idmap, proc, connec_entity_name, id_update_only = false)
|
416
415
|
request_per_call = @opts[:request_per_batch_call] || 100
|
417
416
|
start = 0
|
418
417
|
while start < array_with_idmap.size
|
@@ -426,7 +425,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
426
425
|
|
427
426
|
# Batch call
|
428
427
|
log_info = id_update_only ? 'with only ids' : ''
|
429
|
-
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending batch request to Connec! #{log_info} for #{self.class.normalize_connec_entity_name(connec_entity_name)}. Batch_request_size: #{batch_request[:ops].size}. Call_number: #{(start/request_per_call) + 1}")
|
428
|
+
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending batch request to Connec! #{log_info} for #{self.class.normalize_connec_entity_name(connec_entity_name)}. Batch_request_size: #{batch_request[:ops].size}. Call_number: #{(start / request_per_call) + 1}")
|
430
429
|
response = @connec_client.batch(batch_request)
|
431
430
|
Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "Received batch response from Connec! for #{self.class.normalize_connec_entity_name(connec_entity_name)}: #{response}")
|
432
431
|
raise "No data received from Connec! when trying to send batch request #{log_info} for #{self.class.connec_entity_name.pluralize}" unless response && !response.body.blank?
|
@@ -435,9 +434,9 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
435
434
|
# Parse batch response
|
436
435
|
response['results'].each_with_index do |result, index|
|
437
436
|
if result['status'] == 200
|
438
|
-
batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find{|id| id['provider'] == 'connec'}['id'], last_push_to_connec: Time.now, message: nil) unless id_update_only # id_update_only only apply for 200 as it's doing PUTs
|
437
|
+
batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find { |id| id['provider'] == 'connec' }['id'], last_push_to_connec: Time.now, message: nil) unless id_update_only # id_update_only only apply for 200 as it's doing PUTs
|
439
438
|
elsif result['status'] == 201
|
440
|
-
batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find{|id| id['provider'] == 'connec'}['id'], last_push_to_connec: Time.now, message: nil)
|
439
|
+
batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find { |id| id['provider'] == 'connec' }['id'], last_push_to_connec: Time.now, message: nil)
|
441
440
|
else
|
442
441
|
Maestrano::Connector::Rails::ConnectorLogger.log('error', @organization, "Error while pushing to Connec!: #{result['body']}")
|
443
442
|
batch_entities[index][:idmap].update(message: result['body'].to_s.truncate(255))
|
@@ -446,10 +445,10 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
446
445
|
start += request_per_call
|
447
446
|
end
|
448
447
|
end
|
449
|
-
|
448
|
+
|
450
449
|
def not_modified_since_last_push_to_connec?(idmap, entity)
|
451
450
|
not_modified = idmap.last_push_to_connec && idmap.last_push_to_connec > self.class.last_update_date_from_external_entity_hash(entity)
|
452
|
-
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Discard #{Maestrano::Connector::Rails::External
|
451
|
+
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Discard #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name} : #{entity}") if not_modified
|
453
452
|
not_modified
|
454
453
|
end
|
455
454
|
|
@@ -459,7 +458,7 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
459
458
|
not_modified
|
460
459
|
end
|
461
460
|
|
462
|
-
def before_date_filtering_limit?(entity, external=true)
|
461
|
+
def before_date_filtering_limit?(entity, external = true)
|
463
462
|
@organization.date_filtering_limit && @organization.date_filtering_limit > (external ? self.class.creation_date_from_external_entity_hash(entity) : entity['created_at'])
|
464
463
|
end
|
465
464
|
|
@@ -468,25 +467,21 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
468
467
|
end
|
469
468
|
|
470
469
|
def solve_conflict(connec_entity, external_entities, external_entity_name, idmap)
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
keep_connec = @opts[:connec_preemption]
|
475
|
-
else
|
476
|
-
keep_connec = is_connec_more_recent?(connec_entity, external_entity)
|
477
|
-
end
|
470
|
+
external_entity = external_entities.find { |entity| connec_entity['id'] == self.class.id_from_external_entity_hash(entity) }
|
471
|
+
# No conflict
|
472
|
+
return map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap) unless external_entity
|
478
473
|
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
|
483
|
-
else
|
484
|
-
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from external kept")
|
485
|
-
nil
|
486
|
-
end
|
474
|
+
# Conflict
|
475
|
+
# We keep the most recently updated entity
|
476
|
+
keep_connec = @opts.key?(:connec_preemption) ? @opts[:connec_preemption] : is_connec_more_recent?(connec_entity, external_entity)
|
487
477
|
|
488
|
-
|
478
|
+
if keep_connec
|
479
|
+
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External.external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from Connec! kept")
|
480
|
+
external_entities.delete(external_entity)
|
489
481
|
map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
|
482
|
+
else
|
483
|
+
Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External.external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from external kept")
|
484
|
+
nil
|
490
485
|
end
|
491
486
|
end
|
492
487
|
|
@@ -504,9 +499,8 @@ module Maestrano::Connector::Rails::Concerns::Entity
|
|
504
499
|
|
505
500
|
response_hash = JSON.parse(response.body)
|
506
501
|
Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "Received page #{page_number} for entity=#{self.class.connec_entity_name}, response=#{response_hash}")
|
507
|
-
raise "Received unrecognized Connec! data when trying to fetch page #{page_number} of #{self.class.normalized_connec_entity_name}: #{response_hash}" unless response_hash[
|
502
|
+
raise "Received unrecognized Connec! data when trying to fetch page #{page_number} of #{self.class.normalized_connec_entity_name}: #{response_hash}" unless response_hash[self.class.normalized_connec_entity_name]
|
508
503
|
|
509
504
|
response_hash
|
510
505
|
end
|
511
|
-
|
512
|
-
end
|
506
|
+
end
|