eco-helpers 0.6.17 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +19 -0
- data/.yardopts +2 -2
- data/Gemfile +6 -0
- data/Rakefile +27 -0
- data/eco-helpers.gemspec +9 -6
- data/lib/eco/api.rb +2 -1
- data/lib/eco/api/common/people.rb +1 -1
- data/lib/eco/api/common/people/base_parser.rb +31 -1
- data/lib/eco/api/common/people/default_parsers.rb +5 -1
- data/lib/eco/api/common/people/default_parsers/csv_parser.rb +37 -0
- data/lib/eco/api/common/people/default_parsers/numeric_parser.rb +0 -1
- data/lib/eco/api/common/people/entries.rb +14 -18
- data/lib/eco/api/common/people/entry_factory.rb +97 -9
- data/lib/eco/api/common/people/person_entry.rb +147 -206
- data/lib/eco/api/common/people/person_entry_attribute_mapper.rb +212 -0
- data/lib/eco/api/common/people/person_factory.rb +10 -12
- data/lib/eco/api/common/people/person_parser.rb +97 -37
- data/lib/eco/api/common/session/base_session.rb +1 -2
- data/lib/eco/api/common/session/file_manager.rb +1 -1
- data/lib/eco/api/organization.rb +2 -1
- data/lib/eco/api/organization/people.rb +54 -22
- data/lib/eco/api/organization/person_schemas.rb +54 -0
- data/lib/eco/api/organization/policy_groups.rb +5 -9
- data/lib/eco/api/organization/{presets.rb → presets_factory.rb} +1 -1
- data/lib/eco/api/policies.rb +10 -0
- data/lib/eco/api/policies/base_policy.rb +14 -0
- data/lib/eco/api/policies/policy.rb +20 -0
- data/lib/eco/api/policies/used_policies.rb +37 -0
- data/lib/eco/api/session.rb +36 -34
- data/lib/eco/api/session/batch.rb +94 -44
- data/lib/eco/api/session/batch_job.rb +108 -48
- data/lib/eco/api/session/batch_jobs.rb +4 -5
- data/lib/eco/api/session/batch_status.rb +70 -11
- data/lib/eco/api/session/config.rb +22 -5
- data/lib/eco/api/session/config/files.rb +10 -1
- data/lib/eco/api/session/config/people.rb +18 -5
- data/lib/eco/api/session/config/policies.rb +29 -0
- data/lib/eco/api/session/config/use_cases.rb +3 -7
- data/lib/eco/api/session/job_groups.rb +9 -10
- data/lib/eco/api/usecases.rb +2 -1
- data/lib/eco/api/usecases/base_case.rb +7 -2
- data/lib/eco/api/usecases/default_cases/change_email_case.rb +4 -2
- data/lib/eco/api/usecases/default_cases/create_case.rb +2 -1
- data/lib/eco/api/usecases/default_cases/create_details_case.rb +3 -1
- data/lib/eco/api/usecases/default_cases/create_details_with_supervisor_case.rb +4 -2
- data/lib/eco/api/usecases/default_cases/hris_case.rb +20 -13
- data/lib/eco/api/usecases/default_cases/new_email_case.rb +3 -1
- data/lib/eco/api/usecases/default_cases/new_id_case.rb +4 -2
- data/lib/eco/api/usecases/default_cases/recover_db_case.rb +9 -5
- data/lib/eco/api/usecases/default_cases/remove_account_case.rb +4 -2
- data/lib/eco/api/usecases/default_cases/set_supervisor_case.rb +4 -2
- data/lib/eco/api/usecases/default_cases/to_csv_case.rb +2 -2
- data/lib/eco/api/usecases/default_cases/to_csv_detailed_case.rb +2 -2
- data/lib/eco/api/usecases/default_cases/update_case.rb +16 -2
- data/lib/eco/api/usecases/default_cases/update_details_case.rb +3 -1
- data/lib/eco/api/usecases/default_cases/upsert_case.rb +25 -3
- data/lib/eco/api/usecases/use_case.rb +23 -140
- data/lib/eco/api/usecases/use_case_chain.rb +95 -0
- data/lib/eco/api/usecases/use_case_io.rb +117 -0
- data/lib/eco/api/usecases/use_group.rb +25 -5
- data/lib/eco/common/base_cli_backup.rb +1 -0
- data/lib/eco/language/models.rb +1 -1
- data/lib/eco/language/models/collection.rb +42 -31
- data/lib/eco/language/models/parser_serializer.rb +68 -0
- data/lib/eco/version.rb +1 -1
- metadata +93 -38
- data/lib/eco/api/common/people/types.rb +0 -47
- data/lib/eco/api/usecases/case_data.rb +0 -13
- data/lib/eco/language/models/attribute_parser.rb +0 -38
- data/lib/eco/lexic/dictionary.rb +0 -33
- data/lib/eco/lexic/dictionary/dictionary.txt +0 -355484
- data/lib/eco/lexic/dictionary/tags.json +0 -38
@@ -1,27 +1,78 @@
|
|
1
1
|
module Eco
|
2
2
|
module API
|
3
3
|
class Session
|
4
|
-
# important! the handler should probably only create logs and save the update with same timestamp
|
5
4
|
class Batch < Common::Session::BaseSession
|
6
5
|
|
7
6
|
DEFAULT_BATCH_BLOCK = 100
|
8
|
-
VALID_METHODS =
|
7
|
+
VALID_METHODS = [:get, :create, :update, :upsert, :delete]
|
9
8
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
9
|
+
class << self
|
10
|
+
# @return [Boolean] `true` if the method is supported, `false` otherwise.
|
11
|
+
def valid_method?(value)
|
12
|
+
VALID_METHODS.include?(value)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# Gets the _people_ of the organization according `params`.
|
17
|
+
# If `people` is not `nil`, scopes to only the people specified.
|
18
|
+
# @note
|
19
|
+
# - If `people` is given keys `page:` and `q` of `params:`.
|
20
|
+
# @param people [Nil, People, Enumerable<Person>, Enumerable<Hash>] target _People_ to launch the batch against.
|
21
|
+
# @param params [Hash] api request options.
|
22
|
+
# @option params [String] :page the page number `page` based on `:per_page`.
|
23
|
+
# @option params [String] :per_page the number of people included per each batch api request.
|
24
|
+
# @option params [String] :q some text to search. Omit this parameter to target all the people.
|
25
|
+
# @return [Array<People>] all the people based on `params`
|
26
|
+
def get_people(people = nil, params: {})
|
27
|
+
return launch(people, method: :get, params: params) if people.is_a?(Enumerable)
|
28
|
+
return get(params: params)
|
15
29
|
end
|
16
30
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
31
|
+
# launches a batch of `method` type using `people` and the specified `params`
|
32
|
+
# @raise Exception
|
33
|
+
# - if `people` is `nil` or is not an `Enumerable`.
|
34
|
+
# - if there's no `api` connection linked to the current `Batch`.
|
35
|
+
# @param people [People, Enumerable<Person>, Enumerable<Hash>] target _People_ to launch the batch against.
|
36
|
+
# @param method [Symbol] the method to launch the batch api request with.
|
37
|
+
# @param params [Hash] api request options.
|
38
|
+
# @option params [String] :per_page the number of people included per each batch api request.
|
39
|
+
# @return [BatchStatus] the `status` of this batch launch.
|
40
|
+
def launch(people, method:, params: {} , silent: false)
|
41
|
+
batch_from(people, method: method, params: params, silent: silent)
|
21
42
|
end
|
22
43
|
|
23
|
-
def
|
24
|
-
|
44
|
+
def search(data, params: {})
|
45
|
+
params = {per_page: DEFAULT_BATCH_BLOCK}.merge(params)
|
46
|
+
|
47
|
+
launch(data, method: :get, params: params, silent: true).tap do |status|
|
48
|
+
status.type = :search
|
49
|
+
status.queue.each do |entry|
|
50
|
+
unless status.success?(entry)
|
51
|
+
email = nil
|
52
|
+
case
|
53
|
+
when entry.respond_to?(:email)
|
54
|
+
email = entry.email
|
55
|
+
when entry.respond_to?(:to_h)
|
56
|
+
email = entry.to_h["email"]
|
57
|
+
end
|
58
|
+
|
59
|
+
people_matching = []
|
60
|
+
email = email.to_s.strip.downcase
|
61
|
+
unless email.empty?
|
62
|
+
people_matching = get(params: params.merge(q: email), silent: true).select do |person|
|
63
|
+
person.email == email
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
case people_matching.length
|
68
|
+
when 1
|
69
|
+
status.set_person_match(entry, people_matching.first)
|
70
|
+
when 2..Float::INFINITY
|
71
|
+
status.set_people_match(entry, people.matching)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
25
76
|
end
|
26
77
|
|
27
78
|
private
|
@@ -30,38 +81,38 @@ module Eco
|
|
30
81
|
BatchStatus.new(enviro, queue: queue, method: method)
|
31
82
|
end
|
32
83
|
|
33
|
-
def
|
84
|
+
def get(params: {}, silent: false)
|
34
85
|
fatal "cannot batch get without api connnection, please provide a valid api connection!" unless people_api = api&.people
|
35
86
|
|
36
87
|
params = {per_page: DEFAULT_BATCH_BLOCK}.merge(params)
|
37
88
|
client = people_api.client
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
end
|
89
|
+
|
90
|
+
looping = !params.key?(:page)
|
91
|
+
page = params[:page] || 1
|
92
|
+
|
93
|
+
people = []; total_pages = nil
|
94
|
+
loop do
|
95
|
+
people_res, response = client_get(client, params: params.merge(page: page), silent: silent)
|
96
|
+
people += people_res
|
97
|
+
|
98
|
+
total_pages ||= response.body["total_pages"]
|
99
|
+
|
100
|
+
msg = "page number: #{page}/#{total_pages}, got num people #{people_res.length}, with total #{people.length} people got"
|
101
|
+
logger.info(msg) unless silent
|
102
|
+
|
103
|
+
break if page >= total_pages || !looping
|
104
|
+
page += 1
|
55
105
|
end
|
106
|
+
|
56
107
|
return people
|
57
108
|
end
|
58
109
|
|
59
|
-
def
|
110
|
+
def client_get(client, params:, silent: false)
|
60
111
|
response = client.get("/people", params: params)
|
61
112
|
unless response.success?
|
62
113
|
msg = "Request failed - params: #{params}"
|
63
114
|
msg += "\n Error message: - Status #{response.status}: #{response.body}"
|
64
|
-
fatal
|
115
|
+
fatal msg
|
65
116
|
end
|
66
117
|
people = []
|
67
118
|
response.body["results"].each do |person_hash|
|
@@ -72,40 +123,39 @@ module Eco
|
|
72
123
|
[people, response]
|
73
124
|
end
|
74
125
|
|
75
|
-
def batch_from(people, method
|
76
|
-
|
126
|
+
def batch_from(people, method:, params: {}, silent: false)
|
127
|
+
fatal "Invalid batch method: #{method}." if !self.class.valid_method?(method)
|
128
|
+
return nil if !people || !people.is_a?(Enumerable)
|
77
129
|
fatal "cannot batch #{method} without api connnection, please provide a valid api connection!" unless people_api = api&.people
|
78
130
|
|
79
131
|
# batch Status
|
80
132
|
status = new_status(people, method)
|
81
133
|
|
82
134
|
# param q does not make sense here, even for GET method
|
83
|
-
params
|
84
|
-
per_page = params
|
135
|
+
params = {per_page: DEFAULT_BATCH_BLOCK}.merge(params)
|
136
|
+
per_page = params[:per_page] || DEFAULT_BATCH_BLOCK
|
85
137
|
|
86
138
|
iteration = 1; done = 0
|
87
139
|
iterations = (people.length.to_f / per_page).ceil
|
88
140
|
|
89
141
|
people.each_slice(per_page) do |slice|
|
90
142
|
msg = "starting batch '#{method}' iteration #{iteration}/#{iterations}, with #{slice.length} entries of #{people.length} -- #{done} done"
|
91
|
-
logger.info(msg)
|
143
|
+
logger.info(msg) unless silent
|
92
144
|
|
93
145
|
people_api.batch do |batch|
|
94
146
|
slice.each do |person|
|
95
|
-
# valid method checked before
|
96
147
|
batch.public_send(method, person) do |response|
|
148
|
+
faltal("Request with no response") unless !!response
|
97
149
|
status[person] = response
|
98
|
-
|
99
|
-
|
100
|
-
end # next person
|
150
|
+
end
|
151
|
+
end
|
101
152
|
end # next batch
|
102
153
|
|
103
154
|
iteration += 1
|
104
155
|
done += slice.length
|
105
156
|
end # next slice
|
106
157
|
|
107
|
-
status.print_errors
|
108
|
-
|
158
|
+
status.print_errors unless silent
|
109
159
|
return status
|
110
160
|
end
|
111
161
|
|
@@ -2,15 +2,26 @@ module Eco
|
|
2
2
|
module API
|
3
3
|
class Session
|
4
4
|
class BatchJob < API::Common::Session::BaseSession
|
5
|
-
TYPES = [:create, :update, :delete
|
5
|
+
TYPES = [:get, :create, :update, :delete]
|
6
6
|
SETS = [:core, :details, :account]
|
7
7
|
|
8
8
|
attr_reader :name, :status
|
9
9
|
|
10
|
+
class << self
|
11
|
+
def valid_type?(value)
|
12
|
+
TYPES.include?(value)
|
13
|
+
end
|
14
|
+
|
15
|
+
def valid_sets?(value)
|
16
|
+
sets = [value].flatten
|
17
|
+
sets.all? { |s| SETS.include?(s) }
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
10
21
|
def initialize(e, name:, type:, sets:)
|
11
22
|
raise "A name is required to refer a job. Given: #{name}" if !name
|
12
|
-
raise "Type should be one of #{TYPES}. Given: #{type}" if !
|
13
|
-
raise "Sets should be some of #{SETS}. Given: #{sets}" if !
|
23
|
+
raise "Type should be one of #{TYPES}. Given: #{type}" if !self.class.valid_type?(type)
|
24
|
+
raise "Sets should be some of #{SETS}. Given: #{sets}" if !self.class.valid_sets?(sets)
|
14
25
|
super(e)
|
15
26
|
|
16
27
|
@name = name
|
@@ -20,9 +31,10 @@ module Eco
|
|
20
31
|
end
|
21
32
|
|
22
33
|
def reset
|
23
|
-
@queue
|
24
|
-
@
|
25
|
-
@
|
34
|
+
@queue = []
|
35
|
+
@queue_hash = {}
|
36
|
+
@callbacks = {}
|
37
|
+
@status = nil
|
26
38
|
end
|
27
39
|
|
28
40
|
def signature
|
@@ -38,82 +50,130 @@ module Eco
|
|
38
50
|
@queue.length > 0
|
39
51
|
end
|
40
52
|
|
41
|
-
def
|
53
|
+
def core?
|
54
|
+
sets.include?(:core)
|
55
|
+
end
|
56
|
+
|
57
|
+
def details?
|
58
|
+
sets.include?(:details)
|
59
|
+
end
|
60
|
+
|
61
|
+
def account?
|
62
|
+
sets.include?(:account)
|
63
|
+
end
|
64
|
+
|
65
|
+
# Adds an entry to the job queue.
|
66
|
+
# @param entry [Person] the person we want to update, carrying the changes to be done.
|
67
|
+
# @param unique [Boolean] specifies if repeated entries should be avoided in the queue.
|
68
|
+
# @yield [person] callback before launching the batch job request against the server.
|
69
|
+
# @yeldparam param [Person] current person object that that should be treated by the callback before launching the batch.
|
70
|
+
# @return [Void]
|
71
|
+
def add(entry, unique: true)
|
42
72
|
unless !entry
|
43
|
-
@
|
44
|
-
|
73
|
+
unless unique && @queue_hash.key?(entry)
|
74
|
+
@queue_hash[entry] = true
|
75
|
+
@queue.push(entry)
|
76
|
+
@callbacks[entry] = Proc.new if block_given?
|
77
|
+
end
|
45
78
|
end
|
46
79
|
end
|
47
80
|
|
48
|
-
def people
|
49
|
-
Eco::API::Organization::People.new(
|
81
|
+
def people(input = @queue)
|
82
|
+
Eco::API::Organization::People.new(input)
|
50
83
|
end
|
51
84
|
|
52
85
|
def processed_queue
|
53
|
-
@queue.map do |
|
54
|
-
callback = @callbacks[
|
55
|
-
|
56
|
-
|
57
|
-
e = nil if as_update(e).empty?
|
86
|
+
pre_queue = @queue.map do |e|
|
87
|
+
if callback = @callbacks[e]
|
88
|
+
callback.call(e)
|
89
|
+
end
|
90
|
+
e = nil if as_update(e).empty?
|
58
91
|
e
|
59
92
|
end.compact
|
93
|
+
apply_policies(pre_queue)
|
94
|
+
end
|
95
|
+
|
96
|
+
def processed_queue
|
97
|
+
@queue.each {|e| @callbacks[e].call(e) if @callbacks.key?(e) }
|
98
|
+
apply_policies(@queue).select {|e| !as_update(e).empty?}
|
60
99
|
end
|
61
100
|
|
62
101
|
def launch(simulate: false)
|
63
102
|
queue = processed_queue
|
64
103
|
launch_feedback(queue, simulate ? 2500 : 800)
|
65
104
|
|
66
|
-
if !simulate
|
67
|
-
|
68
|
-
|
69
|
-
|
105
|
+
if !simulate
|
106
|
+
if queue.length > 0
|
107
|
+
backup_update(queue)
|
108
|
+
@status = session.batch.launch(queue, method: @type)
|
109
|
+
@status.root = self
|
110
|
+
end
|
70
111
|
end
|
71
112
|
|
72
|
-
|
113
|
+
post_launch(queue: queue, simulate: simulate)
|
114
|
+
|
115
|
+
logger.info("Simulate: this would have launched: '#{@type}'") if simulate
|
73
116
|
return @status
|
74
117
|
end
|
75
118
|
|
76
|
-
def core?
|
77
|
-
sets.include?(:core)
|
78
|
-
end
|
79
119
|
|
80
|
-
|
81
|
-
sets.include?(:dettails)
|
82
|
-
end
|
83
|
-
|
84
|
-
def account?
|
85
|
-
sets.include?(:account)
|
86
|
-
end
|
120
|
+
private
|
87
121
|
|
88
|
-
def
|
89
|
-
|
122
|
+
def post_launch(queue: [], simulate: false)
|
123
|
+
if !simulate && @status
|
124
|
+
@status.queue.map do |entry|
|
125
|
+
if @status.success?(entry)
|
126
|
+
entry.consolidate! if entry.respond_to?(:consolidate!)
|
127
|
+
#else # shouldn't probably reset, as the model remains dirty? (well tracaked)
|
128
|
+
# entry.reset! if entry.respond_to?(:reset!)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
elsif simulate
|
132
|
+
queue.map do |entry|
|
133
|
+
entry.consolidate! if entry.respond_to?(:consolidate!)
|
134
|
+
end
|
135
|
+
end
|
90
136
|
end
|
91
137
|
|
92
|
-
def
|
93
|
-
|
94
|
-
|
138
|
+
def apply_policies(pre_queue)
|
139
|
+
pre_queue.tap do |entries|
|
140
|
+
policies = session.config.api_policies.policies
|
141
|
+
unless policies.empty?
|
142
|
+
policies.launch(people: people(entries), session: session)
|
143
|
+
end
|
144
|
+
end
|
95
145
|
end
|
96
146
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
if @type == :delete
|
102
|
-
hash = update.as_json.slice("id", "external_id")
|
147
|
+
def as_update(entry)
|
148
|
+
hash = entry if entry.is_a?(Hash)
|
149
|
+
if only_ids?
|
150
|
+
hash = entry.as_json.slice("id", "external_id", "email")
|
103
151
|
else
|
104
|
-
|
152
|
+
if entry.is_a?(Ecoportal::API::V1::Person)
|
153
|
+
hash = entry.as_update
|
154
|
+
if hfields = hash.dig("details", "fields")
|
155
|
+
hash["details"]["fields"] = hfields.map do |fld|
|
156
|
+
fld.merge!("alt_id" => entry.details.get_field(fld["id"]).alt_id) if entry.details
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
105
161
|
fields = hash&.dig('details', 'fields')
|
106
162
|
fields&.map! { |fld| fld&.slice("id", "alt_id", "value") }
|
107
163
|
end
|
108
164
|
hash || {}
|
109
165
|
end
|
110
166
|
|
167
|
+
def only_ids?
|
168
|
+
[:delete, :get].include?(@type)
|
169
|
+
end
|
170
|
+
|
111
171
|
def sets_title
|
112
172
|
"#{@sets.map {|s| s.to_s}.join(", ")}"
|
113
173
|
end
|
114
174
|
|
115
175
|
def launch_feedback(data, max_chars = 800)
|
116
|
-
if !data || !data.is_a?(
|
176
|
+
if !data || !data.is_a?(Enumerable) || data.empty?
|
117
177
|
logger.warn("#{"*" * 20} Nothing for #{signature} so far :) #{"*" * 20}")
|
118
178
|
return
|
119
179
|
end
|
@@ -122,8 +182,8 @@ module Eco
|
|
122
182
|
|
123
183
|
sample_length = 1
|
124
184
|
sample = data.slice(0, 20).map do |entry|
|
125
|
-
update
|
126
|
-
max_chars
|
185
|
+
update = as_update(entry)
|
186
|
+
max_chars -= update.pretty_inspect.length
|
127
187
|
sample_length += 1 if max_chars > 0
|
128
188
|
update
|
129
189
|
end
|
@@ -135,8 +195,8 @@ module Eco
|
|
135
195
|
|
136
196
|
def backup_update(data)
|
137
197
|
data_body = data.map { |u| as_update(u) }
|
138
|
-
dir
|
139
|
-
file
|
198
|
+
dir = config.people.requests_folder
|
199
|
+
file = File.join(dir, "#{@type}_data.json")
|
140
200
|
file_manager.save_json(data_body, file, :timestamp)
|
141
201
|
end
|
142
202
|
|
@@ -26,11 +26,10 @@ module Eco
|
|
26
26
|
def new(name, type:, sets:)
|
27
27
|
fatal "Can't create job named '#{name}' because it already exists." if exists?(name)
|
28
28
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
job
|
29
|
+
BatchJob.new(enviro, name: name, type: type, sets: sets).tap do |job|
|
30
|
+
@jobs[name] = job
|
31
|
+
@callbacks[job] = Proc.new if block_given?
|
32
|
+
end
|
34
33
|
end
|
35
34
|
|
36
35
|
def pending?
|