eco-helpers 2.0.44 → 2.0.48
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +59 -3
- data/eco-helpers.gemspec +1 -1
- data/lib/eco/api/common/people/entry_factory.rb +17 -1
- data/lib/eco/api/common/session/s3_uploader.rb +4 -2
- data/lib/eco/api/microcases/people_load.rb +1 -1
- data/lib/eco/api/session.rb +5 -0
- data/lib/eco/api/usecases/default_cases/clean_unknown_tags_case.rb +37 -1
- data/lib/eco/api/usecases/default_cases/samples/sftp_case.rb +109 -0
- data/lib/eco/api/usecases/default_cases/samples.rb +12 -0
- data/lib/eco/api/usecases/default_cases.rb +1 -0
- data/lib/eco/api/usecases/ooze_samples/ooze_base_case.rb +9 -5
- data/lib/eco/api/usecases/ooze_samples/ooze_run_base_case.rb +2 -2
- data/lib/eco/api/usecases/ooze_samples/register_export_case.rb +184 -0
- data/lib/eco/api/usecases/ooze_samples/register_update_case.rb +39 -1
- data/lib/eco/api/usecases/ooze_samples.rb +1 -0
- data/lib/eco/cli/config/default/usecases.rb +3 -0
- data/lib/eco/cli/config/default/workflow.rb +11 -2
- data/lib/eco/version.rb +1 -1
- metadata +6 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 39d2573c1aecfe5839a1867d549ae47af247cbb7819f4733a8c0a07ff98d9e65
|
4
|
+
data.tar.gz: c2318e2dd890bdf4a973332c6849c271bb8593c3f799c2add8b34064b022d54e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9319378014f0c596dfd3f458e2fe8f3852906c22bf04234f29c5bc28d3e2b677f1e801310d37fcb4505e555b1b258e5ba2a0c0a46c2ac5b7c32c96e274fda45a
|
7
|
+
data.tar.gz: 61cfb4fe7e2a7d125e3cc3c0c25175bd732fe048654e7b11044c4b897414d51253ab16f7b97d8178d4ec6167d4e216ec13af7666efc72fb278d332d5a6723c27
|
data/CHANGELOG.md
CHANGED
@@ -1,14 +1,70 @@
|
|
1
1
|
# Change Log
|
2
2
|
All notable changes to this project will be documented in this file.
|
3
3
|
|
4
|
-
## [2.0.
|
4
|
+
## [2.0.48] - 2022-02-xx
|
5
5
|
|
6
6
|
### Added
|
7
|
+
### Changed
|
8
|
+
- upgraded `ecoportal-api-v2` dependency
|
9
|
+
|
10
|
+
### Fixed
|
11
|
+
- `Eco::API::Common::People::EntryFactory#get_file_content`
|
12
|
+
- Call to `String#scrub` to prevent _invalid byte sequence_ error by replacing those characters by their coding identifier
|
13
|
+
- This has been implemented with a max tolerance of 5 byte sequence errors
|
14
|
+
- Errors are logged (although could not get to identify the line :/)
|
15
|
+
|
16
|
+
## [2.0.47] - 2022-02-03
|
7
17
|
|
18
|
+
### Added
|
19
|
+
- `Eco::API::UseCases::DefaultCases::CleanUnknownTags`
|
20
|
+
- It is possible to extend or override `register_tags`
|
21
|
+
- New command option `-include-register-tags` (will not ignore register tags; they will be removed too)
|
22
|
+
- `Eco::API::UseCases::DefaultCases::Samples::Sftp`
|
23
|
+
- Default sample case to list, get and archive SFTP remote files
|
8
24
|
### Changed
|
9
|
-
-
|
10
|
-
|
25
|
+
- upgraded `ecoportal-api-v2` dependency
|
26
|
+
|
27
|
+
### Fixed
|
28
|
+
- `Eco::API::Common::Session::S3Uploader#upload_file`: check if file exists before opening.
|
29
|
+
|
30
|
+
## [2.0.46] - 2021-12-24
|
31
|
+
|
32
|
+
### Added
|
33
|
+
- `Eco::API::UseCases::OozeSamples::RegisterUpdateCase`
|
34
|
+
- added counter of retrieved entries (to be able to compare with matched initially scoped results)
|
35
|
+
- added also counter of failed to retrieve oozes
|
36
|
+
- added also counter of double ups in search pagination
|
37
|
+
- counter for updated pages and pages that failed update
|
38
|
+
- `Eco::API::Session#enviro_name`: to identify in which environment we are running
|
39
|
+
|
40
|
+
### Changed
|
41
|
+
- upgraded `ecoportal-api-v2` dependency
|
42
|
+
- `Eco::API::UseCases::OozeSamples::OozeBaseCase#update_ooze`
|
43
|
+
- To return `false` if there was request made to the server
|
44
|
+
- To return `Reponse` if there was request made to the server
|
45
|
+
|
11
46
|
### Fixed
|
47
|
+
- Changed the `workflow` in
|
48
|
+
- `on(:end)` to do not recache people if there were no `:sync` or `:transform` cases
|
49
|
+
- `Eco::API::UseCases::OozeSamples::OozeRunBaseCase#update_ooze`
|
50
|
+
- It's parent had parameter (it as a child should so)
|
51
|
+
- `Eco::API::UseCases::OozeSamples::OozeBaseCase#process_ooze`
|
52
|
+
- It was not retrieving the `ooze` unless explicitly passed as parameter
|
53
|
+
- This did not play well with child implementations that expect it to retrieve it based on `ooze_id` cli argument
|
54
|
+
|
55
|
+
## [2.0.45] - 2021-12-15
|
56
|
+
|
57
|
+
### Added
|
58
|
+
- `Eco::API::UseCases::OozeSamples::RegisterExportCase`
|
59
|
+
|
60
|
+
### Changed
|
61
|
+
- Upgrade `ecoportal-api` dependency
|
62
|
+
- Upgrade `ecoportal-api-v2` dependency
|
63
|
+
|
64
|
+
## [2.0.44] - 2021-11-25
|
65
|
+
|
66
|
+
### Changed
|
67
|
+
- Upgrade `ecoportal-api` dependency
|
12
68
|
|
13
69
|
## [2.0.43] - 2021-11-25
|
14
70
|
|
data/eco-helpers.gemspec
CHANGED
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
|
|
31
31
|
spec.add_development_dependency "redcarpet", ">= 3.5.1", "< 3.6"
|
32
32
|
|
33
33
|
spec.add_dependency 'ecoportal-api', '>= 0.8.4', '< 0.9'
|
34
|
-
spec.add_dependency 'ecoportal-api-v2', '>= 0.8.
|
34
|
+
spec.add_dependency 'ecoportal-api-v2', '>= 0.8.25', '< 0.9'
|
35
35
|
spec.add_dependency 'aws-sdk-s3', '>= 1.83.0', '< 2'
|
36
36
|
spec.add_dependency 'aws-sdk-ses', '>= 1.36.0', '< 2'
|
37
37
|
spec.add_dependency 'dotenv', '>= 2.7.6', '< 2.8'
|
@@ -196,7 +196,23 @@ module Eco
|
|
196
196
|
ext = File.extname(file)
|
197
197
|
encoding ||= Eco::API::Common::Session::FileManager.encoding(file)
|
198
198
|
encoding = (encoding != "utf-8")? "#{encoding}|utf-8": encoding
|
199
|
-
|
199
|
+
read_with_tolerance(file, encoding: encoding)
|
200
|
+
end
|
201
|
+
|
202
|
+
def read_with_tolerance(file, encoding:)
|
203
|
+
if content = File.read(file, encoding: encoding)
|
204
|
+
tolerance = 5
|
205
|
+
content.scrub do |bytes|
|
206
|
+
replacement = '<' + bytes.unpack('H*')[0] + '>'
|
207
|
+
if tolerance <= 0
|
208
|
+
logger.error("There were more than 5 encoding errors in the file '#{file}'.")
|
209
|
+
return content
|
210
|
+
else
|
211
|
+
logger.error("Encoding problem in file '#{file}': '#{replacement}'.")
|
212
|
+
replacement
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
200
216
|
end
|
201
217
|
|
202
218
|
def fatal(msg)
|
@@ -32,8 +32,10 @@ module Eco
|
|
32
32
|
# @param path [String] the target file to be uploaded
|
33
33
|
# @return [String] S3 path to the uploaded `path` file
|
34
34
|
def upload_file(path)
|
35
|
-
File.
|
36
|
-
|
35
|
+
if File.exist?(path)
|
36
|
+
File.open(path, "rb") do |f|
|
37
|
+
upload(File.basename(path), f)
|
38
|
+
end
|
37
39
|
end
|
38
40
|
end
|
39
41
|
|
@@ -30,7 +30,7 @@ module Eco
|
|
30
30
|
people_load(modifier: modifier - [:newest, :file])
|
31
31
|
end
|
32
32
|
when modifier.include?(:api)
|
33
|
-
logger.info("Going to get all the people via API")
|
33
|
+
logger.info("Going to get all the people via API (load)")
|
34
34
|
|
35
35
|
start = Time.now
|
36
36
|
session.batch.get_people.tap do |people|
|
data/lib/eco/api/session.rb
CHANGED
@@ -17,6 +17,11 @@ module Eco
|
|
17
17
|
logger.debug("LINE COMMAND: #{$0} #{ARGV.join(" ")}")
|
18
18
|
end
|
19
19
|
|
20
|
+
# @retrun [String] the name of the current environment
|
21
|
+
def enviro_name
|
22
|
+
config.active_enviro
|
23
|
+
end
|
24
|
+
|
20
25
|
# @return [Eco::API::Session::Batch] provides helper to launch batch operations.
|
21
26
|
def batch
|
22
27
|
@batch ||= Batch.new(enviro)
|
@@ -2,6 +2,8 @@ class Eco::API::UseCases::DefaultCases::CleanUnknownTags < Eco::API::Common::Loa
|
|
2
2
|
name "clean-unknown-tags"
|
3
3
|
type :transform
|
4
4
|
|
5
|
+
UNKNOWN_LOCS_FILE = "unknown_locations.txt"
|
6
|
+
|
5
7
|
REGISTER_TAGS = [
|
6
8
|
"EVENT", "INJURY", "RISK", "CONTRACTOR", "PERMIT",
|
7
9
|
"AUDIT", "JSEA",
|
@@ -20,18 +22,52 @@ class Eco::API::UseCases::DefaultCases::CleanUnknownTags < Eco::API::Common::Loa
|
|
20
22
|
people.each do |person|
|
21
23
|
unknown_tags = person.filter_tags.select {|tag| !tag?(tag)}
|
22
24
|
person.filter_tags -= unknown_tags
|
25
|
+
unknown_tag!(*unknown_tags)
|
23
26
|
update.add(person)
|
24
27
|
end
|
25
28
|
end
|
26
29
|
|
27
30
|
private
|
28
31
|
|
32
|
+
def unknown_tag!(*tags)
|
33
|
+
tags.each do |value|
|
34
|
+
unless unknown.include?(value)
|
35
|
+
unknown << value
|
36
|
+
File.open(unknown_tags_file, 'a') do |fd|
|
37
|
+
fd.write("#{value}\n")
|
38
|
+
end
|
39
|
+
puts "Custom detected: '#{value}'"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def register_tags
|
45
|
+
REGISTER_TAGS
|
46
|
+
end
|
47
|
+
|
48
|
+
def unknown
|
49
|
+
@unknown ||= []
|
50
|
+
end
|
51
|
+
|
29
52
|
def tag?(value)
|
30
|
-
tagtree.tag?(value)
|
53
|
+
tagtree.tag?(value).yield_self do |is_loc|
|
54
|
+
next true if is_loc
|
55
|
+
if clean_register_tags?
|
56
|
+
register_tags.any? {|reg| value == reg}
|
57
|
+
end
|
58
|
+
end
|
31
59
|
end
|
32
60
|
|
33
61
|
def tagtree
|
34
62
|
@tagtree ||= ASSETS.config.tagtree
|
35
63
|
end
|
36
64
|
|
65
|
+
def unknown_tags_file
|
66
|
+
File.join(".","#{session.enviro_name}_#{UNKNOWN_LOCS_FILE}")
|
67
|
+
end
|
68
|
+
|
69
|
+
def clean_register_tags?
|
70
|
+
!options.dig(:usecase, :include_register_tags)
|
71
|
+
end
|
72
|
+
|
37
73
|
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
class Eco::API::UseCases::DefaultCases::Samples::Sftp < Eco::API::Common::Loaders::UseCase
|
2
|
+
name "sftp-sample"
|
3
|
+
type :other
|
4
|
+
|
5
|
+
attr_reader :session
|
6
|
+
|
7
|
+
def main(session, options, usecase)
|
8
|
+
@session = session
|
9
|
+
options[:end_get] = false
|
10
|
+
raise "The SFTP is not configured" unless session.sftp?
|
11
|
+
case options.dig(:sftp, :command)
|
12
|
+
when :list
|
13
|
+
list_folder
|
14
|
+
when :get
|
15
|
+
get_files
|
16
|
+
when :get_last
|
17
|
+
get_last
|
18
|
+
when :archive
|
19
|
+
archive_files
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def file_pattern
|
26
|
+
raise "You should redefine the file_pattern function as a RegEx expression that matches the target remote file"
|
27
|
+
end
|
28
|
+
|
29
|
+
# Ex: "/IN/Personnel"
|
30
|
+
def remote_subfolder
|
31
|
+
raise "You should redefine remote_subfolder as the folder where the target file sits"
|
32
|
+
end
|
33
|
+
|
34
|
+
def archive_subfolder
|
35
|
+
"Archive"
|
36
|
+
end
|
37
|
+
|
38
|
+
def with_remote_files
|
39
|
+
sftp.files(remote_folder, pattern: file_pattern).each do |remote_file|
|
40
|
+
yield(remote_file) if block_given?
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def list_folder
|
45
|
+
puts "Listing remote folder: '#{remote_folder}':"
|
46
|
+
with_remote_files {|file| puts file.longname}
|
47
|
+
end
|
48
|
+
|
49
|
+
def get_files
|
50
|
+
with_remote_files.tap do |files|
|
51
|
+
unless files.empty?
|
52
|
+
file_names = files.map {|file| to_remote_path(file.name)}
|
53
|
+
puts "Getting the following files:"
|
54
|
+
puts file_names
|
55
|
+
sftp.download(file_names, local_folder: local_folder)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def get_last
|
61
|
+
with_remote_files.last.tap do |file|
|
62
|
+
if file
|
63
|
+
file_name = to_remote_path(file.name)
|
64
|
+
puts "Getting the following file: #{file_name}"
|
65
|
+
sftp.download(file_name, local_folder: local_folder)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def archive_files
|
71
|
+
with_remote_files do |file|
|
72
|
+
source = to_remote_path(file.name) # should probably be file.longname
|
73
|
+
dest = to_remote_path("#{archive_subfolder}/#{file.name}")
|
74
|
+
move_file(source, dest)
|
75
|
+
end.tap do |files|
|
76
|
+
puts "Moved the file(s) to the #{archive_subfolder} folder" unless files.empty?
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def move_file(source, dest)
|
81
|
+
sftp.move(source, dest, 0x0001) do |response|
|
82
|
+
if response.ok?
|
83
|
+
puts "#{source}\n -to-> #{dest}"
|
84
|
+
else
|
85
|
+
puts "Could not move file #{source}"
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def to_remote_path(file)
|
91
|
+
remote_folder + "/" + file
|
92
|
+
end
|
93
|
+
|
94
|
+
def local_folder
|
95
|
+
"."
|
96
|
+
end
|
97
|
+
|
98
|
+
def remote_folder
|
99
|
+
@remote_folder ||= sftp_config.remote_folder + remote_subfolder
|
100
|
+
end
|
101
|
+
|
102
|
+
def sftp_config
|
103
|
+
session.config.sftp
|
104
|
+
end
|
105
|
+
|
106
|
+
def sftp
|
107
|
+
session.sftp
|
108
|
+
end
|
109
|
+
end
|
@@ -20,10 +20,11 @@ class Eco::API::UseCases::OozeSamples::OozeBaseCase < Eco::API::Common::Loaders:
|
|
20
20
|
end
|
21
21
|
|
22
22
|
# Write here your script
|
23
|
-
def process_ooze(
|
23
|
+
def process_ooze(ooz = target)
|
24
|
+
ooz ||= ooze
|
24
25
|
raise "You need to inherit from this class ('#{self.class}') and call super with a block" unless block_given?
|
25
|
-
yield(
|
26
|
-
update_ooze(
|
26
|
+
yield(ooz)
|
27
|
+
update_ooze(ooz)
|
27
28
|
end
|
28
29
|
|
29
30
|
protected
|
@@ -109,10 +110,12 @@ class Eco::API::UseCases::OozeSamples::OozeBaseCase < Eco::API::Common::Loaders:
|
|
109
110
|
exit_error "Stage '#{id_name}' doesn't exist in ooze '#{ooze_id}'"
|
110
111
|
end
|
111
112
|
|
113
|
+
# It fill update the ooze only if it's dirty (it carries changes)
|
114
|
+
# @return [Boolean, Response] `false` if there was not request against the server, `Response` otherwise
|
112
115
|
def update_ooze(ooze = target)
|
113
116
|
if !options[:simulate]
|
114
|
-
return unless dirty?(ooze)
|
115
|
-
|
117
|
+
return false unless dirty?(ooze)
|
118
|
+
|
116
119
|
ooze.validate.tap do |validation|
|
117
120
|
raise validation if validation.is_a?(String)
|
118
121
|
end
|
@@ -133,6 +136,7 @@ class Eco::API::UseCases::OozeSamples::OozeBaseCase < Eco::API::Common::Loaders:
|
|
133
136
|
end
|
134
137
|
backup_patch!(ooze)
|
135
138
|
exit(0) if dirty?(ooze) && dry_count > DRY_COUNT
|
139
|
+
false
|
136
140
|
end
|
137
141
|
end
|
138
142
|
|
@@ -39,9 +39,9 @@ class Eco::API::UseCases::OozeSamples::OozeRunBaseCase < Eco::API::UseCases::Ooz
|
|
39
39
|
options.dig(:source, :stage_id)
|
40
40
|
end
|
41
41
|
|
42
|
-
def update_ooze
|
42
|
+
def update_ooze(ooz = target)
|
43
43
|
prompt_to_confirm!
|
44
|
-
super
|
44
|
+
super(ooz)
|
45
45
|
end
|
46
46
|
|
47
47
|
def exit_if_no_changes!
|
@@ -0,0 +1,184 @@
|
|
1
|
+
# Use case to update a register
|
2
|
+
# @note
|
3
|
+
# - You can define methods `filters` and `search` to change the target entries of the register
|
4
|
+
# - You need to define the `process_ooze` method
|
5
|
+
# This case expects `options[:source][:register_id]`
|
6
|
+
class Eco::API::UseCases::OozeSamples::RegisterExportCase < Eco::API::Common::Loaders::UseCase
|
7
|
+
|
8
|
+
class << self
|
9
|
+
# @return [Integer] the number of pages to be processed in each batch
|
10
|
+
def batch_size(size = nil)
|
11
|
+
@batch_size ||= 25
|
12
|
+
return @batch_size unless size
|
13
|
+
@batch_size = size
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
include Eco::API::UseCases::OozeSamples::Helpers
|
18
|
+
|
19
|
+
name "register-export-case"
|
20
|
+
type :other
|
21
|
+
|
22
|
+
attr_reader :session, :options, :usecase
|
23
|
+
attr_reader :target
|
24
|
+
|
25
|
+
def main(session, options, usecase)
|
26
|
+
options[:end_get] = false
|
27
|
+
@session = session; @options = options; @usecase = usecase
|
28
|
+
@target = nil
|
29
|
+
raise "You need to inherit from this class ('#{self.class}') and call super with a block" unless block_given?
|
30
|
+
with_each_entry do |ooze|
|
31
|
+
process_ooze(ooze)
|
32
|
+
end
|
33
|
+
yield
|
34
|
+
end
|
35
|
+
|
36
|
+
# Write here your script
|
37
|
+
def process_ooze(ooze = target)
|
38
|
+
raise "You need to inherit from this class ('#{self.class}') and call super with a block" unless block_given?
|
39
|
+
yield(ooze)
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def new_target(object)
|
45
|
+
@target = object
|
46
|
+
end
|
47
|
+
|
48
|
+
def with_each_entry
|
49
|
+
batched_search_results do |page_results|
|
50
|
+
page_results.each do |page_result|
|
51
|
+
if ooz = build_full_ooze(page_result.id)
|
52
|
+
yield(ooz)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
# It builds a full page model (not updatable)
|
59
|
+
# @note If it's a page instance with stages, there's where it will be handy
|
60
|
+
# @param ooze_id [String]
|
61
|
+
# @return [Ecoportal::API::V2::Page]
|
62
|
+
def build_full_ooze(ooze_id)
|
63
|
+
if page = ooze(ooze_id)
|
64
|
+
return page unless page.is_a?(Ecoportal::API::V2::Pages::PageStage)
|
65
|
+
secs_doc = page.sections.doc
|
66
|
+
flds_doc = page.components.doc
|
67
|
+
pending_stage_ids = page.stages.map(&:id) - [page.current_stage_id]
|
68
|
+
pending_stage_ids.each do |id|
|
69
|
+
if page = stage(id, ooze: page)
|
70
|
+
page.sections.doc.each do |sec_doc|
|
71
|
+
unless secs_doc.find {|sec| sec["id"] == sec_doc["id"]}
|
72
|
+
secs_doc << sec_doc
|
73
|
+
end
|
74
|
+
end
|
75
|
+
page.components.doc.each do |comp_doc|
|
76
|
+
unless flds_doc.find {|fld| fld["id"] == comp_doc["id"]}
|
77
|
+
flds_doc << comp_doc
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
Ecoportal::API::V2::Page.new(page.doc)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
#def update_oozes(batched_oozes = batch_queue)
|
87
|
+
# batched_oozes.each do |ooze|
|
88
|
+
# update_ooze(ooze)
|
89
|
+
# end
|
90
|
+
# batched_oozes.clear
|
91
|
+
#end
|
92
|
+
|
93
|
+
def batched_search_results
|
94
|
+
raise "Missing block. It yields in slices of #{self.class.batch_size} results" unless block_given?
|
95
|
+
results_preview
|
96
|
+
results = []
|
97
|
+
apiv2.registers.search(register_id, search_options) do |page_result|
|
98
|
+
results << page_result
|
99
|
+
if results.length >= self.class.batch_size
|
100
|
+
yield(results)
|
101
|
+
results = []
|
102
|
+
end
|
103
|
+
end
|
104
|
+
yield(results) unless results.empty?
|
105
|
+
end
|
106
|
+
|
107
|
+
def ooze(ooze_id = nil, stage_id: nil)
|
108
|
+
return target unless ooze_id
|
109
|
+
apiv2.pages.get(ooze_id, stage_id: stage_id).tap do |ooze|
|
110
|
+
if ooze
|
111
|
+
new_target(ooze)
|
112
|
+
logger.info("Got #{object_reference(ooze)}")
|
113
|
+
else
|
114
|
+
exit_error "Could not get ooze '#{ooze_id}'"
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def stage(id_name = nil, ooze: target)
|
120
|
+
if ooze_id = ooze && ooze.id
|
121
|
+
exit_error "#{object_reference(ooze)} does not have stages!" unless ooze.stages?
|
122
|
+
else
|
123
|
+
exit_error "There's no target ooze to get retrieve stages from"
|
124
|
+
end
|
125
|
+
|
126
|
+
if stg = ooze.stages[id_name] || ooze.stages.get_by_name(id_name)
|
127
|
+
return ooze if ooze.respond_to?(:current_stage_id) && (ooze.current_stage_id == stg.id)
|
128
|
+
return apiv2.pages.get(ooze_id, stage_id: stg.id).tap do |stage|
|
129
|
+
if stage
|
130
|
+
new_target(stage)
|
131
|
+
logger.info("Got #{object_reference(stage)} from #{object_reference(ooze)}")
|
132
|
+
else
|
133
|
+
exit_error "Could not get stage '#{id_name}' in ooze '#{ooze_id}'"
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
exit_error "Stage '#{id_name}' doesn't exist in ooze '#{ooze_id}'"
|
138
|
+
end
|
139
|
+
|
140
|
+
def results_preview
|
141
|
+
apiv2.registers.search(register_id, search_options.merge(only_first: true)).tap do |search_results|
|
142
|
+
str_results = "Total target entries: #{search_results.total} (out of #{search_results.total_before_filtering})"
|
143
|
+
session.prompt_user("Do you want to proceed (y/N):", explanation: str_results, default: "N", timeout: 10) do |res|
|
144
|
+
unless res.upcase.start_with?("Y")
|
145
|
+
puts "..."
|
146
|
+
logger.info "Aborting script..."
|
147
|
+
exit(0)
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
def search_options
|
154
|
+
@search_options ||= {}.tap do |opts|
|
155
|
+
opts.merge!(sort: "created_at")
|
156
|
+
opts.merge!(dir: "asc")
|
157
|
+
opts.merge!(query: conf_search) if conf_search
|
158
|
+
opts.merge!(filters: conf_filters)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
def conf_filters
|
163
|
+
return filters if self.respond_to?(:filters)
|
164
|
+
[]
|
165
|
+
end
|
166
|
+
|
167
|
+
def conf_search
|
168
|
+
return search if self.respond_to?(:search)
|
169
|
+
end
|
170
|
+
|
171
|
+
def register_id
|
172
|
+
options.dig(:source, :register_id)
|
173
|
+
end
|
174
|
+
|
175
|
+
def apiv2
|
176
|
+
@apiv2 ||= session.api(version: :oozes)
|
177
|
+
end
|
178
|
+
|
179
|
+
def exit_error(msg)
|
180
|
+
logger.error(msg)
|
181
|
+
exit(1)
|
182
|
+
end
|
183
|
+
|
184
|
+
end
|
@@ -17,12 +17,30 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
17
17
|
name "register-update-case"
|
18
18
|
type :other
|
19
19
|
|
20
|
+
attr_reader :retrieved_oozes, :non_retrieved_oozes
|
21
|
+
attr_reader :total_search_oozes, :dupped_search_oozes
|
22
|
+
attr_reader :ooze_result_ids, :updated_oozes, :failed_update_oozes
|
23
|
+
|
20
24
|
def main(session, options, usecase, &block)
|
25
|
+
@retrieved_oozes = 0
|
26
|
+
@non_retrieved_oozes = 0
|
27
|
+
@dupped_search_oozes = 0
|
28
|
+
@ooze_result_ids = {}
|
29
|
+
@updated_oozes = 0
|
30
|
+
@failed_update_oozes = 0
|
31
|
+
|
21
32
|
super(session, options, usecase) do
|
22
33
|
with_each_entry do |ooze|
|
23
34
|
process_ooze(&block)
|
24
35
|
end
|
25
36
|
end
|
37
|
+
msg = "Run end:\n"
|
38
|
+
msg += " • Search results: #{total_search_oozes}\n"
|
39
|
+
msg += " • Duplicated search results #{dupped_search_oozes}\n"
|
40
|
+
msg += " • Retrieved a total of #{retrieved_oozes}\n"
|
41
|
+
msg += " • Could not get #{non_retrieved_oozes} oozes.\n"
|
42
|
+
msg += " • Updated #{updated_oozes} oozes (failed update on #{failed_update_oozes} oozes).\n"
|
43
|
+
logger.info(msg)
|
26
44
|
end
|
27
45
|
|
28
46
|
def process_ooze(ooze = target)
|
@@ -47,7 +65,14 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
47
65
|
def with_each_entry
|
48
66
|
batched_search_results do |page_results|
|
49
67
|
page_results.each do |page_result|
|
68
|
+
if ooze_result_ids[page_result.id]
|
69
|
+
@dupped_search_oozes += 1
|
70
|
+
else
|
71
|
+
ooze_result_ids[page_result.id] = true
|
72
|
+
end
|
73
|
+
|
50
74
|
if pending = queue_shift(page_result.id)
|
75
|
+
|
51
76
|
if dirty?(pending)
|
52
77
|
msg = "Inconsistent search results. "
|
53
78
|
msg << "Launching update on '#{object_reference(pending)}' to be able to queue it back"
|
@@ -56,7 +81,11 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
56
81
|
end
|
57
82
|
end
|
58
83
|
if ooz = ooze(page_result.id)
|
84
|
+
@retrieved_oozes += 1
|
59
85
|
yield(ooz)
|
86
|
+
else
|
87
|
+
@non_retrieved_oozes += 1
|
88
|
+
console.warn "Could not get page #{page_result.id}"
|
60
89
|
end
|
61
90
|
end
|
62
91
|
update_oozes
|
@@ -113,7 +142,15 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
113
142
|
|
114
143
|
def update_oozes(batched_oozes = batch_queue)
|
115
144
|
batched_oozes.each do |ooze|
|
116
|
-
update_ooze(ooze)
|
145
|
+
update_ooze(ooze).tap do |result|
|
146
|
+
if result.is_a?(Ecoportal::API::Common::Response)
|
147
|
+
if result.success?
|
148
|
+
@updated_oozes += 1
|
149
|
+
else
|
150
|
+
@failed_update_oozes +=1
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
117
154
|
end
|
118
155
|
batched_oozes.clear
|
119
156
|
end
|
@@ -127,6 +164,7 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
127
164
|
|
128
165
|
def results_preview
|
129
166
|
apiv2.registers.search(register_id, search_options.merge(only_first: true)).tap do |search_results|
|
167
|
+
@total_search_oozes = search_results.total
|
130
168
|
str_results = "Total target entries: #{search_results.total} (out of #{search_results.total_before_filtering})"
|
131
169
|
session.prompt_user("Do you want to proceed (y/N):", explanation: str_results, default: "N", timeout: 10) do |res|
|
132
170
|
unless res.upcase.start_with?("Y")
|
@@ -10,6 +10,7 @@ end
|
|
10
10
|
|
11
11
|
require_relative 'ooze_samples/helpers'
|
12
12
|
require_relative 'ooze_samples/ooze_base_case'
|
13
|
+
require_relative 'ooze_samples/register_export_case'
|
13
14
|
require_relative 'ooze_samples/ooze_run_base_case'
|
14
15
|
require_relative 'ooze_samples/ooze_update_case'
|
15
16
|
require_relative 'ooze_samples/ooze_from_doc_case'
|
@@ -90,6 +90,9 @@ ASSETS.cli.config do |cnf|
|
|
90
90
|
desc = "Cleans from filter_tags those tags that are not present in the tagtree (as per tagtree.json file)."
|
91
91
|
desc += " It will preserve standard register tags of most common registers (i.e. EVENT, RISK)."
|
92
92
|
cases.add("-clean-unknown-tags", :transform, desc, case_name: "clean-unknown-tags")
|
93
|
+
.add_option("-include-register-tags", "Will also try to remove register tags") do |options|
|
94
|
+
options.deep_merge!(usecase: {include_register_tags: true})
|
95
|
+
end
|
93
96
|
|
94
97
|
desc = "Removes the landing page or sets it to -page-id"
|
95
98
|
cases.add("-reset-landing-page", :transform, desc, case_name: "reset-landing-page")
|
@@ -3,6 +3,8 @@ ASSETS.cli.config do |config|
|
|
3
3
|
|
4
4
|
io = nil
|
5
5
|
rescued = false
|
6
|
+
cases_with_input = nil
|
7
|
+
cases_with_output = nil
|
6
8
|
|
7
9
|
# default rescue
|
8
10
|
wf.rescue do |exception, io|
|
@@ -165,9 +167,16 @@ ASSETS.cli.config do |config|
|
|
165
167
|
wf.on(:end) do |wf_end, io|
|
166
168
|
get_people = io.options.dig(:people, :get)
|
167
169
|
partial_update = get_people && get_people.dig(:type) == :partial
|
170
|
+
|
168
171
|
unless !io.options[:end_get] || io.options[:dry_run] || partial_update
|
169
|
-
|
170
|
-
|
172
|
+
people_update_cases = config.usecases.active(io: io).any? do |usecase, data|
|
173
|
+
[:transform, :sync].any? {|type| usecase.type == type}
|
174
|
+
end
|
175
|
+
|
176
|
+
if !people_update_cases
|
177
|
+
# Prevent getting people when there were no use cases that used them
|
178
|
+
io.session.logger.info("Won't be recaching people, as there haven't been any targetted updates")
|
179
|
+
elsif !io.people
|
171
180
|
people = io.session.micro.people_cache
|
172
181
|
io = io.new(people: people)
|
173
182
|
end
|
data/lib/eco/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: eco-helpers
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.48
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oscar Segura
|
@@ -136,7 +136,7 @@ dependencies:
|
|
136
136
|
requirements:
|
137
137
|
- - ">="
|
138
138
|
- !ruby/object:Gem::Version
|
139
|
-
version: 0.8.
|
139
|
+
version: 0.8.25
|
140
140
|
- - "<"
|
141
141
|
- !ruby/object:Gem::Version
|
142
142
|
version: '0.9'
|
@@ -146,7 +146,7 @@ dependencies:
|
|
146
146
|
requirements:
|
147
147
|
- - ">="
|
148
148
|
- !ruby/object:Gem::Version
|
149
|
-
version: 0.8.
|
149
|
+
version: 0.8.25
|
150
150
|
- - "<"
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0.9'
|
@@ -546,6 +546,8 @@ files:
|
|
546
546
|
- lib/eco/api/usecases/default_cases/remove_account_trans_case.rb
|
547
547
|
- lib/eco/api/usecases/default_cases/reset_landing_page_case.rb
|
548
548
|
- lib/eco/api/usecases/default_cases/restore_db_case.rb
|
549
|
+
- lib/eco/api/usecases/default_cases/samples.rb
|
550
|
+
- lib/eco/api/usecases/default_cases/samples/sftp_case.rb
|
549
551
|
- lib/eco/api/usecases/default_cases/set_default_tag_case.rb
|
550
552
|
- lib/eco/api/usecases/default_cases/set_supervisor_case.rb
|
551
553
|
- lib/eco/api/usecases/default_cases/supers_cyclic_identify_case.rb
|
@@ -564,6 +566,7 @@ files:
|
|
564
566
|
- lib/eco/api/usecases/ooze_samples/ooze_from_doc_case.rb
|
565
567
|
- lib/eco/api/usecases/ooze_samples/ooze_run_base_case.rb
|
566
568
|
- lib/eco/api/usecases/ooze_samples/ooze_update_case.rb
|
569
|
+
- lib/eco/api/usecases/ooze_samples/register_export_case.rb
|
567
570
|
- lib/eco/api/usecases/ooze_samples/register_update_case.rb
|
568
571
|
- lib/eco/api/usecases/ooze_samples/target_oozes_update_case.rb
|
569
572
|
- lib/eco/api/usecases/use_case.rb
|