eco-helpers 2.0.44 → 2.0.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +59 -3
- data/eco-helpers.gemspec +1 -1
- data/lib/eco/api/common/people/entry_factory.rb +17 -1
- data/lib/eco/api/common/session/s3_uploader.rb +4 -2
- data/lib/eco/api/microcases/people_load.rb +1 -1
- data/lib/eco/api/session.rb +5 -0
- data/lib/eco/api/usecases/default_cases/clean_unknown_tags_case.rb +37 -1
- data/lib/eco/api/usecases/default_cases/samples/sftp_case.rb +109 -0
- data/lib/eco/api/usecases/default_cases/samples.rb +12 -0
- data/lib/eco/api/usecases/default_cases.rb +1 -0
- data/lib/eco/api/usecases/ooze_samples/ooze_base_case.rb +9 -5
- data/lib/eco/api/usecases/ooze_samples/ooze_run_base_case.rb +2 -2
- data/lib/eco/api/usecases/ooze_samples/register_export_case.rb +184 -0
- data/lib/eco/api/usecases/ooze_samples/register_update_case.rb +39 -1
- data/lib/eco/api/usecases/ooze_samples.rb +1 -0
- data/lib/eco/cli/config/default/usecases.rb +3 -0
- data/lib/eco/cli/config/default/workflow.rb +11 -2
- data/lib/eco/version.rb +1 -1
- metadata +6 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 39d2573c1aecfe5839a1867d549ae47af247cbb7819f4733a8c0a07ff98d9e65
|
|
4
|
+
data.tar.gz: c2318e2dd890bdf4a973332c6849c271bb8593c3f799c2add8b34064b022d54e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9319378014f0c596dfd3f458e2fe8f3852906c22bf04234f29c5bc28d3e2b677f1e801310d37fcb4505e555b1b258e5ba2a0c0a46c2ac5b7c32c96e274fda45a
|
|
7
|
+
data.tar.gz: 61cfb4fe7e2a7d125e3cc3c0c25175bd732fe048654e7b11044c4b897414d51253ab16f7b97d8178d4ec6167d4e216ec13af7666efc72fb278d332d5a6723c27
|
data/CHANGELOG.md
CHANGED
|
@@ -1,14 +1,70 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
All notable changes to this project will be documented in this file.
|
|
3
3
|
|
|
4
|
-
## [2.0.
|
|
4
|
+
## [2.0.48] - 2022-02-xx
|
|
5
5
|
|
|
6
6
|
### Added
|
|
7
|
+
### Changed
|
|
8
|
+
- upgraded `ecoportal-api-v2` dependency
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
- `Eco::API::Common::People::EntryFactory#get_file_content`
|
|
12
|
+
- Call to `String#scrub` to prevent _invalid byte sequence_ error by replacing those characters by their coding identifier
|
|
13
|
+
- This has been implemented with a max tolerance of 5 byte sequence errors
|
|
14
|
+
- Errors are logged (although could not get to identify the line :/)
|
|
15
|
+
|
|
16
|
+
## [2.0.47] - 2022-02-03
|
|
7
17
|
|
|
18
|
+
### Added
|
|
19
|
+
- `Eco::API::UseCases::DefaultCases::CleanUnknownTags`
|
|
20
|
+
- It is possible to extend or override `register_tags`
|
|
21
|
+
- New command option `-include-register-tags` (will not ignore register tags; they will be removed too)
|
|
22
|
+
- `Eco::API::UseCases::DefaultCases::Samples::Sftp`
|
|
23
|
+
- Default sample case to list, get and archive SFTP remote files
|
|
8
24
|
### Changed
|
|
9
|
-
-
|
|
10
|
-
|
|
25
|
+
- upgraded `ecoportal-api-v2` dependency
|
|
26
|
+
|
|
27
|
+
### Fixed
|
|
28
|
+
- `Eco::API::Common::Session::S3Uploader#upload_file`: check if file exists before opening.
|
|
29
|
+
|
|
30
|
+
## [2.0.46] - 2021-12-24
|
|
31
|
+
|
|
32
|
+
### Added
|
|
33
|
+
- `Eco::API::UseCases::OozeSamples::RegisterUpdateCase`
|
|
34
|
+
- added counter of retrieved entries (to be able to compare with matched initially scoped results)
|
|
35
|
+
- added also counter of failed to retrieve oozes
|
|
36
|
+
- added also counter of double ups in search pagination
|
|
37
|
+
- counter for updated pages and pages that failed update
|
|
38
|
+
- `Eco::API::Session#enviro_name`: to identify in which environment we are running
|
|
39
|
+
|
|
40
|
+
### Changed
|
|
41
|
+
- upgraded `ecoportal-api-v2` dependency
|
|
42
|
+
- `Eco::API::UseCases::OozeSamples::OozeBaseCase#update_ooze`
|
|
43
|
+
- To return `false` if there was request made to the server
|
|
44
|
+
- To return `Reponse` if there was request made to the server
|
|
45
|
+
|
|
11
46
|
### Fixed
|
|
47
|
+
- Changed the `workflow` in
|
|
48
|
+
- `on(:end)` to do not recache people if there were no `:sync` or `:transform` cases
|
|
49
|
+
- `Eco::API::UseCases::OozeSamples::OozeRunBaseCase#update_ooze`
|
|
50
|
+
- It's parent had parameter (it as a child should so)
|
|
51
|
+
- `Eco::API::UseCases::OozeSamples::OozeBaseCase#process_ooze`
|
|
52
|
+
- It was not retrieving the `ooze` unless explicitly passed as parameter
|
|
53
|
+
- This did not play well with child implementations that expect it to retrieve it based on `ooze_id` cli argument
|
|
54
|
+
|
|
55
|
+
## [2.0.45] - 2021-12-15
|
|
56
|
+
|
|
57
|
+
### Added
|
|
58
|
+
- `Eco::API::UseCases::OozeSamples::RegisterExportCase`
|
|
59
|
+
|
|
60
|
+
### Changed
|
|
61
|
+
- Upgrade `ecoportal-api` dependency
|
|
62
|
+
- Upgrade `ecoportal-api-v2` dependency
|
|
63
|
+
|
|
64
|
+
## [2.0.44] - 2021-11-25
|
|
65
|
+
|
|
66
|
+
### Changed
|
|
67
|
+
- Upgrade `ecoportal-api` dependency
|
|
12
68
|
|
|
13
69
|
## [2.0.43] - 2021-11-25
|
|
14
70
|
|
data/eco-helpers.gemspec
CHANGED
|
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
|
|
|
31
31
|
spec.add_development_dependency "redcarpet", ">= 3.5.1", "< 3.6"
|
|
32
32
|
|
|
33
33
|
spec.add_dependency 'ecoportal-api', '>= 0.8.4', '< 0.9'
|
|
34
|
-
spec.add_dependency 'ecoportal-api-v2', '>= 0.8.
|
|
34
|
+
spec.add_dependency 'ecoportal-api-v2', '>= 0.8.25', '< 0.9'
|
|
35
35
|
spec.add_dependency 'aws-sdk-s3', '>= 1.83.0', '< 2'
|
|
36
36
|
spec.add_dependency 'aws-sdk-ses', '>= 1.36.0', '< 2'
|
|
37
37
|
spec.add_dependency 'dotenv', '>= 2.7.6', '< 2.8'
|
|
@@ -196,7 +196,23 @@ module Eco
|
|
|
196
196
|
ext = File.extname(file)
|
|
197
197
|
encoding ||= Eco::API::Common::Session::FileManager.encoding(file)
|
|
198
198
|
encoding = (encoding != "utf-8")? "#{encoding}|utf-8": encoding
|
|
199
|
-
|
|
199
|
+
read_with_tolerance(file, encoding: encoding)
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def read_with_tolerance(file, encoding:)
|
|
203
|
+
if content = File.read(file, encoding: encoding)
|
|
204
|
+
tolerance = 5
|
|
205
|
+
content.scrub do |bytes|
|
|
206
|
+
replacement = '<' + bytes.unpack('H*')[0] + '>'
|
|
207
|
+
if tolerance <= 0
|
|
208
|
+
logger.error("There were more than 5 encoding errors in the file '#{file}'.")
|
|
209
|
+
return content
|
|
210
|
+
else
|
|
211
|
+
logger.error("Encoding problem in file '#{file}': '#{replacement}'.")
|
|
212
|
+
replacement
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
end
|
|
200
216
|
end
|
|
201
217
|
|
|
202
218
|
def fatal(msg)
|
|
@@ -32,8 +32,10 @@ module Eco
|
|
|
32
32
|
# @param path [String] the target file to be uploaded
|
|
33
33
|
# @return [String] S3 path to the uploaded `path` file
|
|
34
34
|
def upload_file(path)
|
|
35
|
-
File.
|
|
36
|
-
|
|
35
|
+
if File.exist?(path)
|
|
36
|
+
File.open(path, "rb") do |f|
|
|
37
|
+
upload(File.basename(path), f)
|
|
38
|
+
end
|
|
37
39
|
end
|
|
38
40
|
end
|
|
39
41
|
|
|
@@ -30,7 +30,7 @@ module Eco
|
|
|
30
30
|
people_load(modifier: modifier - [:newest, :file])
|
|
31
31
|
end
|
|
32
32
|
when modifier.include?(:api)
|
|
33
|
-
logger.info("Going to get all the people via API")
|
|
33
|
+
logger.info("Going to get all the people via API (load)")
|
|
34
34
|
|
|
35
35
|
start = Time.now
|
|
36
36
|
session.batch.get_people.tap do |people|
|
data/lib/eco/api/session.rb
CHANGED
|
@@ -17,6 +17,11 @@ module Eco
|
|
|
17
17
|
logger.debug("LINE COMMAND: #{$0} #{ARGV.join(" ")}")
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
+
# @retrun [String] the name of the current environment
|
|
21
|
+
def enviro_name
|
|
22
|
+
config.active_enviro
|
|
23
|
+
end
|
|
24
|
+
|
|
20
25
|
# @return [Eco::API::Session::Batch] provides helper to launch batch operations.
|
|
21
26
|
def batch
|
|
22
27
|
@batch ||= Batch.new(enviro)
|
|
@@ -2,6 +2,8 @@ class Eco::API::UseCases::DefaultCases::CleanUnknownTags < Eco::API::Common::Loa
|
|
|
2
2
|
name "clean-unknown-tags"
|
|
3
3
|
type :transform
|
|
4
4
|
|
|
5
|
+
UNKNOWN_LOCS_FILE = "unknown_locations.txt"
|
|
6
|
+
|
|
5
7
|
REGISTER_TAGS = [
|
|
6
8
|
"EVENT", "INJURY", "RISK", "CONTRACTOR", "PERMIT",
|
|
7
9
|
"AUDIT", "JSEA",
|
|
@@ -20,18 +22,52 @@ class Eco::API::UseCases::DefaultCases::CleanUnknownTags < Eco::API::Common::Loa
|
|
|
20
22
|
people.each do |person|
|
|
21
23
|
unknown_tags = person.filter_tags.select {|tag| !tag?(tag)}
|
|
22
24
|
person.filter_tags -= unknown_tags
|
|
25
|
+
unknown_tag!(*unknown_tags)
|
|
23
26
|
update.add(person)
|
|
24
27
|
end
|
|
25
28
|
end
|
|
26
29
|
|
|
27
30
|
private
|
|
28
31
|
|
|
32
|
+
def unknown_tag!(*tags)
|
|
33
|
+
tags.each do |value|
|
|
34
|
+
unless unknown.include?(value)
|
|
35
|
+
unknown << value
|
|
36
|
+
File.open(unknown_tags_file, 'a') do |fd|
|
|
37
|
+
fd.write("#{value}\n")
|
|
38
|
+
end
|
|
39
|
+
puts "Custom detected: '#{value}'"
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def register_tags
|
|
45
|
+
REGISTER_TAGS
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def unknown
|
|
49
|
+
@unknown ||= []
|
|
50
|
+
end
|
|
51
|
+
|
|
29
52
|
def tag?(value)
|
|
30
|
-
tagtree.tag?(value)
|
|
53
|
+
tagtree.tag?(value).yield_self do |is_loc|
|
|
54
|
+
next true if is_loc
|
|
55
|
+
if clean_register_tags?
|
|
56
|
+
register_tags.any? {|reg| value == reg}
|
|
57
|
+
end
|
|
58
|
+
end
|
|
31
59
|
end
|
|
32
60
|
|
|
33
61
|
def tagtree
|
|
34
62
|
@tagtree ||= ASSETS.config.tagtree
|
|
35
63
|
end
|
|
36
64
|
|
|
65
|
+
def unknown_tags_file
|
|
66
|
+
File.join(".","#{session.enviro_name}_#{UNKNOWN_LOCS_FILE}")
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def clean_register_tags?
|
|
70
|
+
!options.dig(:usecase, :include_register_tags)
|
|
71
|
+
end
|
|
72
|
+
|
|
37
73
|
end
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
class Eco::API::UseCases::DefaultCases::Samples::Sftp < Eco::API::Common::Loaders::UseCase
|
|
2
|
+
name "sftp-sample"
|
|
3
|
+
type :other
|
|
4
|
+
|
|
5
|
+
attr_reader :session
|
|
6
|
+
|
|
7
|
+
def main(session, options, usecase)
|
|
8
|
+
@session = session
|
|
9
|
+
options[:end_get] = false
|
|
10
|
+
raise "The SFTP is not configured" unless session.sftp?
|
|
11
|
+
case options.dig(:sftp, :command)
|
|
12
|
+
when :list
|
|
13
|
+
list_folder
|
|
14
|
+
when :get
|
|
15
|
+
get_files
|
|
16
|
+
when :get_last
|
|
17
|
+
get_last
|
|
18
|
+
when :archive
|
|
19
|
+
archive_files
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def file_pattern
|
|
26
|
+
raise "You should redefine the file_pattern function as a RegEx expression that matches the target remote file"
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Ex: "/IN/Personnel"
|
|
30
|
+
def remote_subfolder
|
|
31
|
+
raise "You should redefine remote_subfolder as the folder where the target file sits"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def archive_subfolder
|
|
35
|
+
"Archive"
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def with_remote_files
|
|
39
|
+
sftp.files(remote_folder, pattern: file_pattern).each do |remote_file|
|
|
40
|
+
yield(remote_file) if block_given?
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def list_folder
|
|
45
|
+
puts "Listing remote folder: '#{remote_folder}':"
|
|
46
|
+
with_remote_files {|file| puts file.longname}
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def get_files
|
|
50
|
+
with_remote_files.tap do |files|
|
|
51
|
+
unless files.empty?
|
|
52
|
+
file_names = files.map {|file| to_remote_path(file.name)}
|
|
53
|
+
puts "Getting the following files:"
|
|
54
|
+
puts file_names
|
|
55
|
+
sftp.download(file_names, local_folder: local_folder)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def get_last
|
|
61
|
+
with_remote_files.last.tap do |file|
|
|
62
|
+
if file
|
|
63
|
+
file_name = to_remote_path(file.name)
|
|
64
|
+
puts "Getting the following file: #{file_name}"
|
|
65
|
+
sftp.download(file_name, local_folder: local_folder)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def archive_files
|
|
71
|
+
with_remote_files do |file|
|
|
72
|
+
source = to_remote_path(file.name) # should probably be file.longname
|
|
73
|
+
dest = to_remote_path("#{archive_subfolder}/#{file.name}")
|
|
74
|
+
move_file(source, dest)
|
|
75
|
+
end.tap do |files|
|
|
76
|
+
puts "Moved the file(s) to the #{archive_subfolder} folder" unless files.empty?
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def move_file(source, dest)
|
|
81
|
+
sftp.move(source, dest, 0x0001) do |response|
|
|
82
|
+
if response.ok?
|
|
83
|
+
puts "#{source}\n -to-> #{dest}"
|
|
84
|
+
else
|
|
85
|
+
puts "Could not move file #{source}"
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def to_remote_path(file)
|
|
91
|
+
remote_folder + "/" + file
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def local_folder
|
|
95
|
+
"."
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def remote_folder
|
|
99
|
+
@remote_folder ||= sftp_config.remote_folder + remote_subfolder
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def sftp_config
|
|
103
|
+
session.config.sftp
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def sftp
|
|
107
|
+
session.sftp
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -20,10 +20,11 @@ class Eco::API::UseCases::OozeSamples::OozeBaseCase < Eco::API::Common::Loaders:
|
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
# Write here your script
|
|
23
|
-
def process_ooze(
|
|
23
|
+
def process_ooze(ooz = target)
|
|
24
|
+
ooz ||= ooze
|
|
24
25
|
raise "You need to inherit from this class ('#{self.class}') and call super with a block" unless block_given?
|
|
25
|
-
yield(
|
|
26
|
-
update_ooze(
|
|
26
|
+
yield(ooz)
|
|
27
|
+
update_ooze(ooz)
|
|
27
28
|
end
|
|
28
29
|
|
|
29
30
|
protected
|
|
@@ -109,10 +110,12 @@ class Eco::API::UseCases::OozeSamples::OozeBaseCase < Eco::API::Common::Loaders:
|
|
|
109
110
|
exit_error "Stage '#{id_name}' doesn't exist in ooze '#{ooze_id}'"
|
|
110
111
|
end
|
|
111
112
|
|
|
113
|
+
# It fill update the ooze only if it's dirty (it carries changes)
|
|
114
|
+
# @return [Boolean, Response] `false` if there was not request against the server, `Response` otherwise
|
|
112
115
|
def update_ooze(ooze = target)
|
|
113
116
|
if !options[:simulate]
|
|
114
|
-
return unless dirty?(ooze)
|
|
115
|
-
|
|
117
|
+
return false unless dirty?(ooze)
|
|
118
|
+
|
|
116
119
|
ooze.validate.tap do |validation|
|
|
117
120
|
raise validation if validation.is_a?(String)
|
|
118
121
|
end
|
|
@@ -133,6 +136,7 @@ class Eco::API::UseCases::OozeSamples::OozeBaseCase < Eco::API::Common::Loaders:
|
|
|
133
136
|
end
|
|
134
137
|
backup_patch!(ooze)
|
|
135
138
|
exit(0) if dirty?(ooze) && dry_count > DRY_COUNT
|
|
139
|
+
false
|
|
136
140
|
end
|
|
137
141
|
end
|
|
138
142
|
|
|
@@ -39,9 +39,9 @@ class Eco::API::UseCases::OozeSamples::OozeRunBaseCase < Eco::API::UseCases::Ooz
|
|
|
39
39
|
options.dig(:source, :stage_id)
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
-
def update_ooze
|
|
42
|
+
def update_ooze(ooz = target)
|
|
43
43
|
prompt_to_confirm!
|
|
44
|
-
super
|
|
44
|
+
super(ooz)
|
|
45
45
|
end
|
|
46
46
|
|
|
47
47
|
def exit_if_no_changes!
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
# Use case to update a register
|
|
2
|
+
# @note
|
|
3
|
+
# - You can define methods `filters` and `search` to change the target entries of the register
|
|
4
|
+
# - You need to define the `process_ooze` method
|
|
5
|
+
# This case expects `options[:source][:register_id]`
|
|
6
|
+
class Eco::API::UseCases::OozeSamples::RegisterExportCase < Eco::API::Common::Loaders::UseCase
|
|
7
|
+
|
|
8
|
+
class << self
|
|
9
|
+
# @return [Integer] the number of pages to be processed in each batch
|
|
10
|
+
def batch_size(size = nil)
|
|
11
|
+
@batch_size ||= 25
|
|
12
|
+
return @batch_size unless size
|
|
13
|
+
@batch_size = size
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
include Eco::API::UseCases::OozeSamples::Helpers
|
|
18
|
+
|
|
19
|
+
name "register-export-case"
|
|
20
|
+
type :other
|
|
21
|
+
|
|
22
|
+
attr_reader :session, :options, :usecase
|
|
23
|
+
attr_reader :target
|
|
24
|
+
|
|
25
|
+
def main(session, options, usecase)
|
|
26
|
+
options[:end_get] = false
|
|
27
|
+
@session = session; @options = options; @usecase = usecase
|
|
28
|
+
@target = nil
|
|
29
|
+
raise "You need to inherit from this class ('#{self.class}') and call super with a block" unless block_given?
|
|
30
|
+
with_each_entry do |ooze|
|
|
31
|
+
process_ooze(ooze)
|
|
32
|
+
end
|
|
33
|
+
yield
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Write here your script
|
|
37
|
+
def process_ooze(ooze = target)
|
|
38
|
+
raise "You need to inherit from this class ('#{self.class}') and call super with a block" unless block_given?
|
|
39
|
+
yield(ooze)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def new_target(object)
|
|
45
|
+
@target = object
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def with_each_entry
|
|
49
|
+
batched_search_results do |page_results|
|
|
50
|
+
page_results.each do |page_result|
|
|
51
|
+
if ooz = build_full_ooze(page_result.id)
|
|
52
|
+
yield(ooz)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# It builds a full page model (not updatable)
|
|
59
|
+
# @note If it's a page instance with stages, there's where it will be handy
|
|
60
|
+
# @param ooze_id [String]
|
|
61
|
+
# @return [Ecoportal::API::V2::Page]
|
|
62
|
+
def build_full_ooze(ooze_id)
|
|
63
|
+
if page = ooze(ooze_id)
|
|
64
|
+
return page unless page.is_a?(Ecoportal::API::V2::Pages::PageStage)
|
|
65
|
+
secs_doc = page.sections.doc
|
|
66
|
+
flds_doc = page.components.doc
|
|
67
|
+
pending_stage_ids = page.stages.map(&:id) - [page.current_stage_id]
|
|
68
|
+
pending_stage_ids.each do |id|
|
|
69
|
+
if page = stage(id, ooze: page)
|
|
70
|
+
page.sections.doc.each do |sec_doc|
|
|
71
|
+
unless secs_doc.find {|sec| sec["id"] == sec_doc["id"]}
|
|
72
|
+
secs_doc << sec_doc
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
page.components.doc.each do |comp_doc|
|
|
76
|
+
unless flds_doc.find {|fld| fld["id"] == comp_doc["id"]}
|
|
77
|
+
flds_doc << comp_doc
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
Ecoportal::API::V2::Page.new(page.doc)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
#def update_oozes(batched_oozes = batch_queue)
|
|
87
|
+
# batched_oozes.each do |ooze|
|
|
88
|
+
# update_ooze(ooze)
|
|
89
|
+
# end
|
|
90
|
+
# batched_oozes.clear
|
|
91
|
+
#end
|
|
92
|
+
|
|
93
|
+
def batched_search_results
|
|
94
|
+
raise "Missing block. It yields in slices of #{self.class.batch_size} results" unless block_given?
|
|
95
|
+
results_preview
|
|
96
|
+
results = []
|
|
97
|
+
apiv2.registers.search(register_id, search_options) do |page_result|
|
|
98
|
+
results << page_result
|
|
99
|
+
if results.length >= self.class.batch_size
|
|
100
|
+
yield(results)
|
|
101
|
+
results = []
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
yield(results) unless results.empty?
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def ooze(ooze_id = nil, stage_id: nil)
|
|
108
|
+
return target unless ooze_id
|
|
109
|
+
apiv2.pages.get(ooze_id, stage_id: stage_id).tap do |ooze|
|
|
110
|
+
if ooze
|
|
111
|
+
new_target(ooze)
|
|
112
|
+
logger.info("Got #{object_reference(ooze)}")
|
|
113
|
+
else
|
|
114
|
+
exit_error "Could not get ooze '#{ooze_id}'"
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def stage(id_name = nil, ooze: target)
|
|
120
|
+
if ooze_id = ooze && ooze.id
|
|
121
|
+
exit_error "#{object_reference(ooze)} does not have stages!" unless ooze.stages?
|
|
122
|
+
else
|
|
123
|
+
exit_error "There's no target ooze to get retrieve stages from"
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
if stg = ooze.stages[id_name] || ooze.stages.get_by_name(id_name)
|
|
127
|
+
return ooze if ooze.respond_to?(:current_stage_id) && (ooze.current_stage_id == stg.id)
|
|
128
|
+
return apiv2.pages.get(ooze_id, stage_id: stg.id).tap do |stage|
|
|
129
|
+
if stage
|
|
130
|
+
new_target(stage)
|
|
131
|
+
logger.info("Got #{object_reference(stage)} from #{object_reference(ooze)}")
|
|
132
|
+
else
|
|
133
|
+
exit_error "Could not get stage '#{id_name}' in ooze '#{ooze_id}'"
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
exit_error "Stage '#{id_name}' doesn't exist in ooze '#{ooze_id}'"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def results_preview
|
|
141
|
+
apiv2.registers.search(register_id, search_options.merge(only_first: true)).tap do |search_results|
|
|
142
|
+
str_results = "Total target entries: #{search_results.total} (out of #{search_results.total_before_filtering})"
|
|
143
|
+
session.prompt_user("Do you want to proceed (y/N):", explanation: str_results, default: "N", timeout: 10) do |res|
|
|
144
|
+
unless res.upcase.start_with?("Y")
|
|
145
|
+
puts "..."
|
|
146
|
+
logger.info "Aborting script..."
|
|
147
|
+
exit(0)
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def search_options
|
|
154
|
+
@search_options ||= {}.tap do |opts|
|
|
155
|
+
opts.merge!(sort: "created_at")
|
|
156
|
+
opts.merge!(dir: "asc")
|
|
157
|
+
opts.merge!(query: conf_search) if conf_search
|
|
158
|
+
opts.merge!(filters: conf_filters)
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def conf_filters
|
|
163
|
+
return filters if self.respond_to?(:filters)
|
|
164
|
+
[]
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def conf_search
|
|
168
|
+
return search if self.respond_to?(:search)
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def register_id
|
|
172
|
+
options.dig(:source, :register_id)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def apiv2
|
|
176
|
+
@apiv2 ||= session.api(version: :oozes)
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def exit_error(msg)
|
|
180
|
+
logger.error(msg)
|
|
181
|
+
exit(1)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
end
|
|
@@ -17,12 +17,30 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
|
17
17
|
name "register-update-case"
|
|
18
18
|
type :other
|
|
19
19
|
|
|
20
|
+
attr_reader :retrieved_oozes, :non_retrieved_oozes
|
|
21
|
+
attr_reader :total_search_oozes, :dupped_search_oozes
|
|
22
|
+
attr_reader :ooze_result_ids, :updated_oozes, :failed_update_oozes
|
|
23
|
+
|
|
20
24
|
def main(session, options, usecase, &block)
|
|
25
|
+
@retrieved_oozes = 0
|
|
26
|
+
@non_retrieved_oozes = 0
|
|
27
|
+
@dupped_search_oozes = 0
|
|
28
|
+
@ooze_result_ids = {}
|
|
29
|
+
@updated_oozes = 0
|
|
30
|
+
@failed_update_oozes = 0
|
|
31
|
+
|
|
21
32
|
super(session, options, usecase) do
|
|
22
33
|
with_each_entry do |ooze|
|
|
23
34
|
process_ooze(&block)
|
|
24
35
|
end
|
|
25
36
|
end
|
|
37
|
+
msg = "Run end:\n"
|
|
38
|
+
msg += " • Search results: #{total_search_oozes}\n"
|
|
39
|
+
msg += " • Duplicated search results #{dupped_search_oozes}\n"
|
|
40
|
+
msg += " • Retrieved a total of #{retrieved_oozes}\n"
|
|
41
|
+
msg += " • Could not get #{non_retrieved_oozes} oozes.\n"
|
|
42
|
+
msg += " • Updated #{updated_oozes} oozes (failed update on #{failed_update_oozes} oozes).\n"
|
|
43
|
+
logger.info(msg)
|
|
26
44
|
end
|
|
27
45
|
|
|
28
46
|
def process_ooze(ooze = target)
|
|
@@ -47,7 +65,14 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
|
47
65
|
def with_each_entry
|
|
48
66
|
batched_search_results do |page_results|
|
|
49
67
|
page_results.each do |page_result|
|
|
68
|
+
if ooze_result_ids[page_result.id]
|
|
69
|
+
@dupped_search_oozes += 1
|
|
70
|
+
else
|
|
71
|
+
ooze_result_ids[page_result.id] = true
|
|
72
|
+
end
|
|
73
|
+
|
|
50
74
|
if pending = queue_shift(page_result.id)
|
|
75
|
+
|
|
51
76
|
if dirty?(pending)
|
|
52
77
|
msg = "Inconsistent search results. "
|
|
53
78
|
msg << "Launching update on '#{object_reference(pending)}' to be able to queue it back"
|
|
@@ -56,7 +81,11 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
|
56
81
|
end
|
|
57
82
|
end
|
|
58
83
|
if ooz = ooze(page_result.id)
|
|
84
|
+
@retrieved_oozes += 1
|
|
59
85
|
yield(ooz)
|
|
86
|
+
else
|
|
87
|
+
@non_retrieved_oozes += 1
|
|
88
|
+
console.warn "Could not get page #{page_result.id}"
|
|
60
89
|
end
|
|
61
90
|
end
|
|
62
91
|
update_oozes
|
|
@@ -113,7 +142,15 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
|
113
142
|
|
|
114
143
|
def update_oozes(batched_oozes = batch_queue)
|
|
115
144
|
batched_oozes.each do |ooze|
|
|
116
|
-
update_ooze(ooze)
|
|
145
|
+
update_ooze(ooze).tap do |result|
|
|
146
|
+
if result.is_a?(Ecoportal::API::Common::Response)
|
|
147
|
+
if result.success?
|
|
148
|
+
@updated_oozes += 1
|
|
149
|
+
else
|
|
150
|
+
@failed_update_oozes +=1
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|
|
117
154
|
end
|
|
118
155
|
batched_oozes.clear
|
|
119
156
|
end
|
|
@@ -127,6 +164,7 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
|
127
164
|
|
|
128
165
|
def results_preview
|
|
129
166
|
apiv2.registers.search(register_id, search_options.merge(only_first: true)).tap do |search_results|
|
|
167
|
+
@total_search_oozes = search_results.total
|
|
130
168
|
str_results = "Total target entries: #{search_results.total} (out of #{search_results.total_before_filtering})"
|
|
131
169
|
session.prompt_user("Do you want to proceed (y/N):", explanation: str_results, default: "N", timeout: 10) do |res|
|
|
132
170
|
unless res.upcase.start_with?("Y")
|
|
@@ -10,6 +10,7 @@ end
|
|
|
10
10
|
|
|
11
11
|
require_relative 'ooze_samples/helpers'
|
|
12
12
|
require_relative 'ooze_samples/ooze_base_case'
|
|
13
|
+
require_relative 'ooze_samples/register_export_case'
|
|
13
14
|
require_relative 'ooze_samples/ooze_run_base_case'
|
|
14
15
|
require_relative 'ooze_samples/ooze_update_case'
|
|
15
16
|
require_relative 'ooze_samples/ooze_from_doc_case'
|
|
@@ -90,6 +90,9 @@ ASSETS.cli.config do |cnf|
|
|
|
90
90
|
desc = "Cleans from filter_tags those tags that are not present in the tagtree (as per tagtree.json file)."
|
|
91
91
|
desc += " It will preserve standard register tags of most common registers (i.e. EVENT, RISK)."
|
|
92
92
|
cases.add("-clean-unknown-tags", :transform, desc, case_name: "clean-unknown-tags")
|
|
93
|
+
.add_option("-include-register-tags", "Will also try to remove register tags") do |options|
|
|
94
|
+
options.deep_merge!(usecase: {include_register_tags: true})
|
|
95
|
+
end
|
|
93
96
|
|
|
94
97
|
desc = "Removes the landing page or sets it to -page-id"
|
|
95
98
|
cases.add("-reset-landing-page", :transform, desc, case_name: "reset-landing-page")
|
|
@@ -3,6 +3,8 @@ ASSETS.cli.config do |config|
|
|
|
3
3
|
|
|
4
4
|
io = nil
|
|
5
5
|
rescued = false
|
|
6
|
+
cases_with_input = nil
|
|
7
|
+
cases_with_output = nil
|
|
6
8
|
|
|
7
9
|
# default rescue
|
|
8
10
|
wf.rescue do |exception, io|
|
|
@@ -165,9 +167,16 @@ ASSETS.cli.config do |config|
|
|
|
165
167
|
wf.on(:end) do |wf_end, io|
|
|
166
168
|
get_people = io.options.dig(:people, :get)
|
|
167
169
|
partial_update = get_people && get_people.dig(:type) == :partial
|
|
170
|
+
|
|
168
171
|
unless !io.options[:end_get] || io.options[:dry_run] || partial_update
|
|
169
|
-
|
|
170
|
-
|
|
172
|
+
people_update_cases = config.usecases.active(io: io).any? do |usecase, data|
|
|
173
|
+
[:transform, :sync].any? {|type| usecase.type == type}
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
if !people_update_cases
|
|
177
|
+
# Prevent getting people when there were no use cases that used them
|
|
178
|
+
io.session.logger.info("Won't be recaching people, as there haven't been any targetted updates")
|
|
179
|
+
elsif !io.people
|
|
171
180
|
people = io.session.micro.people_cache
|
|
172
181
|
io = io.new(people: people)
|
|
173
182
|
end
|
data/lib/eco/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: eco-helpers
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.0.
|
|
4
|
+
version: 2.0.48
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Oscar Segura
|
|
@@ -136,7 +136,7 @@ dependencies:
|
|
|
136
136
|
requirements:
|
|
137
137
|
- - ">="
|
|
138
138
|
- !ruby/object:Gem::Version
|
|
139
|
-
version: 0.8.
|
|
139
|
+
version: 0.8.25
|
|
140
140
|
- - "<"
|
|
141
141
|
- !ruby/object:Gem::Version
|
|
142
142
|
version: '0.9'
|
|
@@ -146,7 +146,7 @@ dependencies:
|
|
|
146
146
|
requirements:
|
|
147
147
|
- - ">="
|
|
148
148
|
- !ruby/object:Gem::Version
|
|
149
|
-
version: 0.8.
|
|
149
|
+
version: 0.8.25
|
|
150
150
|
- - "<"
|
|
151
151
|
- !ruby/object:Gem::Version
|
|
152
152
|
version: '0.9'
|
|
@@ -546,6 +546,8 @@ files:
|
|
|
546
546
|
- lib/eco/api/usecases/default_cases/remove_account_trans_case.rb
|
|
547
547
|
- lib/eco/api/usecases/default_cases/reset_landing_page_case.rb
|
|
548
548
|
- lib/eco/api/usecases/default_cases/restore_db_case.rb
|
|
549
|
+
- lib/eco/api/usecases/default_cases/samples.rb
|
|
550
|
+
- lib/eco/api/usecases/default_cases/samples/sftp_case.rb
|
|
549
551
|
- lib/eco/api/usecases/default_cases/set_default_tag_case.rb
|
|
550
552
|
- lib/eco/api/usecases/default_cases/set_supervisor_case.rb
|
|
551
553
|
- lib/eco/api/usecases/default_cases/supers_cyclic_identify_case.rb
|
|
@@ -564,6 +566,7 @@ files:
|
|
|
564
566
|
- lib/eco/api/usecases/ooze_samples/ooze_from_doc_case.rb
|
|
565
567
|
- lib/eco/api/usecases/ooze_samples/ooze_run_base_case.rb
|
|
566
568
|
- lib/eco/api/usecases/ooze_samples/ooze_update_case.rb
|
|
569
|
+
- lib/eco/api/usecases/ooze_samples/register_export_case.rb
|
|
567
570
|
- lib/eco/api/usecases/ooze_samples/register_update_case.rb
|
|
568
571
|
- lib/eco/api/usecases/ooze_samples/target_oozes_update_case.rb
|
|
569
572
|
- lib/eco/api/usecases/use_case.rb
|