eco-helpers 3.0.23 → 3.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +39 -2
- data/eco-helpers.gemspec +2 -2
- data/lib/eco/api/common/people/person_entry_attribute_mapper.rb +1 -1
- data/lib/eco/api/common/session/sftp.rb +1 -1
- data/lib/eco/api/microcases/person_update.rb +2 -2
- data/lib/eco/api/session/config/tagtree.rb +6 -5
- data/lib/eco/api/session/config.rb +1 -0
- data/lib/eco/api/usecases/cli/dsl.rb +3 -5
- data/lib/eco/api/usecases/default/locations/tagtree_extract_case.rb +1 -1
- data/lib/eco/api/usecases/default/people/amend/clean_unknown_tags_case.rb +2 -5
- data/lib/eco/api/usecases/default/people/amend/restore_db_case.rb +1 -1
- data/lib/eco/api/usecases/default/people/migrate/remap_tags_case.rb +4 -4
- data/lib/eco/api/usecases/default_cases/samples/sftp.rb +3 -2
- data/lib/eco/api/usecases/graphql/helpers/base/error_handling.rb +1 -65
- data/lib/eco/api/usecases/graphql/helpers/location/base.rb +2 -0
- data/lib/eco/api/usecases/graphql/helpers/location/command/diffs/stages.rb +2 -2
- data/lib/eco/api/usecases/graphql/helpers/location/command/result.rb +1 -1
- data/lib/eco/api/usecases/graphql/helpers/location/command/results.rb +3 -3
- data/lib/eco/api/usecases/graphql/helpers/location/tags_remap.rb +1 -1
- data/lib/eco/api/usecases/graphql/utils/sftp.rb +1 -129
- data/lib/eco/api/usecases/lib/error_handling.rb +70 -0
- data/lib/eco/api/usecases/lib/file_pattern.rb +37 -0
- data/lib/eco/api/usecases/lib/sftp.rb +173 -0
- data/lib/eco/api/usecases/lib.rb +12 -0
- data/lib/eco/api/usecases/ooze_samples/helpers/rescuable.rb +1 -1
- data/lib/eco/api/usecases/ooze_samples/helpers_migration/typed_fields_pairing.rb +4 -4
- data/lib/eco/api/usecases/ooze_samples/register_update_case.rb +8 -8
- data/lib/eco/api/usecases/samples/drivers/sftp_sample.rb +8 -95
- data/lib/eco/api/usecases/samples/drivers/url_pull_sample.rb +0 -1
- data/lib/eco/api/usecases/samples.rb +0 -1
- data/lib/eco/api/usecases.rb +1 -0
- data/lib/eco/data/locations/node_base/tag_validations.rb +2 -2
- data/lib/eco/data/locations/node_base/treeify.rb +17 -17
- data/lib/eco/data/locations/node_diff/nodes_diff.rb +8 -8
- data/lib/eco/data/locations/node_level/cleaner.rb +14 -9
- data/lib/eco/version.rb +1 -1
- metadata +10 -6
@@ -0,0 +1,173 @@
|
|
1
|
+
module Eco::API::UseCases::Lib
|
2
|
+
module Sftp
|
3
|
+
class UnspecifiedRemoteFolder < ArgumentError; end
|
4
|
+
|
5
|
+
include Eco::Language::AuxiliarLogger
|
6
|
+
|
7
|
+
attr_reader :session, :options
|
8
|
+
|
9
|
+
private
|
10
|
+
|
11
|
+
def upload(local_file, subfolder: remote_subfolder, gid: sftp_group_id)
|
12
|
+
sftp_session.upload(
|
13
|
+
local_file,
|
14
|
+
remote_folder: remote_folder(subfolder),
|
15
|
+
gid: gid
|
16
|
+
)
|
17
|
+
end
|
18
|
+
alias_method :sftp_upload, :upload
|
19
|
+
|
20
|
+
def sftp_group_id
|
21
|
+
if self.class.const_defined?(:SFTP_GROUP)
|
22
|
+
self.class.const_get(:SFTP_GROUP)
|
23
|
+
elsif (group_id = options.dig(:sftp, :group))
|
24
|
+
group_id
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def sftp_download_files(
|
29
|
+
subfolder: remote_subfolder,
|
30
|
+
pattern: nil,
|
31
|
+
local_folder: self.local_folder,
|
32
|
+
&block
|
33
|
+
)
|
34
|
+
remote_files = with_remote_files(subfolder: subfolder, pattern: pattern)
|
35
|
+
return [] if remote_files.empty?
|
36
|
+
|
37
|
+
file_names = remote_files.map {|file| to_remote_path(file.name, subfolder: subfolder)}
|
38
|
+
|
39
|
+
log(:info) {
|
40
|
+
msg = "Getting the following files into the local folder '#{local_folder}':\n"
|
41
|
+
msg << ' * '
|
42
|
+
msg << file_names.join("\n * ")
|
43
|
+
msg
|
44
|
+
}
|
45
|
+
|
46
|
+
sftp.download(file_names, local_folder: local_folder, &block)
|
47
|
+
end
|
48
|
+
|
49
|
+
def sftp_move_file(source, dest)
|
50
|
+
sftp.move(source, dest, 0x0001) do |response|
|
51
|
+
if response.ok?
|
52
|
+
log(:info) { "#{source}\n -to-> #{dest}" }
|
53
|
+
else
|
54
|
+
log(:info) { "Could not move file #{source}" }
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def sftp_archive_file!(filename, folder: 'Archive')
|
60
|
+
basename = File.basename(filename)
|
61
|
+
remote_file = to_remote_path(basename)
|
62
|
+
archive_folder = to_remote_path(folder)
|
63
|
+
archived_file = "#{archive_folder}/#{basename}"
|
64
|
+
|
65
|
+
sftp_move_file(remote_file, archived_file)
|
66
|
+
end
|
67
|
+
|
68
|
+
def remove_local_n_remote_file(file)
|
69
|
+
full_path = File.expand_path(file)
|
70
|
+
filename = File.basename(full_path)
|
71
|
+
|
72
|
+
if File.exist?(full_path)
|
73
|
+
File.delete(full_path).tap do
|
74
|
+
log(:info) { "Deleted file '#{full_path}'" }
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
sftp_archive_file!(filename)
|
79
|
+
end
|
80
|
+
|
81
|
+
def ensure_remote_empty(subfolder: remote_subfolder, pattern: nil)
|
82
|
+
files = with_remote_files(subfolder: subfolder, pattern: pattern)
|
83
|
+
return if files.empty?
|
84
|
+
|
85
|
+
msg = "There are still files in the remote folder that will be deleted: '#{subfolder}':\n"
|
86
|
+
msg << ' * '
|
87
|
+
str = files.map(&:longname).join("\n * ")
|
88
|
+
str << "\n"
|
89
|
+
msg << str
|
90
|
+
|
91
|
+
session.prompt_user(
|
92
|
+
'Do you want to proceed to delete? (Y/n):',
|
93
|
+
explanation: msg,
|
94
|
+
default: 'Y',
|
95
|
+
timeout: 3
|
96
|
+
) do |response|
|
97
|
+
next unless response.upcase.start_with?('Y')
|
98
|
+
|
99
|
+
files.each do |file|
|
100
|
+
remote_full_path = to_remote_path(file.name, subfolder: subfolder)
|
101
|
+
res = sftp_session.remove(remote_full_path)
|
102
|
+
log(:info) {
|
103
|
+
"Deleted remote file: '#{remote_full_path}' (#{res})"
|
104
|
+
}
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def to_remote_path(file, subfolder: remote_subfolder)
|
110
|
+
[remote_folder(subfolder), file].join('/')
|
111
|
+
end
|
112
|
+
|
113
|
+
# `remote_target_folder` overrides `sftp_config.remote_folder` as well as `remote_subfolder`
|
114
|
+
# `remote_folder` overrides `sftp_config.remote_folder` but NOT `remote_subfolder`
|
115
|
+
def remote_folder(subfolder = remote_subfolder)
|
116
|
+
rm_tf = options.dig(:sftp, :remote_target_folder)
|
117
|
+
return rm_tf if rm_tf
|
118
|
+
|
119
|
+
rm_fd = options.dig(:sftp, :remote_folder) || sftp_config.remote_folder
|
120
|
+
[rm_fd, subfolder].compact.join('/')
|
121
|
+
end
|
122
|
+
|
123
|
+
def remote_subfolder(required: true)
|
124
|
+
remote_subdir = options.dig(:sftp, :remote_subfolder)
|
125
|
+
remote_subdir ||= self.class::REMOTE_FOLDER if self.class.const_defined?(:REMOTE_FOLDER)
|
126
|
+
remote_subdir ||= self.class::REMOTE_SUBFOLDER if self.class.const_defined?(:REMOTE_SUBFOLDER)
|
127
|
+
return remote_subdir if remote_subdir
|
128
|
+
return unless required
|
129
|
+
|
130
|
+
msg = "(#{self.class}) You should redefine #remote_subfolder or REMOTE_FOLDER"
|
131
|
+
msg << 'as the folder where the target file sits. Ex: IN/Personnel'
|
132
|
+
raise UnspecifiedRemoteFolder, msg
|
133
|
+
end
|
134
|
+
|
135
|
+
def local_folder
|
136
|
+
if (local_dir = options.dig(:sftp, :local_folder))
|
137
|
+
local_dir
|
138
|
+
elsif self.class.const_defined?(:LOCAL_FOLDER)
|
139
|
+
self.class::LOCAL_FOLDER
|
140
|
+
else
|
141
|
+
'.'
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
# @param subfolder [String] the subfolder.
|
146
|
+
def with_remote_files(subfolder: remote_subfolder, pattern: nil)
|
147
|
+
sftp.files(remote_folder(subfolder), pattern: pattern).each do |remote_file|
|
148
|
+
yield(remote_file) if block_given?
|
149
|
+
end
|
150
|
+
rescue ArgumentError
|
151
|
+
raise
|
152
|
+
rescue ::Net::SFTP::StatusException => err
|
153
|
+
log(:error) {
|
154
|
+
msg = "(#{self.class}) There was an error trying to access "
|
155
|
+
msg << "the remote folder '#{subfolder}': #{err}"
|
156
|
+
msg
|
157
|
+
}
|
158
|
+
[]
|
159
|
+
end
|
160
|
+
|
161
|
+
def sftp_config
|
162
|
+
session.config.sftp
|
163
|
+
end
|
164
|
+
|
165
|
+
def sftp_session
|
166
|
+
sftp.sftp_session
|
167
|
+
end
|
168
|
+
|
169
|
+
def sftp
|
170
|
+
session.sftp
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
@@ -46,7 +46,7 @@ class Eco::API::UseCases::OozeSamples
|
|
46
46
|
end
|
47
47
|
return nil if src_filtered.empty?
|
48
48
|
msg << src_filtered.map do |src_fld|
|
49
|
-
"
|
49
|
+
" * #{object_reference(src_fld)}"
|
50
50
|
end.join("\n")
|
51
51
|
end
|
52
52
|
|
@@ -60,7 +60,7 @@ class Eco::API::UseCases::OozeSamples
|
|
60
60
|
end
|
61
61
|
return nil if dst_filtered.empty?
|
62
62
|
msg << dst_filtered.map do |dst_fld|
|
63
|
-
"
|
63
|
+
" * #{object_reference(dst_fld)}"
|
64
64
|
end.join("\n")
|
65
65
|
end
|
66
66
|
|
@@ -76,9 +76,9 @@ class Eco::API::UseCases::OozeSamples
|
|
76
76
|
dst_exclude_ximport && ximport?(dst_fld)
|
77
77
|
end
|
78
78
|
next if dst_filtered.empty?
|
79
|
-
msg_flds << "
|
79
|
+
msg_flds << " * #{object_reference(src_fld)}\n"
|
80
80
|
msg_flds << dst_filtered.map do |dst_fld|
|
81
|
-
"
|
81
|
+
" * #{object_reference(dst_fld)}"
|
82
82
|
end.compact.join("\n")
|
83
83
|
end
|
84
84
|
end
|
@@ -108,8 +108,8 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
108
108
|
msg = "Something is wrong. Native case 'RegisterUpdateCase' is "
|
109
109
|
if (elem != object) && dirty?(elem)
|
110
110
|
msg << "trying to queue different objects with same page id:\n"
|
111
|
-
msg << "
|
112
|
-
msg << "
|
111
|
+
msg << " * already queued (changes will go): #{object_reference(elem)}\n"
|
112
|
+
msg << " * tried to queue (lost changes): #{object_reference(object)}"
|
113
113
|
log(:warn) { msg }
|
114
114
|
end
|
115
115
|
else
|
@@ -234,13 +234,13 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
|
|
234
234
|
def kpis_message
|
235
235
|
msg = []
|
236
236
|
msg << "Run end:"
|
237
|
-
msg << "
|
238
|
-
msg << "
|
239
|
-
msg << "
|
240
|
-
msg << "
|
241
|
-
msg << "
|
237
|
+
msg << " * Search results: #{total_search_oozes}"
|
238
|
+
msg << " * Duplicated search results #{dupped_search_oozes}"
|
239
|
+
msg << " * Retrieved a total of #{retrieved_oozes}"
|
240
|
+
msg << " * Could not get #{non_retrieved_oozes} oozes."
|
241
|
+
msg << " * Updated #{updated_oozes} oozes (attempted: #{attempted_ooze_updates})."
|
242
242
|
msg << " - Failed update on #{failed_update_oozes} oozes."
|
243
|
-
msg << "
|
243
|
+
msg << " * Created #{created_oozes} oozes (out of: #{ooze_create_attempts})."
|
244
244
|
msg.join("\n")
|
245
245
|
end
|
246
246
|
|
@@ -1,13 +1,12 @@
|
|
1
1
|
# rubocop:disable Naming/AccessorMethodName
|
2
2
|
class Eco::API::UseCases::Samples::Drivers::Sftp < Eco::API::Common::Loaders::UseCase
|
3
|
-
class WrongConst < ArgumentError; end
|
4
|
-
class MissRemoteFolder < ArgumentError; end
|
5
|
-
|
6
3
|
require_relative 'cli/sftp_cli'
|
4
|
+
|
7
5
|
name 'sftp'
|
8
6
|
type :other
|
9
7
|
|
10
|
-
|
8
|
+
include Eco::API::UseCases::Lib::Sftp
|
9
|
+
include Eco::API::UseCases::Lib::FilePattern
|
11
10
|
|
12
11
|
def main(session, options, _usecase)
|
13
12
|
options[:end_get] = false
|
@@ -27,81 +26,13 @@ class Eco::API::UseCases::Samples::Drivers::Sftp < Eco::API::Common::Loaders::Us
|
|
27
26
|
|
28
27
|
private
|
29
28
|
|
30
|
-
# Can't pass this via CLI option, as it breaks the regular expression
|
31
|
-
def file_pattern(require: true)
|
32
|
-
fpc = file_pattern_const
|
33
|
-
return fpc if fpc
|
34
|
-
return unless require
|
35
|
-
|
36
|
-
msg = "(#{self.class}) You should redefine the file_pattern function "
|
37
|
-
msg << 'as a RegEx expression that matches the target remote file'
|
38
|
-
raise WrongConst, msg
|
39
|
-
end
|
40
|
-
|
41
|
-
def file_pattern_const
|
42
|
-
if (fpc = options.dig(:sftp, :file_pattern_const))
|
43
|
-
raise WrongConst, "(#{self.class}) Invalid file pattern const referral: #{fpc}" unless fpc.match(CONST_REFERRAL)
|
44
|
-
|
45
|
-
begin
|
46
|
-
self.eval(fpc)
|
47
|
-
rescue NameError
|
48
|
-
self.class.const_get(fpc)
|
49
|
-
end
|
50
|
-
end
|
51
|
-
rescue NameError
|
52
|
-
raise WrongConst, "(#{self.class}) Unknown constant: #{fpc}"
|
53
|
-
end
|
54
|
-
|
55
|
-
# Ex: '/IN/Personnel'
|
56
|
-
def remote_subfolder(require: true)
|
57
|
-
rm_sf = options.dig(:sftp, :remote_subfolder)
|
58
|
-
|
59
|
-
return rm_sf if rm_sf
|
60
|
-
return unless require
|
61
|
-
|
62
|
-
msg = "(#{self.class}) You should redefine remote_subfolder "
|
63
|
-
msg << 'as the folder where the target file sits. Ex: /IN/Personnel'
|
64
|
-
raise MissRemoteFolder, msg
|
65
|
-
end
|
66
|
-
|
67
|
-
# `remote_target_folder` overrides `sftp_config.remote_folder` as well as `remote_subfolder`
|
68
|
-
# `remote_folder` overrides `sftp_config.remote_folder` but NOT `remote_subfolder`
|
69
|
-
def remote_folder
|
70
|
-
rm_tf = options.dig(:sftp, :remote_target_folder)
|
71
|
-
rm_fd = options.dig(:sftp, :remote_folder) || sftp_config.remote_folder
|
72
|
-
rm_tf || [rm_fd, remote_subfolder].compact.join('/')
|
73
|
-
end
|
74
|
-
|
75
|
-
def to_remote_path(file)
|
76
|
-
[remote_folder, file].compact.join('/')
|
77
|
-
end
|
78
|
-
|
79
|
-
def local_folder
|
80
|
-
options.dig(:sftp, :local_folder) || '.'
|
81
|
-
end
|
82
|
-
|
83
|
-
def with_remote_files(folder: remote_folder, pattern: file_pattern)
|
84
|
-
sftp.files(folder, pattern: pattern).each do |remote_file|
|
85
|
-
yield(remote_file) if block_given?
|
86
|
-
end
|
87
|
-
rescue ArgumentError
|
88
|
-
raise
|
89
|
-
rescue ::Net::SFTP::StatusException => err
|
90
|
-
log(:error) {
|
91
|
-
msg = "(#{self.class}) There was an error trying to access "
|
92
|
-
msg << "the remote folder '#{remote_folder}': #{err}"
|
93
|
-
msg
|
94
|
-
}
|
95
|
-
[]
|
96
|
-
end
|
97
|
-
|
98
29
|
def list_folder
|
99
30
|
puts "Listing remote folder: '#{remote_folder}' (host: #{sftp.host}):"
|
100
|
-
with_remote_files {|file| puts file.longname}
|
31
|
+
with_remote_files(pattern: file_pattern) {|file| puts file.longname}
|
101
32
|
end
|
102
33
|
|
103
34
|
def get_files
|
104
|
-
with_remote_files.tap do |files|
|
35
|
+
with_remote_files(pattern: file_pattern).tap do |files|
|
105
36
|
next if files.empty?
|
106
37
|
|
107
38
|
file_names = files.map {|file| to_remote_path(file.name)}
|
@@ -114,7 +45,7 @@ class Eco::API::UseCases::Samples::Drivers::Sftp < Eco::API::Common::Loaders::Us
|
|
114
45
|
end
|
115
46
|
|
116
47
|
def get_last
|
117
|
-
with_remote_files.last.tap do |file|
|
48
|
+
with_remote_files(pattern: file_pattern).last.tap do |file|
|
118
49
|
next unless file
|
119
50
|
|
120
51
|
file_name = to_remote_path(file.name)
|
@@ -125,10 +56,10 @@ class Eco::API::UseCases::Samples::Drivers::Sftp < Eco::API::Common::Loaders::Us
|
|
125
56
|
end
|
126
57
|
|
127
58
|
def archive_files
|
128
|
-
with_remote_files do |file|
|
59
|
+
with_remote_files(pattern: file_pattern) do |file|
|
129
60
|
source = to_remote_path(file.name) # should probably be file.longname
|
130
61
|
dest = to_remote_path("#{archive_subfolder}/#{file.name}")
|
131
|
-
|
62
|
+
sftp_move_file(source, dest)
|
132
63
|
end.tap do |files|
|
133
64
|
next if files.empty?
|
134
65
|
|
@@ -136,27 +67,9 @@ class Eco::API::UseCases::Samples::Drivers::Sftp < Eco::API::Common::Loaders::Us
|
|
136
67
|
end
|
137
68
|
end
|
138
69
|
|
139
|
-
def move_file(source, dest)
|
140
|
-
sftp.move(source, dest, 0x0001) do |response|
|
141
|
-
if response.ok?
|
142
|
-
puts "#{source}\n -to-> #{dest}"
|
143
|
-
else
|
144
|
-
puts "Could not move file #{source}"
|
145
|
-
end
|
146
|
-
end
|
147
|
-
end
|
148
|
-
|
149
70
|
def archive_subfolder
|
150
71
|
'Archive'
|
151
72
|
end
|
152
|
-
|
153
|
-
def sftp_config
|
154
|
-
session.config.sftp
|
155
|
-
end
|
156
|
-
|
157
|
-
def sftp
|
158
|
-
session.sftp
|
159
|
-
end
|
160
73
|
end
|
161
74
|
|
162
75
|
# rubocop:enable Naming/AccessorMethodName
|
data/lib/eco/api/usecases.rb
CHANGED
@@ -176,6 +176,7 @@ require_relative 'usecases/use_case_chain'
|
|
176
176
|
require_relative 'usecases/base_io'
|
177
177
|
require_relative 'usecases/use_case_io'
|
178
178
|
require_relative 'usecases/cli'
|
179
|
+
require_relative 'usecases/lib'
|
179
180
|
require_relative 'usecases/graphql'
|
180
181
|
require_relative 'usecases/default'
|
181
182
|
require_relative 'usecases/samples'
|
@@ -17,11 +17,11 @@ module Eco::Data::Locations::NodeBase
|
|
17
17
|
if partial != str
|
18
18
|
invalid_chars = identify_invalid_characters(str)
|
19
19
|
log(:warn) {
|
20
|
-
"
|
20
|
+
"* #{ref}Invalid characters _#{invalid_chars}_ <<_removed_: '#{str}' :_converted_>> '#{result}'"
|
21
21
|
}
|
22
22
|
elsif blanks_x2
|
23
23
|
log(:warn) {
|
24
|
-
"
|
24
|
+
"* #{ref}Double blanks removed: '#{str}' :_converted_>> '#{result}'"
|
25
25
|
}
|
26
26
|
end
|
27
27
|
invalid_warned!(str)
|
@@ -141,8 +141,8 @@ module Eco::Data::Locations::NodeBase
|
|
141
141
|
msg = []
|
142
142
|
msg << "BUG in this library (open issue with maintainers)."
|
143
143
|
msg << "There were no skipped nodes nor missin referred parents, and yet:"
|
144
|
-
msg << "
|
145
|
-
msg << "
|
144
|
+
msg << " * the tree nodes count: #{done_ids.count} ..."
|
145
|
+
msg << " * doesn't match the original nodes count: #{nodes.count}"
|
146
146
|
raise msg.join("\n")
|
147
147
|
end
|
148
148
|
|
@@ -151,11 +151,11 @@ module Eco::Data::Locations::NodeBase
|
|
151
151
|
|
152
152
|
msg = []
|
153
153
|
msg << "There are #{unlinked_parent_ids.count} referred parent_id's NOT linked to the root:"
|
154
|
-
msg << "
|
155
|
-
msg << "
|
156
|
-
msg << "
|
157
|
-
msg << "
|
158
|
-
msg << "
|
154
|
+
msg << " * total_nodes: #{nodes.count}"
|
155
|
+
msg << " * tracked_nodes: #{tracked_nodes.count}"
|
156
|
+
msg << " * untracked_nodes: #{untracked_nodes.count}"
|
157
|
+
msg << " * unlinked_parents: #{unlinked_parent_ids.count}"
|
158
|
+
msg << " * skipped (repeated) nodes: #{skipped.count}#{str_skipped}" unless skipped.empty?
|
159
159
|
warns << msg.join("\n")
|
160
160
|
|
161
161
|
unlinked_nodes = nodes - skipped
|
@@ -182,16 +182,16 @@ module Eco::Data::Locations::NodeBase
|
|
182
182
|
|
183
183
|
msg = []
|
184
184
|
msg << "After treeifying via the unlinked_parents:"
|
185
|
-
msg << "
|
186
|
-
msg << "
|
187
|
-
msg << "
|
188
|
-
msg << "
|
189
|
-
msg << "
|
185
|
+
msg << " * total_nodes: #{nodes.count}"
|
186
|
+
msg << " * tracked_nodes: #{tracked_nodes.count}"
|
187
|
+
msg << " * untracked_nodes: #{untracked_nodes.count}"
|
188
|
+
msg << " * missing_parents: #{str_unlinked_parents}"
|
189
|
+
msg << " * skipped in this step: #{residual_skipped.count}#{str_skipped}" unless residual_skipped.empty?
|
190
190
|
warns << msg.join("\n")
|
191
191
|
end
|
192
192
|
|
193
193
|
str_skipped = (skipped.count < 15 ? " => #{skipped.map(&:id).join(', ')}" : '')
|
194
|
-
warns << "
|
194
|
+
warns << " * total skipped (repeated) nodes: #{skipped.count}!!#{str_skipped}" unless skipped.empty?
|
195
195
|
nil
|
196
196
|
end
|
197
197
|
|
@@ -205,7 +205,7 @@ module Eco::Data::Locations::NodeBase
|
|
205
205
|
parent_ids = parents.keys & node_ids
|
206
206
|
missing_parent_ids = parents.keys - parent_ids
|
207
207
|
missing_parents = parents.slice(*missing_parent_ids)
|
208
|
-
warns << "
|
208
|
+
warns << " * missing_parents: #{missing_parents.count}"
|
209
209
|
|
210
210
|
nil_parent_nodes = missing_parents.each_with_object([]) do |(_id, nds), mem|
|
211
211
|
nds.each {|node| node.parent_id = nil}
|
@@ -285,7 +285,7 @@ module Eco::Data::Locations::NodeBase
|
|
285
285
|
if (prev_parent || prev_level) && node_dup # && !done_node
|
286
286
|
str = "Integrity issue in Treeify. "
|
287
287
|
str << "A Node with tracked level or parent should be present in done_ids, but it isn't."
|
288
|
-
str << "\n
|
288
|
+
str << "\n * #{node_str}."
|
289
289
|
raise str
|
290
290
|
end
|
291
291
|
# From here on, do NOT expect `node_dup` where `node` has tracked `parent` or `level`.
|
@@ -300,7 +300,7 @@ module Eco::Data::Locations::NodeBase
|
|
300
300
|
if lev_dup && (multi_parent || !done_node || done_node.tracked_level != lev)
|
301
301
|
str = "Integrity issue in Treeify. "
|
302
302
|
str << "A Node with ID already in level_ids should have same tracked_level as current level."
|
303
|
-
str << "\n
|
303
|
+
str << "\n * #{node_str}."
|
304
304
|
raise str
|
305
305
|
end
|
306
306
|
# From here on, do NOT expect `lev_up` where there isn't `done_node` or it has different level or parent.
|
@@ -325,7 +325,7 @@ module Eco::Data::Locations::NodeBase
|
|
325
325
|
unless cyclic || double_up
|
326
326
|
str = "Integrity issue in Treeify. "
|
327
327
|
str << "Skipping is only applicable to double_ups or cyclic nodes."
|
328
|
-
str << "\n
|
328
|
+
str << "\n * #{node_str}."
|
329
329
|
raise str
|
330
330
|
end
|
331
331
|
|
@@ -73,35 +73,35 @@ class Eco::Data::Locations::NodeDiff
|
|
73
73
|
|
74
74
|
msg << "Identified #{diffs.count} differences #{comp}:"
|
75
75
|
msg << when_present(insert) do |count|
|
76
|
-
"
|
76
|
+
" * #{count} nodes to insert"
|
77
77
|
end
|
78
78
|
|
79
79
|
msg << when_present(update) do |count|
|
80
|
-
"
|
80
|
+
" * #{count} nodes to update"
|
81
81
|
end
|
82
82
|
|
83
83
|
msg << when_present(unarchive) do |count|
|
84
|
-
"
|
84
|
+
" * #{count} nodes to unarchive (includes ancestors of target nodes)"
|
85
85
|
end
|
86
86
|
|
87
87
|
msg << when_present(id) do |count|
|
88
|
-
"
|
88
|
+
" * #{count} nodes to change id\n"
|
89
89
|
end
|
90
90
|
|
91
91
|
msg << when_present(name) do |count|
|
92
|
-
"
|
92
|
+
" * #{count} nodes to change name"
|
93
93
|
end
|
94
94
|
|
95
95
|
msg << when_present(classifications) do |count|
|
96
|
-
"
|
96
|
+
" * #{count} nodes to change classifications"
|
97
97
|
end
|
98
98
|
|
99
99
|
msg << when_present(move) do |count|
|
100
|
-
"
|
100
|
+
" * #{count} nodes to move"
|
101
101
|
end
|
102
102
|
|
103
103
|
msg << when_present(archive) do |count|
|
104
|
-
"
|
104
|
+
" * #{count} nodes to archive"
|
105
105
|
end
|
106
106
|
|
107
107
|
msg.compact.join("\n")
|
@@ -11,7 +11,7 @@ class Eco::Data::Locations::NodeLevel
|
|
11
11
|
# 3. It covers the gap if present by decoupling merged parent(s) from the same node (see node.decouple)
|
12
12
|
# 4. Then, it delegates the filling in of parents to `fill_in_parents` function.
|
13
13
|
# @return [Array<NodeLevel>] child to parent relationships solved and no double-ups.
|
14
|
-
def tidy_nodes(nodes, prev_node: nil, main: true)
|
14
|
+
def tidy_nodes(nodes, prev_node: nil, main: true) # rubocop:disable Metrics/AbcSize
|
15
15
|
reset_trackers! if main
|
16
16
|
|
17
17
|
prev_level = prev_node&.actual_level || 0
|
@@ -30,13 +30,16 @@ class Eco::Data::Locations::NodeLevel
|
|
30
30
|
msg = "(Row: #{node.row_num}) ID/Tag '#{node.id}' (lev #{level}) jumps #{gap} level(s) (expected #{prev_level + 1})."
|
31
31
|
#puts " " + node.tags_array.pretty_inspect
|
32
32
|
missing_nodes = node.decouple(gap)
|
33
|
-
msg << "\n Adding missing upper node(s): "
|
33
|
+
msg << "\n Adding missing upper node(s): "
|
34
|
+
msg << missing_nodes.map(&:raw_tag).pretty_inspect
|
35
|
+
|
34
36
|
log(:debug) { msg }
|
35
37
|
# The very top missing node (first in list) should be checked against prev_level
|
36
38
|
# alongside any descendants in missing_nodes (when gap 2+)
|
37
39
|
tidied_nodes = tidy_nodes(missing_nodes, prev_node: prev_node, main: false)
|
38
40
|
out.push(*tidied_nodes)
|
39
41
|
end
|
42
|
+
|
40
43
|
out << node
|
41
44
|
done_ids << node.id
|
42
45
|
prev_node = node
|
@@ -53,21 +56,23 @@ class Eco::Data::Locations::NodeLevel
|
|
53
56
|
# Although with normalized nodes parents are self-contained
|
54
57
|
# we use this method
|
55
58
|
def fill_in_parents(nodes)
|
56
|
-
nodes.tap do
|
59
|
+
nodes.tap do
|
57
60
|
prev_nodes = empty_level_tracker_hash(11)
|
58
61
|
nodes.each do |node|
|
59
62
|
expected_parent_id = node.clean_parent_id&.upcase
|
60
|
-
msg
|
61
|
-
if parent_node = prev_nodes[node.actual_level - 1]
|
63
|
+
msg = "Expecting node '#{node.id}' to have parent: '#{expected_parent_id}'\n"
|
64
|
+
if (parent_node = prev_nodes[node.actual_level - 1])
|
62
65
|
node.parentId = parent_node.id
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
+
unless expected_parent_id == node.parentId
|
67
|
+
log(:warn) {
|
68
|
+
msg + " * We got '#{parent_node.id}' instead"
|
69
|
+
}
|
70
|
+
end
|
66
71
|
elsif node.actual_level == 1
|
67
72
|
# expected to not have parent
|
68
73
|
else
|
69
74
|
log(:warn) {
|
70
|
-
msg
|
75
|
+
"#{msg} but we did not get parent."
|
71
76
|
}
|
72
77
|
end
|
73
78
|
prev_nodes[node.actual_level] = node
|
data/lib/eco/version.rb
CHANGED