eco-helpers 3.0.18 → 3.0.20
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/CHANGELOG.md +34 -3
- data/eco-helpers.gemspec +3 -3
- data/lib/eco/api/common/loaders/config/session.rb +12 -0
- data/lib/eco/api/common/loaders/config/workflow/mailer.rb +17 -4
- data/lib/eco/api/common/loaders/config.rb +10 -2
- data/lib/eco/api/common/loaders/parser.rb +10 -0
- data/lib/eco/api/common/people/default_parsers/csv_parser.rb +21 -208
- data/lib/eco/api/common/people/default_parsers/helpers/expected_headers.rb +206 -0
- data/lib/eco/api/common/people/default_parsers/helpers/null_parsing.rb +36 -0
- data/lib/eco/api/common/people/default_parsers/helpers.rb +15 -0
- data/lib/eco/api/common/people/default_parsers/json_parser.rb +56 -0
- data/lib/eco/api/common/people/default_parsers/xls_parser.rb +13 -14
- data/lib/eco/api/common/people/default_parsers.rb +2 -0
- data/lib/eco/api/common/people/entry_factory.rb +15 -4
- data/lib/eco/api/common/session/sftp.rb +5 -0
- data/lib/eco/api/custom/mailer.rb +1 -0
- data/lib/eco/api/error.rb +4 -0
- data/lib/eco/api/session/batch/job.rb +14 -16
- data/lib/eco/api/session/batch/jobs.rb +6 -8
- data/lib/eco/api/session/batch/launcher/mode_size.rb +5 -2
- data/lib/eco/api/session/batch/launcher/retry.rb +6 -1
- data/lib/eco/api/session/batch/launcher/status_handling.rb +4 -2
- data/lib/eco/api/session/batch/launcher.rb +3 -3
- data/lib/eco/api/session/config/api.rb +1 -0
- data/lib/eco/api/session/config/apis/one_off.rb +6 -6
- data/lib/eco/api/session/config/workflow.rb +16 -3
- data/lib/eco/api/session.rb +13 -7
- data/lib/eco/api/usecases/default/locations/tagtree_extract_case.rb +1 -0
- data/lib/eco/api/usecases/default/locations/tagtree_upload_case.rb +2 -0
- data/lib/eco/api/usecases/default/utils/cli/group_csv_cli.rb +26 -0
- data/lib/eco/api/usecases/default/utils/cli/json_to_csv_cli.rb +10 -0
- data/lib/eco/api/usecases/default/utils/cli/sort_csv_cli.rb +17 -0
- data/lib/eco/api/usecases/default/utils/cli/split_json_cli.rb +15 -0
- data/lib/eco/api/usecases/default/utils/group_csv_case.rb +213 -0
- data/lib/eco/api/usecases/default/utils/json_to_csv_case.rb +71 -0
- data/lib/eco/api/usecases/default/utils/sort_csv_case.rb +127 -0
- data/lib/eco/api/usecases/default/utils/split_json_case.rb +224 -0
- data/lib/eco/api/usecases/default/utils.rb +4 -0
- data/lib/eco/api/usecases/default_cases/samples/sftp_case.rb +22 -15
- data/lib/eco/api/usecases/ooze_cases/export_register_case.rb +6 -6
- data/lib/eco/api/usecases/ooze_samples/helpers/exportable_register.rb +1 -0
- data/lib/eco/api/usecases/ooze_samples/ooze_base_case.rb +1 -1
- data/lib/eco/api/usecases/ooze_samples/ooze_run_base_case.rb +8 -5
- data/lib/eco/cli_default/workflow.rb +10 -4
- data/lib/eco/csv/stream.rb +2 -0
- data/lib/eco/csv.rb +3 -2
- data/lib/eco/language/methods/delegate_missing.rb +4 -3
- data/lib/eco/version.rb +1 -1
- metadata +22 -9
@@ -71,7 +71,7 @@ module Eco
|
|
71
71
|
tap_status(status: status, enviro: enviro, queue: data, method: method) do |overall_status|
|
72
72
|
pending_for_server_error = data.to_a[0..]
|
73
73
|
|
74
|
-
batch_mode_on(*RETRY_ON, options: options, allow_job_mode: job_mode) do |
|
74
|
+
batch_mode_on(*RETRY_ON, options: options, allow_job_mode: job_mode) do |as_job_mode, per_page|
|
75
75
|
iteration = 0
|
76
76
|
done = 0
|
77
77
|
iterations = (data.length.to_f / per_page).ceil
|
@@ -79,7 +79,7 @@ module Eco
|
|
79
79
|
start_time = Time.now
|
80
80
|
|
81
81
|
data.each_slice(per_page) do |slice|
|
82
|
-
iteration
|
82
|
+
iteration += 1
|
83
83
|
|
84
84
|
msg = "starting batch '#{method}' iteration #{iteration}/#{iterations}, "
|
85
85
|
msg << "with #{slice.length} entries of #{data.length} -- #{done} done"
|
@@ -89,7 +89,7 @@ module Eco
|
|
89
89
|
start_slice = Time.now
|
90
90
|
|
91
91
|
offer_retry_on(*RETRY_ON, retries_left: TIMEOUT_RETRIES) do
|
92
|
-
people_api.batch(job_mode:
|
92
|
+
people_api.batch(job_mode: as_job_mode) do |batch|
|
93
93
|
slice.each do |person|
|
94
94
|
batch.public_send(method, person) do |response|
|
95
95
|
faltal("Request with no response") unless response
|
@@ -4,14 +4,14 @@ module Eco
|
|
4
4
|
class Config
|
5
5
|
class Apis
|
6
6
|
module OneOff
|
7
|
-
private
|
8
|
-
|
9
7
|
def one_off?
|
10
|
-
@is_one_off ||=
|
8
|
+
@is_one_off ||= # rubocop:disable Naming/MemoizedInstanceVariableName
|
11
9
|
SCR.get_arg('-api-key') ||
|
12
10
|
SCR.get_arg('-one-off')
|
13
11
|
end
|
14
12
|
|
13
|
+
private
|
14
|
+
|
15
15
|
def one_off_key
|
16
16
|
return @one_off_key if instance_variable_defined?(:@one_off_key)
|
17
17
|
|
@@ -48,10 +48,10 @@ module Eco
|
|
48
48
|
return @one_off_org if instance_variable_defined?(:@one_off_org)
|
49
49
|
|
50
50
|
msg = "You should specify -org NAME when using -api-key or -one-off"
|
51
|
-
raise msg unless org = SCR.get_arg('-org', with_param: true)
|
51
|
+
raise msg unless (org = SCR.get_arg('-org', with_param: true))
|
52
52
|
|
53
53
|
str_org = "#{org.downcase.split(/[^a-z]+/).join('_')}_#{one_off_enviro.gsub('.', '_')}"
|
54
|
-
@one_off_org
|
54
|
+
@one_off_org ||= str_org.to_sym
|
55
55
|
end
|
56
56
|
|
57
57
|
def one_off_enviro
|
@@ -83,7 +83,7 @@ module Eco
|
|
83
83
|
|
84
84
|
true
|
85
85
|
rescue StandardError => err
|
86
|
-
puts
|
86
|
+
puts err.to_s
|
87
87
|
false
|
88
88
|
end
|
89
89
|
end
|
@@ -141,7 +141,8 @@ module Eco
|
|
141
141
|
# @yieldreturn [Eco::API::UseCases::BaseIO] the `io` input/output object carried througout all the _workflow_
|
142
142
|
# @return [Eco::API::Session::Config::Workflow] the current stage object (to ease chainig).
|
143
143
|
def rescue(&block)
|
144
|
-
return @rescue unless
|
144
|
+
return @rescue unless block_given?
|
145
|
+
|
145
146
|
@rescue = block
|
146
147
|
self
|
147
148
|
end
|
@@ -150,7 +151,8 @@ module Eco
|
|
150
151
|
|
151
152
|
# Called on `SystemExit` exception
|
152
153
|
def exit_handle(&block)
|
153
|
-
return @exit_handle unless
|
154
|
+
return @exit_handle unless block_given?
|
155
|
+
|
154
156
|
@exit_handle = block
|
155
157
|
self
|
156
158
|
end
|
@@ -171,6 +173,7 @@ module Eco
|
|
171
173
|
# @return [Eco::API::Session::Config::Workflow] the current stage object (to ease chainig).
|
172
174
|
def before(key = nil, &block)
|
173
175
|
raise ArgumentError, "A block should be given." unless block_given?
|
176
|
+
|
174
177
|
if key
|
175
178
|
stage(key).before(&block)
|
176
179
|
else
|
@@ -195,6 +198,7 @@ module Eco
|
|
195
198
|
# @return [Eco::API::Session::Config::Workflow] the current stage object (to ease chainig).
|
196
199
|
def after(key = nil, &block)
|
197
200
|
raise ArgumentError, "A block should be given." unless block_given?
|
201
|
+
|
198
202
|
if key
|
199
203
|
stage(key).after(&block)
|
200
204
|
else
|
@@ -267,6 +271,7 @@ module Eco
|
|
267
271
|
io.evaluate(self, io, &c)
|
268
272
|
end
|
269
273
|
end
|
274
|
+
|
270
275
|
io
|
271
276
|
end
|
272
277
|
|
@@ -276,6 +281,7 @@ module Eco
|
|
276
281
|
io.evaluate(self, io, &c)
|
277
282
|
end
|
278
283
|
end
|
284
|
+
|
279
285
|
io
|
280
286
|
end
|
281
287
|
|
@@ -305,6 +311,7 @@ module Eco
|
|
305
311
|
io.evaluate(self, io, &@on)
|
306
312
|
end
|
307
313
|
end
|
314
|
+
|
308
315
|
io
|
309
316
|
ensure
|
310
317
|
@pending = false
|
@@ -341,7 +348,11 @@ module Eco
|
|
341
348
|
|
342
349
|
def stage(key)
|
343
350
|
self.class.validate_stage(key)
|
344
|
-
@stages[key] ||= self.class.workflow_class(key).new(
|
351
|
+
@stages[key] ||= self.class.workflow_class(key).new(
|
352
|
+
key,
|
353
|
+
_parent: self,
|
354
|
+
config: config
|
355
|
+
)
|
345
356
|
end
|
346
357
|
|
347
358
|
# helper to treat trigger the exit and rescue handlers
|
@@ -354,6 +365,7 @@ module Eco
|
|
354
365
|
io = io_result(io: io) do
|
355
366
|
io.evaluate(err, io, &exit_handle)
|
356
367
|
end
|
368
|
+
|
357
369
|
exit err.status
|
358
370
|
rescue Interrupt => _int
|
359
371
|
raise
|
@@ -362,6 +374,7 @@ module Eco
|
|
362
374
|
io = io_result(io: io) do
|
363
375
|
io.evaluate(err, io, &self.rescue)
|
364
376
|
end
|
377
|
+
|
365
378
|
raise
|
366
379
|
end
|
367
380
|
end
|
data/lib/eco/api/session.rb
CHANGED
@@ -70,6 +70,7 @@ module Eco
|
|
70
70
|
)
|
71
71
|
if live && api?(version: :graphql)
|
72
72
|
return live_tree(include_archived: include_archived, **kargs, &block) unless merge
|
73
|
+
|
73
74
|
live_trees(include_archived: include_archived, **kargs, &block).inject(&:merge)
|
74
75
|
else
|
75
76
|
config.tagtree(recache: recache)
|
@@ -118,10 +119,12 @@ module Eco
|
|
118
119
|
# @return [Eco::Data::Mapper] the mappings between the internal and external attribute/property names.
|
119
120
|
def fields_mapper
|
120
121
|
return @fields_mapper if instance_variable_defined?(:@fields_mapper)
|
122
|
+
|
121
123
|
mappings = []
|
122
124
|
if (map_file = config.people.fields_mapper)
|
123
125
|
mappings = map_file ? file_manager.load_json(map_file) : []
|
124
126
|
end
|
127
|
+
|
125
128
|
@fields_mapper = Eco::Data::Mapper.new(mappings)
|
126
129
|
end
|
127
130
|
|
@@ -132,7 +135,9 @@ module Eco
|
|
132
135
|
# If `schema` is `nil` or not provided it uses the currently associated to the `session`
|
133
136
|
def entry_factory(schema: nil)
|
134
137
|
schema = to_schema(schema) || self.schema
|
138
|
+
|
135
139
|
return @entry_factories[schema&.id] if @entry_factories.key?(schema&.id)
|
140
|
+
|
136
141
|
unless @entry_factories.empty?
|
137
142
|
@entry_factories[schema&.id] = @entry_factories.values.first.newFactory(schema: schema)
|
138
143
|
return @entry_factories[schema&.id]
|
@@ -164,9 +169,9 @@ module Eco
|
|
164
169
|
# @param phase [Symbol] the phase when this parser should be active.
|
165
170
|
# @return [Object] the parsed attribute.
|
166
171
|
def parse_attribute(attr, source, phase = :internal, deps: {})
|
167
|
-
|
168
|
-
|
169
|
-
|
172
|
+
msg = "There are no parsers defined"
|
173
|
+
raise msg unless (parsers = entry_factory.person_parser)
|
174
|
+
|
170
175
|
parsers.parse(attr, source, phase, deps: deps)
|
171
176
|
end
|
172
177
|
|
@@ -388,18 +393,19 @@ module Eco
|
|
388
393
|
|
389
394
|
# from schema `id` or `name` to a PersonSchema object
|
390
395
|
def to_schema(value)
|
391
|
-
return nil unless value
|
392
396
|
sch = nil
|
397
|
+
return unless value
|
398
|
+
|
393
399
|
case value
|
394
400
|
when String
|
395
|
-
|
396
|
-
|
397
|
-
end
|
401
|
+
msg = "The schema with id or name '#{value}' does not exist."
|
402
|
+
fatal msg unless (sch = schemas.schema(value))
|
398
403
|
when Ecoportal::API::V1::PersonSchema
|
399
404
|
sch = value
|
400
405
|
else
|
401
406
|
fatal "Required String or Ecoportal::API::V1::PersonSchema. Given: #{value}"
|
402
407
|
end
|
408
|
+
|
403
409
|
sch
|
404
410
|
end
|
405
411
|
end
|
@@ -27,6 +27,7 @@ class Eco::API::UseCases::Default::Locations::TagtreeUpload < Eco::API::UseCases
|
|
27
27
|
comms << insert_command(tree, pid: pid) unless top_id?(tree.id)
|
28
28
|
pid = tree.id
|
29
29
|
end
|
30
|
+
|
30
31
|
tree.nodes.map do |node|
|
31
32
|
insert_commands(node, pid: pid)
|
32
33
|
end.flatten(1).tap do |subs|
|
@@ -54,6 +55,7 @@ class Eco::API::UseCases::Default::Locations::TagtreeUpload < Eco::API::UseCases
|
|
54
55
|
|
55
56
|
def top_id?(node_id = nil)
|
56
57
|
return top_id.is_a?(String) if node_id.nil?
|
58
|
+
|
57
59
|
node_id == top_id
|
58
60
|
end
|
59
61
|
|
@@ -0,0 +1,26 @@
|
|
1
|
+
class Eco::API::UseCases::Default::Utils::GroupCsv
|
2
|
+
class Cli < Eco::API::UseCases::Cli
|
3
|
+
str_desc = 'Groups the csv rows by a pivot field. '
|
4
|
+
str_desc << 'It assumes the sorting field is sorted '
|
5
|
+
str_desc << '(same values should be consecutive)'
|
6
|
+
|
7
|
+
desc str_desc
|
8
|
+
|
9
|
+
callback do |_session, options, _usecase|
|
10
|
+
if (file = SCR.get_file(cli_name, required: true, should_exist: true))
|
11
|
+
options.deep_merge!(input: {file: {name: file}})
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
add_option("-start-at", "Get only the last N-start_at rows") do |options|
|
16
|
+
count = SCR.get_arg("-start-at", with_param: true)
|
17
|
+
options.deep_merge!(input: {file: {start_at: count}})
|
18
|
+
end
|
19
|
+
|
20
|
+
add_option('-by', 'The column that should be used to group') do |options|
|
21
|
+
if (file = SCR.get_arg("-by", with_param: true))
|
22
|
+
options.deep_merge!(input: {group_by_field: file})
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
class Eco::API::UseCases::Default::Utils::JsonToCsv
|
2
|
+
class Cli < Eco::API::UseCases::Cli
|
3
|
+
desc "Transforms an input JSON file into a CSV one."
|
4
|
+
|
5
|
+
callback do |_sess, options, _case|
|
6
|
+
file = SCR.get_file(cli_name, required: true, should_exist: true)
|
7
|
+
options.deep_merge!(source: {file: file})
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class Eco::API::UseCases::Default::Utils::SortCsv
|
2
|
+
class Cli < Eco::API::UseCases::Cli
|
3
|
+
desc 'Sorts the CSV by column -by'
|
4
|
+
|
5
|
+
callback do |_session, options, _usecase|
|
6
|
+
if (file = SCR.get_file(cli_name, required: true, should_exist: true))
|
7
|
+
options.deep_merge!(input: {file: file})
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
add_option('-by', 'The column that should be used to sorting') do |options|
|
12
|
+
if (file = SCR.get_arg("-by", with_param: true))
|
13
|
+
options.deep_merge!(input: {sort_by: file})
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class Eco::API::UseCases::Default::Utils::SplitJson
|
2
|
+
class Cli < Eco::API::UseCases::Cli
|
3
|
+
desc 'Splits a json input file into multiple files'
|
4
|
+
|
5
|
+
callback do |_sess, options, _case|
|
6
|
+
file = SCR.get_file(cli_name, required: true, should_exist: true)
|
7
|
+
options.deep_merge!(source: {file: file})
|
8
|
+
end
|
9
|
+
|
10
|
+
add_option("-max-items", "The max count of items of the output files") do |options|
|
11
|
+
count = SCR.get_arg("-max-items", with_param: true)
|
12
|
+
options.deep_merge!(output: {file: {max_items: count}})
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,213 @@
|
|
1
|
+
# This script assumes that for the `GROUP_BY_FIELD` rows are consecutive.
|
2
|
+
# @note you might run first the `sort-csv` case.
|
3
|
+
# @note you must inherit from this case and define the constants.
|
4
|
+
#
|
5
|
+
# GROUP_BY_FIELD = 'target_csv_field'.freeze
|
6
|
+
# GROUPED_FIELDS = [
|
7
|
+
# 'joined_field_1',
|
8
|
+
# 'joined_field_2',
|
9
|
+
# 'joined_field_3',
|
10
|
+
# ].freeze
|
11
|
+
#
|
12
|
+
class Eco::API::UseCases::Default::Utils::GroupCsv < Eco::API::Custom::UseCase
|
13
|
+
name 'group-csv'
|
14
|
+
type :other
|
15
|
+
|
16
|
+
require_relative 'cli/group_csv_cli'
|
17
|
+
|
18
|
+
def main(*_args)
|
19
|
+
if simulate?
|
20
|
+
count = Eco::CSV.count(input_file)
|
21
|
+
log(:info) { "CSV '#{input_file}' has #{count} rows." }
|
22
|
+
else
|
23
|
+
generate_file
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def generate_file # rubocop:disable Metrics/AbcSize
|
30
|
+
row_count = 0
|
31
|
+
in_index = nil
|
32
|
+
|
33
|
+
CSV.open(output_filename, 'wb') do |out_csv|
|
34
|
+
first = true
|
35
|
+
|
36
|
+
puts "\n"
|
37
|
+
|
38
|
+
streamed_input.for_each(start_at_idx: start_at) do |row, idx|
|
39
|
+
if first
|
40
|
+
first = false
|
41
|
+
headers!(row)
|
42
|
+
out_csv << headers
|
43
|
+
require_group_by_field!(row, file: input_file)
|
44
|
+
end
|
45
|
+
|
46
|
+
in_index = idx
|
47
|
+
next unless !block_given? || yield(row, idx)
|
48
|
+
|
49
|
+
next unless pivotable?(row, idx)
|
50
|
+
next unless (last_group = pivot_row(row))
|
51
|
+
|
52
|
+
row_count += 1
|
53
|
+
|
54
|
+
if (row_count % 500).zero?
|
55
|
+
print "... Done #{row_count} rows \r"
|
56
|
+
$stdout.flush
|
57
|
+
end
|
58
|
+
|
59
|
+
out_csv << last_group.values_at(*headers)
|
60
|
+
end
|
61
|
+
|
62
|
+
# finalize
|
63
|
+
if (lrow = pivot_row)
|
64
|
+
row_count += 1
|
65
|
+
out_csv << lrow.values_at(*headers)
|
66
|
+
end
|
67
|
+
ensure
|
68
|
+
msg = "Generated file '#{output_filename}' "
|
69
|
+
msg << "with #{row_count} rows (out of #{in_index})."
|
70
|
+
|
71
|
+
log(:info) { msg } unless simulate?
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
# It tracks the current grouped row
|
76
|
+
# @return [Nil, Hash] the last grouped row when `row` doesn't belong
|
77
|
+
# or `nil` otherwise
|
78
|
+
def pivot_row(row = nil)
|
79
|
+
@group ||= {}
|
80
|
+
return @group unless row
|
81
|
+
|
82
|
+
pivot_value = row[group_by_field]
|
83
|
+
|
84
|
+
unless (last_pivot = @group[group_by_field])
|
85
|
+
last_pivot = @group[group_by_field] = pivot_value
|
86
|
+
end
|
87
|
+
|
88
|
+
last = @group
|
89
|
+
@group = {group_by_field => pivot_value} unless pivot_value == last_pivot
|
90
|
+
|
91
|
+
headers_rest.each do |field|
|
92
|
+
curr_values = row[field].to_s.split('|').compact.uniq
|
93
|
+
pivot_values = @group[field].to_s.split('|').compact.uniq
|
94
|
+
@group[field] = (pivot_values | curr_values).join('|')
|
95
|
+
end
|
96
|
+
|
97
|
+
last unless last == @group
|
98
|
+
end
|
99
|
+
|
100
|
+
attr_reader :group
|
101
|
+
attr_reader :headers, :headers_rest
|
102
|
+
|
103
|
+
def headers!(row)
|
104
|
+
return if headers?
|
105
|
+
|
106
|
+
@headers_rest = grouped_fields & row.headers
|
107
|
+
@headers_rest -= [group_by_field]
|
108
|
+
@headers = [group_by_field, *headers_rest]
|
109
|
+
end
|
110
|
+
|
111
|
+
def headers?
|
112
|
+
instance_variable_defined?(:@headers)
|
113
|
+
end
|
114
|
+
|
115
|
+
def pivotable?(row, idx)
|
116
|
+
return true unless row[group_by_field].to_s.strip.empty?
|
117
|
+
|
118
|
+
msg = "Row #{idx} doesn't have value for pivot field '#{group_by_field}'"
|
119
|
+
msg << ". Skipping (discared) ..."
|
120
|
+
log(:warn) { msg }
|
121
|
+
false
|
122
|
+
end
|
123
|
+
|
124
|
+
def streamed_input
|
125
|
+
@streamed_input ||= Eco::CSV::Stream.new(input_file)
|
126
|
+
end
|
127
|
+
|
128
|
+
def input_file
|
129
|
+
options.dig(:input, :file, :name)
|
130
|
+
end
|
131
|
+
|
132
|
+
def start_at
|
133
|
+
return nil unless (num = options.dig(:input, :file, :start_at))
|
134
|
+
|
135
|
+
num = num.to_i
|
136
|
+
num = nil if num.zero?
|
137
|
+
num
|
138
|
+
end
|
139
|
+
|
140
|
+
def output_filename
|
141
|
+
return nil unless input_name
|
142
|
+
|
143
|
+
File.join(input_dir, "#{input_name}_grouped#{input_ext}")
|
144
|
+
end
|
145
|
+
|
146
|
+
def input_name
|
147
|
+
@input_name ||= File.basename(input_basename, input_ext)
|
148
|
+
end
|
149
|
+
|
150
|
+
def input_ext
|
151
|
+
@input_ext ||= input_basename.split('.')[1..].join('.').then do |name|
|
152
|
+
".#{name}"
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def input_basename
|
157
|
+
@input_basename ||= File.basename(input_full_filename)
|
158
|
+
end
|
159
|
+
|
160
|
+
def input_dir
|
161
|
+
@input_dir = File.dirname(input_full_filename)
|
162
|
+
end
|
163
|
+
|
164
|
+
def input_full_filename
|
165
|
+
@input_full_filename ||= File.expand_path(input_file)
|
166
|
+
end
|
167
|
+
|
168
|
+
def require_group_by_field!(row, file:)
|
169
|
+
return true if row.key?(group_by_field)
|
170
|
+
|
171
|
+
msg = "Pivot field '#{group_by_field}' missing in header of file '#{file}'"
|
172
|
+
log(:error) { msg }
|
173
|
+
raise msg
|
174
|
+
end
|
175
|
+
|
176
|
+
def group_by_field
|
177
|
+
return @group_by_field if instance_variable_defined?(:@group_by_field)
|
178
|
+
|
179
|
+
return (@group_by_field = opts_group_by) if opts_group_by
|
180
|
+
|
181
|
+
unless self.class.const_defined?(:GROUP_BY_FIELD)
|
182
|
+
msg = "(#{self.class}) You must define GROUP_BY_FIELD constant"
|
183
|
+
log(:error) { msg }
|
184
|
+
raise msg
|
185
|
+
end
|
186
|
+
|
187
|
+
@group_by_field = self.class::GROUP_BY_FIELD
|
188
|
+
end
|
189
|
+
|
190
|
+
def grouped_fields
|
191
|
+
return @grouped_fields if instance_variable_defined?(:@grouped_fields)
|
192
|
+
|
193
|
+
unless self.class.const_defined?(:GROUPED_FIELDS)
|
194
|
+
msg = "(#{self.class}) You must define GROUPED_FIELDS constant"
|
195
|
+
log(:error) { msg }
|
196
|
+
raise msg
|
197
|
+
end
|
198
|
+
|
199
|
+
@grouped_fields ||= [self.class::GROUPED_FIELDS].flatten.compact.tap do |flds|
|
200
|
+
next unless flds.empty?
|
201
|
+
|
202
|
+
log(:warn) {
|
203
|
+
msg = "There were no fields to be grouped/joined. "
|
204
|
+
msg << "This is equivalent to launch a unique operation."
|
205
|
+
msg
|
206
|
+
}
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
def opts_group_by
|
211
|
+
options.dig(:input, :group_by_field)
|
212
|
+
end
|
213
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
class Eco::API::UseCases::Default::Utils::JsonToCsv < Eco::API::Common::Loaders::UseCase
|
2
|
+
require_relative 'cli/json_to_csv_cli'
|
3
|
+
|
4
|
+
name 'json-to-csv'
|
5
|
+
type :other
|
6
|
+
|
7
|
+
def main(*_args)
|
8
|
+
return if simulate?
|
9
|
+
|
10
|
+
CSV.open(out_filename, 'w') do |csv|
|
11
|
+
csv << all_keys
|
12
|
+
data.each do |item|
|
13
|
+
csv << item.values_at(*all_keys)
|
14
|
+
end
|
15
|
+
ensure
|
16
|
+
log(:info) {
|
17
|
+
"Generated output file: '#{File.expand_path(out_filename)}'."
|
18
|
+
}
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def all_keys
|
25
|
+
@all_keys ||= data.each_with_object([]) do |item, head|
|
26
|
+
head.concat(item.keys - head)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def data
|
31
|
+
@data ||= parse_json_file.tap do |dt|
|
32
|
+
ensure_array!(dt)
|
33
|
+
|
34
|
+
log(:info) {
|
35
|
+
"Loaded #{dt.count} items (from file '#{File.basename(input_file)}')"
|
36
|
+
}
|
37
|
+
|
38
|
+
exit 0 if simulate?
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def out_filename
|
43
|
+
@out_filename ||= ''.then do
|
44
|
+
input_basename = File.basename(input_file)
|
45
|
+
base_name = File.basename(input_basename, '.json')
|
46
|
+
"#{base_name}.csv"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def input_file
|
51
|
+
options.dig(:source, :file)
|
52
|
+
end
|
53
|
+
|
54
|
+
def ensure_array!(data)
|
55
|
+
return if data.is_a?(Array)
|
56
|
+
|
57
|
+
msg = "Expecting JSON file to contain an Array. Given: #{data.class}"
|
58
|
+
log(:error) { msg }
|
59
|
+
raise msg
|
60
|
+
end
|
61
|
+
|
62
|
+
def parse_json_file(filename = input_file)
|
63
|
+
fd = File.open(filename)
|
64
|
+
JSON.load fd # rubocop:disable Security/JSONLoad
|
65
|
+
rescue JSON::ParserError => err
|
66
|
+
log(:error) { "Parsing error on file '#{filename}'" }
|
67
|
+
raise err
|
68
|
+
ensure
|
69
|
+
fd&.close
|
70
|
+
end
|
71
|
+
end
|