eco-helpers 2.0.21 → 2.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +28 -3
- data/eco-helpers.gemspec +0 -1
- data/lib/eco/api/common/base_loader.rb +9 -5
- data/lib/eco/api/common/people/default_parsers.rb +1 -0
- data/lib/eco/api/common/people/default_parsers/xls_parser.rb +53 -0
- data/lib/eco/api/common/people/entry_factory.rb +26 -12
- data/lib/eco/api/common/people/person_parser.rb +1 -1
- data/lib/eco/api/session.rb +11 -5
- data/lib/eco/api/usecases.rb +2 -2
- data/lib/eco/api/usecases/base_case.rb +2 -2
- data/lib/eco/api/usecases/base_io.rb +17 -4
- data/lib/eco/cli/config/default/input.rb +61 -8
- data/lib/eco/cli/config/default/options.rb +1 -1
- data/lib/eco/cli/config/default/workflow.rb +1 -1
- data/lib/eco/cli/scripting/args_helpers.rb +2 -2
- data/lib/eco/version.rb +1 -1
- metadata +2 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ba43099c7902b3e2623640a196abd86f39215403d28e7a614da78b2422629323
|
4
|
+
data.tar.gz: 7e3701cbeb7b3a0a65453dc002d6f4c5cb6e6ec4103e6cd0178c1af333d29d7d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f14ab296c2cff2dc694ef9182593329ca1bc86cf08ca32ba879cd5e74d658d1d123add425bc62a981182b9e4da06b33d225d613cd1a9d9cbfd7284e6b6d24ea5
|
7
|
+
data.tar.gz: f020822ad4ae2e18f1f0424bb3a47417ff9a6bc78a439517cee3cd9212548efe4c89256579d73ccde27709dc5d2ea3d3e74bd072de23d4c089ccf3c7a107fc07
|
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,34 @@
|
|
1
1
|
# Change Log
|
2
2
|
All notable changes to this project will be documented in this file.
|
3
3
|
|
4
|
-
## [2.0.
|
4
|
+
## [2.0.22] - 2021-06-18
|
5
|
+
|
6
|
+
### Added
|
7
|
+
- exposed `logger` in `BaseLoader` and
|
8
|
+
- support for multiple input files
|
9
|
+
* `Eco::API::Common::People::EntryFactory#entries`:
|
10
|
+
- refactored to allow multiple input files parsing
|
11
|
+
- moreover to `idx`, hash entries will get their `source_file`
|
12
|
+
* Input callback at `lib/eco/cli/config/default/input` refactored format detection and enabled folder input
|
13
|
+
* `SCR.get_file` language extended to also mention folder (not just file)
|
14
|
+
- support for `.xls` and `.xlsx` files
|
15
|
+
* `Eco::API::Common::People::DefaultParsers::XLSParser` the Excel files **parser**
|
16
|
+
* `Eco::API::Common::People::PersonParser` added `:xls` as an accepted format
|
17
|
+
* `Eco::API::Session#fields_mapper` exposed mapper through a method to allow **headers detection**
|
18
|
+
- The external names of the fields are the column headers of the input file
|
19
|
+
* `Eco::API::UseCases::BaseIO` when arguments validation rails, now it raises with specific `MissingParameter` error
|
20
|
+
|
21
|
+
### Changed
|
22
|
+
- dry out `BaseLoader` (only session is set as instance variable)
|
23
|
+
- removed `creek` **dependency** (it was not used anywhere in the gem)
|
24
|
+
* we just kept `roo` and `roo-xls`
|
25
|
+
- custom `Error` classes now all inherit from `StandardError` (rather than `Exception`)
|
26
|
+
|
27
|
+
|
28
|
+
### Fixed
|
29
|
+
|
30
|
+
|
31
|
+
## [2.0.21] - 2021-06-04
|
5
32
|
|
6
33
|
### Added
|
7
34
|
- `Eco::CSV::Table`, support to create the table out of an `Array<Hash>`
|
@@ -11,8 +38,6 @@ All notable changes to this project will be documented in this file.
|
|
11
38
|
### Changed
|
12
39
|
- `Eco::API::Common::People::EntryFactory` slight **refactor** to boost better support for multiple input formats
|
13
40
|
|
14
|
-
### Fixed
|
15
|
-
|
16
41
|
|
17
42
|
## [2.0.20] - 2021-05-31
|
18
43
|
|
data/eco-helpers.gemspec
CHANGED
@@ -51,15 +51,19 @@ module Eco
|
|
51
51
|
private
|
52
52
|
|
53
53
|
def session
|
54
|
-
|
54
|
+
ASSETS.session
|
55
55
|
end
|
56
56
|
|
57
|
-
def
|
58
|
-
session.
|
57
|
+
def config
|
58
|
+
session.config
|
59
59
|
end
|
60
60
|
|
61
|
-
def
|
62
|
-
|
61
|
+
def logger
|
62
|
+
session.logger
|
63
|
+
end
|
64
|
+
|
65
|
+
def micro
|
66
|
+
session.micro
|
63
67
|
end
|
64
68
|
|
65
69
|
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
class Eco::API::Common::People::DefaultParsers::XLSParser < Eco::API::Common::Loaders::Parser
|
2
|
+
attribute :xls
|
3
|
+
|
4
|
+
attr_accessor :already_required
|
5
|
+
attr_reader :file
|
6
|
+
|
7
|
+
def parser(file, deps)
|
8
|
+
@file = file
|
9
|
+
rows.tap {|r| @file = nil}
|
10
|
+
end
|
11
|
+
|
12
|
+
def serializer(array_hash, deps)
|
13
|
+
raise "Not implemented. TODO: using axlsx or rubyXL gems. See: https://spin.atomicobject.com/2017/03/22/parsing-excel-files-ruby/"
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def headers
|
19
|
+
raise "You should implement this method"
|
20
|
+
end
|
21
|
+
|
22
|
+
def sheet_name
|
23
|
+
0
|
24
|
+
end
|
25
|
+
|
26
|
+
def workbook
|
27
|
+
require_reading_libs!
|
28
|
+
Roo::Spreadsheet.open(file)
|
29
|
+
end
|
30
|
+
|
31
|
+
def spreadheet(name_or_index = sheet_name)
|
32
|
+
workbook.sheet(name_or_index)
|
33
|
+
end
|
34
|
+
|
35
|
+
def rows(target = headers)
|
36
|
+
begin
|
37
|
+
spreadheet.parse(header_search: target)
|
38
|
+
rescue Roo::HeaderRowNotFoundError => e
|
39
|
+
missing = JSON.parse(e.message)
|
40
|
+
logger.warn("The input file is missing these headers: #{missing}")
|
41
|
+
present = target - missing
|
42
|
+
rows(present)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def require_reading_libs!
|
47
|
+
return if already_required
|
48
|
+
require 'roo'
|
49
|
+
require 'roo-xls'
|
50
|
+
already_required = true
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
@@ -101,33 +101,47 @@ module Eco
|
|
101
101
|
data = []
|
102
102
|
content, file, encoding, format = kargs.values_at(:content, :file, :encoding, :format)
|
103
103
|
|
104
|
-
|
104
|
+
# Support for multiple file
|
105
|
+
if file.is_a?(Array)
|
106
|
+
return file.each_with_object([]) do |f, out|
|
107
|
+
logger.info("Parsing file '#{f}'")
|
108
|
+
curr = to_array_of_hashes(**kargs.merge(file: f))
|
109
|
+
out.concat(curr)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
# Get content only when it's not :xls
|
113
|
+
# note: even if content was provided, file takes precedence
|
114
|
+
content = get_file_content(file, format, encoding) if (format != :xls) && file
|
105
115
|
|
106
116
|
case content
|
107
|
-
when !content
|
108
|
-
logger.error("Could not obtain any data out of these: #{kargs}")
|
109
|
-
exit(1)
|
110
117
|
when Hash
|
111
118
|
logger.error("Input data as 'Hash' not supported. Expecting 'Enumerable' or 'String'")
|
112
119
|
exit(1)
|
113
120
|
when String
|
114
|
-
|
115
|
-
j = (format == :csv)? i + 2 : i + 1
|
116
|
-
entry_hash.tap {|hash| hash["idx"] = j}
|
117
|
-
end
|
118
|
-
to_array_of_hashes(content: data)
|
121
|
+
to_array_of_hashes(content: person_parser.parse(format, content))
|
119
122
|
when Enumerable
|
120
123
|
sample = content.to_a.first
|
121
124
|
case sample
|
122
125
|
when Hash, Array, ::CSV::Row
|
123
126
|
Eco::CSV::Table.new(content).to_array_of_hashes
|
124
127
|
else
|
125
|
-
logger.error("Input 'Array' of '#{sample.class}' is not supported.")
|
128
|
+
logger.error("Input content 'Array' of '#{sample.class}' is not supported.")
|
126
129
|
end
|
127
130
|
else
|
128
|
-
|
129
|
-
|
131
|
+
if file && format == :xls
|
132
|
+
person_parser.parse(format, file)
|
133
|
+
else
|
134
|
+
logger.error("Could not obtain any data out of these: #{kargs}. Given content: '#{content.class}'")
|
135
|
+
exit(1)
|
136
|
+
end
|
137
|
+
end.tap do |out_array|
|
138
|
+
start_from_two = (format == :csv) || format == :xls
|
139
|
+
out_array.each_with_index do |entry_hash, i|
|
140
|
+
entry_hash["idx"] = start_from_two ? i + 2 : i + 1
|
141
|
+
entry_hash["source_file"] = file
|
142
|
+
end
|
130
143
|
end
|
144
|
+
|
131
145
|
end
|
132
146
|
|
133
147
|
|
@@ -16,7 +16,7 @@ module Eco
|
|
16
16
|
CORE_ATTRS = ["id", "external_id", "email", "name", "supervisor_id", "filter_tags", "freemium"]
|
17
17
|
ACCOUNT_ATTRS = ["policy_group_ids", "default_tag", "send_invites", "landing_page_id", "login_provider_ids"]
|
18
18
|
TYPE = [:select, :text, :date, :number, :phone_number, :boolean, :multiple]
|
19
|
-
FORMAT = [:csv, :xml, :json]
|
19
|
+
FORMAT = [:csv, :xml, :json, :xls]
|
20
20
|
|
21
21
|
attr_reader :schema
|
22
22
|
attr_reader :details_attrs, :all_model_attrs
|
data/lib/eco/api/session.rb
CHANGED
@@ -66,6 +66,16 @@ module Eco
|
|
66
66
|
@presets_factory ||= Eco::API::Organization::PresetsFactory.new(enviro: enviro)
|
67
67
|
end
|
68
68
|
|
69
|
+
# @return [Eco::Data::Mapper] the mappings between the internal and external attribute/property names.
|
70
|
+
def fields_mapper
|
71
|
+
return @fields_mapper if instance_variable_defined?(:@fields_mapper)
|
72
|
+
mappings = []
|
73
|
+
if map_file = config.people.fields_mapper
|
74
|
+
mappings = map_file ? file_manager.load_json(map_file) : []
|
75
|
+
end
|
76
|
+
@fields_mapper = Eco::Data::Mapper.new(mappings)
|
77
|
+
end
|
78
|
+
|
69
79
|
# Helper to obtain a EntryFactory
|
70
80
|
# @param schema [String, Ecoportal::API::V1::PersonSchema] `schema` to which associate the EntryFactory,
|
71
81
|
# where `String` can be the _name_ or the _id_ of the schema.
|
@@ -79,15 +89,11 @@ module Eco
|
|
79
89
|
return @entry_factories[schema&.id]
|
80
90
|
end
|
81
91
|
|
82
|
-
mappings = []
|
83
|
-
if map_file = config.people.fields_mapper
|
84
|
-
mappings = map_file ? file_manager.load_json(map_file) : []
|
85
|
-
end
|
86
92
|
@entry_factories[schema&.id] = Eco::API::Common::People::EntryFactory.new(
|
87
93
|
enviro,
|
88
94
|
schema: schema,
|
89
95
|
person_parser: config.people.parser,
|
90
|
-
attr_map:
|
96
|
+
attr_map: fields_mapper
|
91
97
|
)
|
92
98
|
end
|
93
99
|
|
data/lib/eco/api/usecases.rb
CHANGED
@@ -2,7 +2,7 @@ module Eco
|
|
2
2
|
module API
|
3
3
|
class UseCases
|
4
4
|
|
5
|
-
class UnkownCase <
|
5
|
+
class UnkownCase < StandardError
|
6
6
|
def initialize(msg = nil, case_name: nil, type: nil)
|
7
7
|
msg ||= "Unkown case"
|
8
8
|
msg += ". Case name '#{case_name}'" if case_name
|
@@ -11,7 +11,7 @@ module Eco
|
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
-
class AmbiguousCaseReference <
|
14
|
+
class AmbiguousCaseReference < StandardError
|
15
15
|
def initialize(msg = nil, case_name: nil)
|
16
16
|
msg ||= "You must specify type when there are multiple cases with same name"
|
17
17
|
msg += ". Case name '#{case_name}'" if case_name
|
@@ -4,7 +4,7 @@ module Eco
|
|
4
4
|
# Core class of UseCases. It basically defines and manages allowed `types`
|
5
5
|
class BaseCase
|
6
6
|
|
7
|
-
class InvalidType <
|
7
|
+
class InvalidType < StandardError
|
8
8
|
def initialize(msg = nil, type:, types:)
|
9
9
|
msg ||= "Invalid type."
|
10
10
|
msg = "Given type '#{type}'. Valid types: #{types}"
|
@@ -13,7 +13,7 @@ module Eco
|
|
13
13
|
end
|
14
14
|
|
15
15
|
extend Eco::API::Common::ClassHelpers
|
16
|
-
|
16
|
+
|
17
17
|
@types = [:import, :filter, :transform, :sync, :error_handler, :export, :other]
|
18
18
|
|
19
19
|
class << self
|
@@ -5,6 +5,19 @@ module Eco
|
|
5
5
|
class BaseIO < BaseCase
|
6
6
|
@types = BaseCase.types
|
7
7
|
|
8
|
+
class MissingParameter < StandardError
|
9
|
+
attr_reader :type, :required, :given
|
10
|
+
|
11
|
+
def initialize(msg = nil, type: nil, required:, given:)
|
12
|
+
@type = type
|
13
|
+
@required = required
|
14
|
+
@given = given
|
15
|
+
msg += " of type '#{type}'" if type
|
16
|
+
msg += " requires an object '#{required}'. Given: #{given}."
|
17
|
+
super(msg)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
8
21
|
class << self
|
9
22
|
def input_required?(type)
|
10
23
|
!valid_type?(type) || [:import, :sync].include?(type)
|
@@ -80,13 +93,13 @@ module Eco
|
|
80
93
|
def validate_args(input:, people:, session:, options:)
|
81
94
|
case
|
82
95
|
when !session.is_a?(Eco::API::Session)
|
83
|
-
raise "
|
96
|
+
raise MissingParameter.new("UseCase", required: :session, given: session.class)
|
84
97
|
when input_required? && !input
|
85
|
-
raise "UseCase
|
98
|
+
raise MissingParameter.new("UseCase", type: type, required: :input, given: input.class)
|
86
99
|
when people_required? && !people.is_a?(Eco::API::Organization::People)
|
87
|
-
raise "UseCase
|
100
|
+
raise MissingParameter.new("UseCase", type: type, required: :people, given: people.class)
|
88
101
|
when !options || (options && !options.is_a?(Hash))
|
89
|
-
raise "
|
102
|
+
raise MissingParameter.new("Use Case options", required: :Hash, given: options.class)
|
90
103
|
end
|
91
104
|
true
|
92
105
|
end
|
@@ -1,18 +1,71 @@
|
|
1
1
|
ASSETS.cli.config do |cnf|
|
2
|
+
formats = {
|
3
|
+
csv: {
|
4
|
+
option: ["-csv"],
|
5
|
+
extname: [".csv", ".txt"]
|
6
|
+
},
|
7
|
+
xml: {
|
8
|
+
option: ["-xml"],
|
9
|
+
extname: [".xml"]
|
10
|
+
},
|
11
|
+
xls: {
|
12
|
+
option: ["-xls", "-xlsx", "-excel"],
|
13
|
+
extname: [".xls", ".xlsx", ".xlsm"]
|
14
|
+
},
|
15
|
+
json: {
|
16
|
+
option: ["-json"],
|
17
|
+
extname: [".json"]
|
18
|
+
}
|
19
|
+
}
|
20
|
+
|
2
21
|
cnf.input(default_option: "-entries-from") do |session, str_opt, options|
|
3
22
|
input = []
|
4
23
|
if SCR.get_arg(str_opt)
|
5
24
|
file = SCR.get_file(str_opt, required: true)
|
25
|
+
|
26
|
+
# Command line check
|
27
|
+
format = formats.reduce(nil) do |matched, (format, selectors)|
|
28
|
+
used = selectors[:option].reduce(false) {|used, option| SCR.get_arg(option) || used}
|
29
|
+
next matched if matched
|
30
|
+
next format if used
|
31
|
+
end
|
32
|
+
|
33
|
+
# File/Folder check
|
34
|
+
file = File.expand_path(file)
|
35
|
+
if File.directory?(file)
|
36
|
+
folder = file
|
37
|
+
file = Dir.glob("#{file}/*").reject {|f| File.directory?(f)}
|
38
|
+
ext = (format && formats[format][:extname]) || [File.extname(file.first)]
|
39
|
+
file = file.select {|f| ext.any? {|e| File.extname(f) == e}}.tap do |files|
|
40
|
+
if files.empty?
|
41
|
+
session.logger.error("Could not find any file with extension: #{ext} in folder '#{folder}'")
|
42
|
+
exit(1)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
else
|
46
|
+
ext = File.extname(file)
|
47
|
+
end
|
48
|
+
|
49
|
+
format ||= formats.reduce(nil) do |matched, (format, selectors)|
|
50
|
+
next matched if matched
|
51
|
+
next format if selectors[:extname].any? {|e| ext == e}
|
52
|
+
end
|
53
|
+
format ||= :csv
|
54
|
+
|
6
55
|
options.deep_merge!(input: {file: {name: file}})
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
56
|
+
options.deep_merge!(input: {file: {format: format}})
|
57
|
+
|
58
|
+
case format
|
59
|
+
when :xml
|
60
|
+
[file].flatten.each {|f| session.config.files.validate(:xml, f)}
|
61
|
+
input = session.entries(file: file, format: format)
|
62
|
+
when :xls
|
63
|
+
input = session.entries(file: file, format: format)
|
64
|
+
when :json
|
65
|
+
input = [file].flatten.reduce(Eco::API::Organization::People.new([])) do |people, file|
|
66
|
+
people.merge(JSON.parse(File.read(file)))
|
67
|
+
end
|
14
68
|
else
|
15
|
-
options.deep_merge!(input: {file: {format: :csv}})
|
16
69
|
input = session.csv_entries(file)
|
17
70
|
end
|
18
71
|
end
|
@@ -41,7 +41,7 @@ ASSETS.cli.config do |cnf|
|
|
41
41
|
session.schema = sch_id
|
42
42
|
end
|
43
43
|
|
44
|
-
desc = "Used to be used to specify the input file when using -get-partial.
|
44
|
+
desc = "Used to be used to specify the input file or folder when using -get-partial."
|
45
45
|
desc += "It can also be useful to obtain `-get-partial` of people base on `:export` use cases (i.e. -people-to-csv)"
|
46
46
|
options_set.add("-entries-from", desc) do |options, session|
|
47
47
|
options.deep_merge!(input: {entries_from: true})
|
@@ -103,7 +103,7 @@ ASSETS.cli.config do |config|
|
|
103
103
|
if !io.options[:dry_run] && partial_update
|
104
104
|
# get target people afresh
|
105
105
|
people = io.session.micro.people_refresh(people: io.people, include_created: true)
|
106
|
-
io = io.new(people: people)
|
106
|
+
io = io.base.new(people: people)
|
107
107
|
else
|
108
108
|
wf_post.skip!
|
109
109
|
msg = "Although there are post_launch cases, they will NOT be RUN"
|
@@ -75,10 +75,10 @@ module Eco
|
|
75
75
|
def get_file(key, required: false, should_exist: true)
|
76
76
|
filename = get_arg(key, with_param: true)
|
77
77
|
if !filename && required
|
78
|
-
puts "You need to specify a file '#{key}
|
78
|
+
puts "You need to specify a file or folder '#{key} file_or_folder'"
|
79
79
|
exit(1)
|
80
80
|
elsif !file_exists?(filename) && should_exist && required
|
81
|
-
puts "This file doesn't exist '#{filename}'"
|
81
|
+
puts "This file/folder doesn't exist '#{filename}'"
|
82
82
|
exit(1)
|
83
83
|
end
|
84
84
|
|
data/lib/eco/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: eco-helpers
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.22
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oscar Segura
|
@@ -350,26 +350,6 @@ dependencies:
|
|
350
350
|
- - "<"
|
351
351
|
- !ruby/object:Gem::Version
|
352
352
|
version: '1.3'
|
353
|
-
- !ruby/object:Gem::Dependency
|
354
|
-
name: creek
|
355
|
-
requirement: !ruby/object:Gem::Requirement
|
356
|
-
requirements:
|
357
|
-
- - ">="
|
358
|
-
- !ruby/object:Gem::Version
|
359
|
-
version: 2.5.2
|
360
|
-
- - "<"
|
361
|
-
- !ruby/object:Gem::Version
|
362
|
-
version: '2.6'
|
363
|
-
type: :runtime
|
364
|
-
prerelease: false
|
365
|
-
version_requirements: !ruby/object:Gem::Requirement
|
366
|
-
requirements:
|
367
|
-
- - ">="
|
368
|
-
- !ruby/object:Gem::Version
|
369
|
-
version: 2.5.2
|
370
|
-
- - "<"
|
371
|
-
- !ruby/object:Gem::Version
|
372
|
-
version: '2.6'
|
373
353
|
description:
|
374
354
|
email:
|
375
355
|
- oscar@ecoportal.co.nz
|
@@ -412,6 +392,7 @@ files:
|
|
412
392
|
- lib/eco/api/common/people/default_parsers/policy_groups_parser.rb
|
413
393
|
- lib/eco/api/common/people/default_parsers/select_parser.rb
|
414
394
|
- lib/eco/api/common/people/default_parsers/send_invites_parser.rb
|
395
|
+
- lib/eco/api/common/people/default_parsers/xls_parser.rb
|
415
396
|
- lib/eco/api/common/people/entries.rb
|
416
397
|
- lib/eco/api/common/people/entry_factory.rb
|
417
398
|
- lib/eco/api/common/people/person_attribute_parser.rb
|