eco-helpers 2.0.21 → 2.0.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5525ed41d4f4b42d96eb0d71f02911e7cfb9112890aff984fb872a60d7515976
4
- data.tar.gz: d6dacbda91325cea0867c253b99458ae18b16e6450431ead852d686dd6e56859
3
+ metadata.gz: ba43099c7902b3e2623640a196abd86f39215403d28e7a614da78b2422629323
4
+ data.tar.gz: 7e3701cbeb7b3a0a65453dc002d6f4c5cb6e6ec4103e6cd0178c1af333d29d7d
5
5
  SHA512:
6
- metadata.gz: 1f4a164e153ac8e3d75bfc7024b2ee7c6c101a1703fb79b58a0d8c8fc31fc9978b6353c2a178a419c94ca22e6b49bf440364706642e7b087a4601f4cd7da3b51
7
- data.tar.gz: 9fdd7340a79b853b0dd51676e9ea2ba773928c68928b62d7781a525b2c27c1032ec7da04cae1f57afb7b290649350280d0aa5d0f2e7b91ba8a46cfe685bbeadb
6
+ metadata.gz: f14ab296c2cff2dc694ef9182593329ca1bc86cf08ca32ba879cd5e74d658d1d123add425bc62a981182b9e4da06b33d225d613cd1a9d9cbfd7284e6b6d24ea5
7
+ data.tar.gz: f020822ad4ae2e18f1f0424bb3a47417ff9a6bc78a439517cee3cd9212548efe4c89256579d73ccde27709dc5d2ea3d3e74bd072de23d4c089ccf3c7a107fc07
data/CHANGELOG.md CHANGED
@@ -1,7 +1,34 @@
1
1
  # Change Log
2
2
  All notable changes to this project will be documented in this file.
3
3
 
4
- ## [2.0.21] - 2021-06-0x
4
+ ## [2.0.22] - 2021-06-18
5
+
6
+ ### Added
7
+ - exposed `logger` in `BaseLoader` and
8
+ - support for multiple input files
9
+ * `Eco::API::Common::People::EntryFactory#entries`:
10
+ - refactored to allow multiple input files parsing
11
+ - moreover to `idx`, hash entries will get their `source_file`
12
+ * Input callback at `lib/eco/cli/config/default/input` refactored format detection and enabled folder input
13
+ * `SCR.get_file` language extended to also mention folder (not just file)
14
+ - support for `.xls` and `.xlsx` files
15
+ * `Eco::API::Common::People::DefaultParsers::XLSParser` the Excel files **parser**
16
+ * `Eco::API::Common::People::PersonParser` added `:xls` as an accepted format
17
+ * `Eco::API::Session#fields_mapper` exposed mapper through a method to allow **headers detection**
18
+ - The external names of the fields are the column headers of the input file
19
+ * `Eco::API::UseCases::BaseIO` when arguments validation rails, now it raises with specific `MissingParameter` error
20
+
21
+ ### Changed
22
+ - dry out `BaseLoader` (only session is set as instance variable)
23
+ - removed `creek` **dependency** (it was not used anywhere in the gem)
24
+ * we just kept `roo` and `roo-xls`
25
+ - custom `Error` classes now all inherit from `StandardError` (rather than `Exception`)
26
+
27
+
28
+ ### Fixed
29
+
30
+
31
+ ## [2.0.21] - 2021-06-04
5
32
 
6
33
  ### Added
7
34
  - `Eco::CSV::Table`, support to create the table out of an `Array<Hash>`
@@ -11,8 +38,6 @@ All notable changes to this project will be documented in this file.
11
38
  ### Changed
12
39
  - `Eco::API::Common::People::EntryFactory` slight **refactor** to boost better support for multiple input formats
13
40
 
14
- ### Fixed
15
-
16
41
 
17
42
  ## [2.0.20] - 2021-05-31
18
43
 
data/eco-helpers.gemspec CHANGED
@@ -42,5 +42,4 @@ Gem::Specification.new do |spec|
42
42
  spec.add_dependency 'jaro_winkler', '>= 1.5.4', '< 1.6'
43
43
  spec.add_dependency 'roo', '>= 2.8.3', '< 2.9'
44
44
  spec.add_dependency 'roo-xls', '>= 1.2.0', '< 1.3'
45
- spec.add_dependency 'creek', '>= 2.5.2', '< 2.6'
46
45
  end
@@ -51,15 +51,19 @@ module Eco
51
51
  private
52
52
 
53
53
  def session
54
- @session ||= ASSETS.session
54
+ ASSETS.session
55
55
  end
56
56
 
57
- def micro
58
- session.micro
57
+ def config
58
+ session.config
59
59
  end
60
60
 
61
- def config
62
- @config ||= ASSETS.config
61
+ def logger
62
+ session.logger
63
+ end
64
+
65
+ def micro
66
+ session.micro
63
67
  end
64
68
 
65
69
  end
@@ -44,3 +44,4 @@ require_relative 'default_parsers/freemium_parser'
44
44
  require_relative 'default_parsers/policy_groups_parser'
45
45
  require_relative 'default_parsers/login_providers_parser'
46
46
  require_relative 'default_parsers/csv_parser'
47
+ require_relative 'default_parsers/xls_parser'
@@ -0,0 +1,53 @@
1
+ class Eco::API::Common::People::DefaultParsers::XLSParser < Eco::API::Common::Loaders::Parser
2
+ attribute :xls
3
+
4
+ attr_accessor :already_required
5
+ attr_reader :file
6
+
7
+ def parser(file, deps)
8
+ @file = file
9
+ rows.tap {|r| @file = nil}
10
+ end
11
+
12
+ def serializer(array_hash, deps)
13
+ raise "Not implemented. TODO: using axlsx or rubyXL gems. See: https://spin.atomicobject.com/2017/03/22/parsing-excel-files-ruby/"
14
+ end
15
+
16
+ private
17
+
18
+ def headers
19
+ raise "You should implement this method"
20
+ end
21
+
22
+ def sheet_name
23
+ 0
24
+ end
25
+
26
+ def workbook
27
+ require_reading_libs!
28
+ Roo::Spreadsheet.open(file)
29
+ end
30
+
31
+ def spreadheet(name_or_index = sheet_name)
32
+ workbook.sheet(name_or_index)
33
+ end
34
+
35
+ def rows(target = headers)
36
+ begin
37
+ spreadheet.parse(header_search: target)
38
+ rescue Roo::HeaderRowNotFoundError => e
39
+ missing = JSON.parse(e.message)
40
+ logger.warn("The input file is missing these headers: #{missing}")
41
+ present = target - missing
42
+ rows(present)
43
+ end
44
+ end
45
+
46
+ def require_reading_libs!
47
+ return if already_required
48
+ require 'roo'
49
+ require 'roo-xls'
50
+ already_required = true
51
+ end
52
+
53
+ end
@@ -101,33 +101,47 @@ module Eco
101
101
  data = []
102
102
  content, file, encoding, format = kargs.values_at(:content, :file, :encoding, :format)
103
103
 
104
- content = get_file_content(file, format, encoding) if file
104
+ # Support for multiple file
105
+ if file.is_a?(Array)
106
+ return file.each_with_object([]) do |f, out|
107
+ logger.info("Parsing file '#{f}'")
108
+ curr = to_array_of_hashes(**kargs.merge(file: f))
109
+ out.concat(curr)
110
+ end
111
+ end
112
+ # Get content only when it's not :xls
113
+ # note: even if content was provided, file takes precedence
114
+ content = get_file_content(file, format, encoding) if (format != :xls) && file
105
115
 
106
116
  case content
107
- when !content
108
- logger.error("Could not obtain any data out of these: #{kargs}")
109
- exit(1)
110
117
  when Hash
111
118
  logger.error("Input data as 'Hash' not supported. Expecting 'Enumerable' or 'String'")
112
119
  exit(1)
113
120
  when String
114
- data = person_parser.parse(format, content).map.each_with_index do |entry_hash, i|
115
- j = (format == :csv)? i + 2 : i + 1
116
- entry_hash.tap {|hash| hash["idx"] = j}
117
- end
118
- to_array_of_hashes(content: data)
121
+ to_array_of_hashes(content: person_parser.parse(format, content))
119
122
  when Enumerable
120
123
  sample = content.to_a.first
121
124
  case sample
122
125
  when Hash, Array, ::CSV::Row
123
126
  Eco::CSV::Table.new(content).to_array_of_hashes
124
127
  else
125
- logger.error("Input 'Array' of '#{sample.class}' is not supported.")
128
+ logger.error("Input content 'Array' of '#{sample.class}' is not supported.")
126
129
  end
127
130
  else
128
- logger.error("Could not obtain any data out of content: '#{content.class}'")
129
- exit(1)
131
+ if file && format == :xls
132
+ person_parser.parse(format, file)
133
+ else
134
+ logger.error("Could not obtain any data out of these: #{kargs}. Given content: '#{content.class}'")
135
+ exit(1)
136
+ end
137
+ end.tap do |out_array|
138
+ start_from_two = (format == :csv) || format == :xls
139
+ out_array.each_with_index do |entry_hash, i|
140
+ entry_hash["idx"] = start_from_two ? i + 2 : i + 1
141
+ entry_hash["source_file"] = file
142
+ end
130
143
  end
144
+
131
145
  end
132
146
 
133
147
 
@@ -16,7 +16,7 @@ module Eco
16
16
  CORE_ATTRS = ["id", "external_id", "email", "name", "supervisor_id", "filter_tags", "freemium"]
17
17
  ACCOUNT_ATTRS = ["policy_group_ids", "default_tag", "send_invites", "landing_page_id", "login_provider_ids"]
18
18
  TYPE = [:select, :text, :date, :number, :phone_number, :boolean, :multiple]
19
- FORMAT = [:csv, :xml, :json]
19
+ FORMAT = [:csv, :xml, :json, :xls]
20
20
 
21
21
  attr_reader :schema
22
22
  attr_reader :details_attrs, :all_model_attrs
@@ -66,6 +66,16 @@ module Eco
66
66
  @presets_factory ||= Eco::API::Organization::PresetsFactory.new(enviro: enviro)
67
67
  end
68
68
 
69
+ # @return [Eco::Data::Mapper] the mappings between the internal and external attribute/property names.
70
+ def fields_mapper
71
+ return @fields_mapper if instance_variable_defined?(:@fields_mapper)
72
+ mappings = []
73
+ if map_file = config.people.fields_mapper
74
+ mappings = map_file ? file_manager.load_json(map_file) : []
75
+ end
76
+ @fields_mapper = Eco::Data::Mapper.new(mappings)
77
+ end
78
+
69
79
  # Helper to obtain a EntryFactory
70
80
  # @param schema [String, Ecoportal::API::V1::PersonSchema] `schema` to which associate the EntryFactory,
71
81
  # where `String` can be the _name_ or the _id_ of the schema.
@@ -79,15 +89,11 @@ module Eco
79
89
  return @entry_factories[schema&.id]
80
90
  end
81
91
 
82
- mappings = []
83
- if map_file = config.people.fields_mapper
84
- mappings = map_file ? file_manager.load_json(map_file) : []
85
- end
86
92
  @entry_factories[schema&.id] = Eco::API::Common::People::EntryFactory.new(
87
93
  enviro,
88
94
  schema: schema,
89
95
  person_parser: config.people.parser,
90
- attr_map: Eco::Data::Mapper.new(mappings)
96
+ attr_map: fields_mapper
91
97
  )
92
98
  end
93
99
 
@@ -2,7 +2,7 @@ module Eco
2
2
  module API
3
3
  class UseCases
4
4
 
5
- class UnkownCase < Exception
5
+ class UnkownCase < StandardError
6
6
  def initialize(msg = nil, case_name: nil, type: nil)
7
7
  msg ||= "Unkown case"
8
8
  msg += ". Case name '#{case_name}'" if case_name
@@ -11,7 +11,7 @@ module Eco
11
11
  end
12
12
  end
13
13
 
14
- class AmbiguousCaseReference < Exception
14
+ class AmbiguousCaseReference < StandardError
15
15
  def initialize(msg = nil, case_name: nil)
16
16
  msg ||= "You must specify type when there are multiple cases with same name"
17
17
  msg += ". Case name '#{case_name}'" if case_name
@@ -4,7 +4,7 @@ module Eco
4
4
  # Core class of UseCases. It basically defines and manages allowed `types`
5
5
  class BaseCase
6
6
 
7
- class InvalidType < Exception
7
+ class InvalidType < StandardError
8
8
  def initialize(msg = nil, type:, types:)
9
9
  msg ||= "Invalid type."
10
10
  msg = "Given type '#{type}'. Valid types: #{types}"
@@ -13,7 +13,7 @@ module Eco
13
13
  end
14
14
 
15
15
  extend Eco::API::Common::ClassHelpers
16
-
16
+
17
17
  @types = [:import, :filter, :transform, :sync, :error_handler, :export, :other]
18
18
 
19
19
  class << self
@@ -5,6 +5,19 @@ module Eco
5
5
  class BaseIO < BaseCase
6
6
  @types = BaseCase.types
7
7
 
8
+ class MissingParameter < StandardError
9
+ attr_reader :type, :required, :given
10
+
11
+ def initialize(msg = nil, type: nil, required:, given:)
12
+ @type = type
13
+ @required = required
14
+ @given = given
15
+ msg += " of type '#{type}'" if type
16
+ msg += " requires an object '#{required}'. Given: #{given}."
17
+ super(msg)
18
+ end
19
+ end
20
+
8
21
  class << self
9
22
  def input_required?(type)
10
23
  !valid_type?(type) || [:import, :sync].include?(type)
@@ -80,13 +93,13 @@ module Eco
80
93
  def validate_args(input:, people:, session:, options:)
81
94
  case
82
95
  when !session.is_a?(Eco::API::Session)
83
- raise "A UseCase needs a Session object. Given: #{session}"
96
+ raise MissingParameter.new("UseCase", required: :session, given: session.class)
84
97
  when input_required? && !input
85
- raise "UseCase of type '#{type}' requires a valid input. None given"
98
+ raise MissingParameter.new("UseCase", type: type, required: :input, given: input.class)
86
99
  when people_required? && !people.is_a?(Eco::API::Organization::People)
87
- raise "UseCase of type '#{type}' requires a People object. Given: #{people}"
100
+ raise MissingParameter.new("UseCase", type: type, required: :people, given: people.class)
88
101
  when !options || (options && !options.is_a?(Hash))
89
- raise "To inject dependencies via ':options' it should be a Hash object. Given: #{options}"
102
+ raise MissingParameter.new("Use Case options", required: :Hash, given: options.class)
90
103
  end
91
104
  true
92
105
  end
@@ -1,18 +1,71 @@
1
1
  ASSETS.cli.config do |cnf|
2
+ formats = {
3
+ csv: {
4
+ option: ["-csv"],
5
+ extname: [".csv", ".txt"]
6
+ },
7
+ xml: {
8
+ option: ["-xml"],
9
+ extname: [".xml"]
10
+ },
11
+ xls: {
12
+ option: ["-xls", "-xlsx", "-excel"],
13
+ extname: [".xls", ".xlsx", ".xlsm"]
14
+ },
15
+ json: {
16
+ option: ["-json"],
17
+ extname: [".json"]
18
+ }
19
+ }
20
+
2
21
  cnf.input(default_option: "-entries-from") do |session, str_opt, options|
3
22
  input = []
4
23
  if SCR.get_arg(str_opt)
5
24
  file = SCR.get_file(str_opt, required: true)
25
+
26
+ # Command line check
27
+ format = formats.reduce(nil) do |matched, (format, selectors)|
28
+ used = selectors[:option].reduce(false) {|used, option| SCR.get_arg(option) || used}
29
+ next matched if matched
30
+ next format if used
31
+ end
32
+
33
+ # File/Folder check
34
+ file = File.expand_path(file)
35
+ if File.directory?(file)
36
+ folder = file
37
+ file = Dir.glob("#{file}/*").reject {|f| File.directory?(f)}
38
+ ext = (format && formats[format][:extname]) || [File.extname(file.first)]
39
+ file = file.select {|f| ext.any? {|e| File.extname(f) == e}}.tap do |files|
40
+ if files.empty?
41
+ session.logger.error("Could not find any file with extension: #{ext} in folder '#{folder}'")
42
+ exit(1)
43
+ end
44
+ end
45
+ else
46
+ ext = File.extname(file)
47
+ end
48
+
49
+ format ||= formats.reduce(nil) do |matched, (format, selectors)|
50
+ next matched if matched
51
+ next format if selectors[:extname].any? {|e| ext == e}
52
+ end
53
+ format ||= :csv
54
+
6
55
  options.deep_merge!(input: {file: {name: file}})
7
- if SCR.get_arg("-xml")
8
- options.deep_merge!(input: {file: {format: :xml}})
9
- session.config.files.validate(:xml, file)
10
- input = session.entries(file: file, format: :xml)
11
- elsif SCR.get_arg("-json")
12
- options.deep_merge!(input: {file: {format: :json}})
13
- input = Eco::API::Organization::People.new(JSON.parse(File.read(file)))
56
+ options.deep_merge!(input: {file: {format: format}})
57
+
58
+ case format
59
+ when :xml
60
+ [file].flatten.each {|f| session.config.files.validate(:xml, f)}
61
+ input = session.entries(file: file, format: format)
62
+ when :xls
63
+ input = session.entries(file: file, format: format)
64
+ when :json
65
+ input = [file].flatten.reduce(Eco::API::Organization::People.new([])) do |people, file|
66
+ people.merge(JSON.parse(File.read(file)))
67
+ end
14
68
  else
15
- options.deep_merge!(input: {file: {format: :csv}})
16
69
  input = session.csv_entries(file)
17
70
  end
18
71
  end
@@ -41,7 +41,7 @@ ASSETS.cli.config do |cnf|
41
41
  session.schema = sch_id
42
42
  end
43
43
 
44
- desc = "Used to be used to specify the input file when using -get-partial. "
44
+ desc = "Used to be used to specify the input file or folder when using -get-partial."
45
45
  desc += "It can also be useful to obtain `-get-partial` of people base on `:export` use cases (i.e. -people-to-csv)"
46
46
  options_set.add("-entries-from", desc) do |options, session|
47
47
  options.deep_merge!(input: {entries_from: true})
@@ -103,7 +103,7 @@ ASSETS.cli.config do |config|
103
103
  if !io.options[:dry_run] && partial_update
104
104
  # get target people afresh
105
105
  people = io.session.micro.people_refresh(people: io.people, include_created: true)
106
- io = io.new(people: people)
106
+ io = io.base.new(people: people)
107
107
  else
108
108
  wf_post.skip!
109
109
  msg = "Although there are post_launch cases, they will NOT be RUN"
@@ -75,10 +75,10 @@ module Eco
75
75
  def get_file(key, required: false, should_exist: true)
76
76
  filename = get_arg(key, with_param: true)
77
77
  if !filename && required
78
- puts "You need to specify a file '#{key} file'"
78
+ puts "You need to specify a file or folder '#{key} file_or_folder'"
79
79
  exit(1)
80
80
  elsif !file_exists?(filename) && should_exist && required
81
- puts "This file doesn't exist '#{filename}'"
81
+ puts "This file/folder doesn't exist '#{filename}'"
82
82
  exit(1)
83
83
  end
84
84
 
data/lib/eco/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Eco
2
- VERSION = "2.0.21"
2
+ VERSION = "2.0.22"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: eco-helpers
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.21
4
+ version: 2.0.22
5
5
  platform: ruby
6
6
  authors:
7
7
  - Oscar Segura
@@ -350,26 +350,6 @@ dependencies:
350
350
  - - "<"
351
351
  - !ruby/object:Gem::Version
352
352
  version: '1.3'
353
- - !ruby/object:Gem::Dependency
354
- name: creek
355
- requirement: !ruby/object:Gem::Requirement
356
- requirements:
357
- - - ">="
358
- - !ruby/object:Gem::Version
359
- version: 2.5.2
360
- - - "<"
361
- - !ruby/object:Gem::Version
362
- version: '2.6'
363
- type: :runtime
364
- prerelease: false
365
- version_requirements: !ruby/object:Gem::Requirement
366
- requirements:
367
- - - ">="
368
- - !ruby/object:Gem::Version
369
- version: 2.5.2
370
- - - "<"
371
- - !ruby/object:Gem::Version
372
- version: '2.6'
373
353
  description:
374
354
  email:
375
355
  - oscar@ecoportal.co.nz
@@ -412,6 +392,7 @@ files:
412
392
  - lib/eco/api/common/people/default_parsers/policy_groups_parser.rb
413
393
  - lib/eco/api/common/people/default_parsers/select_parser.rb
414
394
  - lib/eco/api/common/people/default_parsers/send_invites_parser.rb
395
+ - lib/eco/api/common/people/default_parsers/xls_parser.rb
415
396
  - lib/eco/api/common/people/entries.rb
416
397
  - lib/eco/api/common/people/entry_factory.rb
417
398
  - lib/eco/api/common/people/person_attribute_parser.rb