eco-helpers 2.0.21 → 2.0.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5525ed41d4f4b42d96eb0d71f02911e7cfb9112890aff984fb872a60d7515976
-  data.tar.gz: d6dacbda91325cea0867c253b99458ae18b16e6450431ead852d686dd6e56859
+  metadata.gz: ba43099c7902b3e2623640a196abd86f39215403d28e7a614da78b2422629323
+  data.tar.gz: 7e3701cbeb7b3a0a65453dc002d6f4c5cb6e6ec4103e6cd0178c1af333d29d7d
 SHA512:
-  metadata.gz: 1f4a164e153ac8e3d75bfc7024b2ee7c6c101a1703fb79b58a0d8c8fc31fc9978b6353c2a178a419c94ca22e6b49bf440364706642e7b087a4601f4cd7da3b51
-  data.tar.gz: 9fdd7340a79b853b0dd51676e9ea2ba773928c68928b62d7781a525b2c27c1032ec7da04cae1f57afb7b290649350280d0aa5d0f2e7b91ba8a46cfe685bbeadb
+  metadata.gz: f14ab296c2cff2dc694ef9182593329ca1bc86cf08ca32ba879cd5e74d658d1d123add425bc62a981182b9e4da06b33d225d613cd1a9d9cbfd7284e6b6d24ea5
+  data.tar.gz: f020822ad4ae2e18f1f0424bb3a47417ff9a6bc78a439517cee3cd9212548efe4c89256579d73ccde27709dc5d2ea3d3e74bd072de23d4c089ccf3c7a107fc07

data/CHANGELOG.md

@@ -1,7 +1,34 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
-## [2.0.21] - 2021-06-0x
+## [2.0.22] - 2021-06-18
+
+### Added
+  - exposed `logger` in `BaseLoader` and
+  - support for multiple input files
+    * `Eco::API::Common::People::EntryFactory#entries`:
+      - refactored to allow multiple input files parsing
+      - moreover to `idx`, hash entries will get their `source_file`
+    * Input callback at `lib/eco/cli/config/default/input` refactored format detection and enabled folder input
+    * `SCR.get_file` language extended to also mention folder (not just file)
+  - support for `.xls` and `.xlsx` files
+    * `Eco::API::Common::People::DefaultParsers::XLSParser` the Excel files **parser**
+    * `Eco::API::Common::People::PersonParser` added `:xls` as an accepted format
+    * `Eco::API::Session#fields_mapper` exposed mapper through a method to allow **headers detection**
+      - The external names of the fields are the column headers of the input file
+    * `Eco::API::UseCases::BaseIO` when arguments validation rails, now it raises with specific `MissingParameter` error
+
+### Changed
+  - dry out `BaseLoader` (only session is set as instance variable)
+  - removed `creek` **dependency** (it was not used anywhere in the gem)
+    * we just kept `roo` and `roo-xls`
+  - custom `Error` classes now all inherit from `StandardError` (rather than `Exception`)
+
+
+### Fixed
+
+
+## [2.0.21] - 2021-06-04
 
 ### Added
   - `Eco::CSV::Table`, support to create the table out of an `Array<Hash>`
@@ -11,8 +38,6 @@ All notable changes to this project will be documented in this file.
 ### Changed
   - `Eco::API::Common::People::EntryFactory` slight **refactor** to boost better support for multiple input formats
 
-### Fixed
-
 
 ## [2.0.20] - 2021-05-31
 

data/eco-helpers.gemspec

@@ -42,5 +42,4 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'jaro_winkler',  '>= 1.5.4', '< 1.6'
   spec.add_dependency 'roo',           '>= 2.8.3', '< 2.9'
   spec.add_dependency 'roo-xls',       '>= 1.2.0', '< 1.3'
-  spec.add_dependency 'creek',         '>= 2.5.2', '< 2.6'
 end

data/lib/eco/api/common/base_loader.rb

@@ -51,15 +51,19 @@ module Eco
         private
 
         def session
-          @session ||= ASSETS.session
+          ASSETS.session
         end
 
-        def micro
-          session.micro
+        def config
+          session.config
         end
 
-        def config
-          @config ||= ASSETS.config
+        def logger
+          session.logger
+        end
+
+        def micro
+          session.micro
         end
 
       end

data/lib/eco/api/common/people/default_parsers.rb

@@ -44,3 +44,4 @@ require_relative 'default_parsers/freemium_parser'
 require_relative 'default_parsers/policy_groups_parser'
 require_relative 'default_parsers/login_providers_parser'
 require_relative 'default_parsers/csv_parser'
+require_relative 'default_parsers/xls_parser'

data/lib/eco/api/common/people/default_parsers/xls_parser.rb

@@ -0,0 +1,53 @@
+class Eco::API::Common::People::DefaultParsers::XLSParser < Eco::API::Common::Loaders::Parser
+  attribute :xls
+
+  attr_accessor :already_required
+  attr_reader   :file
+
+  def parser(file, deps)
+    @file = file
+    rows.tap {|r| @file = nil}
+  end
+
+  def serializer(array_hash, deps)
+    raise "Not implemented. TODO: using axlsx or rubyXL gems. See: https://spin.atomicobject.com/2017/03/22/parsing-excel-files-ruby/"
+  end
+
+  private
+
+  def headers
+    raise "You should implement this method"
+  end
+
+  def sheet_name
+    0
+  end
+
+  def workbook
+    require_reading_libs!
+    Roo::Spreadsheet.open(file)
+  end
+
+  def spreadheet(name_or_index = sheet_name)
+    workbook.sheet(name_or_index)
+  end
+
+  def rows(target = headers)
+    begin
+      spreadheet.parse(header_search: target)
+    rescue Roo::HeaderRowNotFoundError => e
+      missing  = JSON.parse(e.message)
+      logger.warn("The input file is missing these headers: #{missing}")
+      present = target - missing
+      rows(present)
+    end
+  end
+
+  def require_reading_libs!
+    return if already_required
+    require 'roo'
+    require 'roo-xls'
+    already_required = true
+  end
+
+end

data/lib/eco/api/common/people/entry_factory.rb

@@ -101,33 +101,47 @@ module Eco
             data = []
             content, file, encoding, format = kargs.values_at(:content, :file, :encoding, :format)
 
-            content = get_file_content(file, format, encoding) if file
+            # Support for multiple file
+            if file.is_a?(Array)
+              return file.each_with_object([]) do |f, out|
+                logger.info("Parsing file '#{f}'")
+                curr = to_array_of_hashes(**kargs.merge(file: f))
+                out.concat(curr)
+              end
+            end
+            # Get content only when it's not :xls
+            # note: even if content was provided, file takes precedence
+            content = get_file_content(file, format, encoding) if (format != :xls) && file
 
             case content
-            when !content
-              logger.error("Could not obtain any data out of these: #{kargs}")
-              exit(1)
             when Hash
               logger.error("Input data as 'Hash' not supported. Expecting 'Enumerable' or 'String'")
               exit(1)
             when String
-              data = person_parser.parse(format, content).map.each_with_index do |entry_hash, i|
-                j = (format == :csv)? i + 2 : i + 1
-                entry_hash.tap {|hash| hash["idx"] = j}
-              end
-              to_array_of_hashes(content: data)
+              to_array_of_hashes(content: person_parser.parse(format, content))
             when Enumerable
               sample = content.to_a.first
               case sample
               when Hash, Array, ::CSV::Row
                 Eco::CSV::Table.new(content).to_array_of_hashes
               else
-                logger.error("Input 'Array' of '#{sample.class}' is not supported.")
+                logger.error("Input content 'Array' of '#{sample.class}' is not supported.")
               end
             else
-              logger.error("Could not obtain any data out of content: '#{content.class}'")
-              exit(1)
+              if file && format == :xls
+                person_parser.parse(format, file)
+              else
+                logger.error("Could not obtain any data out of these: #{kargs}. Given content: '#{content.class}'")
+                exit(1)
+              end
+            end.tap do |out_array|
+              start_from_two = (format == :csv) || format == :xls
+              out_array.each_with_index do |entry_hash, i|
+                entry_hash["idx"] = start_from_two ? i + 2 : i + 1
+                entry_hash["source_file"] = file
+              end
             end
+
           end
 
 

data/lib/eco/api/common/people/person_parser.rb

@@ -16,7 +16,7 @@ module Eco
           CORE_ATTRS    = ["id", "external_id", "email", "name", "supervisor_id", "filter_tags", "freemium"]
           ACCOUNT_ATTRS = ["policy_group_ids", "default_tag", "send_invites", "landing_page_id", "login_provider_ids"]
           TYPE          = [:select, :text, :date, :number, :phone_number, :boolean, :multiple]
-          FORMAT        = [:csv, :xml, :json]
+          FORMAT        = [:csv, :xml, :json, :xls]
 
           attr_reader :schema
           attr_reader :details_attrs, :all_model_attrs

data/lib/eco/api/session.rb

@@ -66,6 +66,16 @@ module Eco
         @presets_factory ||= Eco::API::Organization::PresetsFactory.new(enviro: enviro)
       end
 
+      # @return [Eco::Data::Mapper] the mappings between the internal and external attribute/property names.
+      def fields_mapper
+        return @fields_mapper if instance_variable_defined?(:@fields_mapper)
+        mappings = []
+        if map_file = config.people.fields_mapper
+          mappings = map_file ? file_manager.load_json(map_file) : []
+        end
+        @fields_mapper = Eco::Data::Mapper.new(mappings)
+      end
+
       # Helper to obtain a EntryFactory
       # @param schema [String, Ecoportal::API::V1::PersonSchema] `schema` to which associate the EntryFactory,
       #  where `String` can be the _name_ or the _id_ of the schema.
@@ -79,15 +89,11 @@ module Eco
           return @entry_factories[schema&.id]
         end
 
-        mappings = []
-        if map_file = config.people.fields_mapper
-          mappings = map_file ? file_manager.load_json(map_file) : []
-        end
         @entry_factories[schema&.id] = Eco::API::Common::People::EntryFactory.new(
           enviro,
           schema:        schema,
           person_parser: config.people.parser,
-          attr_map:      Eco::Data::Mapper.new(mappings)
+          attr_map:      fields_mapper
         )
       end
 

data/lib/eco/api/usecases.rb

@@ -2,7 +2,7 @@ module Eco
   module API
     class UseCases
 
-      class UnkownCase < Exception
+      class UnkownCase < StandardError
         def initialize(msg = nil, case_name: nil, type: nil)
           msg ||= "Unkown case"
           msg += ". Case name '#{case_name}'" if case_name
@@ -11,7 +11,7 @@ module Eco
         end
       end
 
-      class AmbiguousCaseReference < Exception
+      class AmbiguousCaseReference < StandardError
         def initialize(msg = nil, case_name: nil)
           msg ||= "You must specify type when there are multiple cases with same name"
           msg += ". Case name '#{case_name}'" if case_name

data/lib/eco/api/usecases/base_case.rb

@@ -4,7 +4,7 @@ module Eco
       # Core class of UseCases. It basically defines and manages allowed `types`
       class BaseCase
 
-        class InvalidType < Exception
+        class InvalidType < StandardError
           def initialize(msg = nil, type:, types:)
             msg ||= "Invalid type."
             msg = "Given type '#{type}'. Valid types: #{types}"
@@ -13,7 +13,7 @@ module Eco
         end
 
         extend Eco::API::Common::ClassHelpers
-        
+
         @types = [:import, :filter, :transform, :sync, :error_handler, :export, :other]
 
         class << self

data/lib/eco/api/usecases/base_io.rb

@@ -5,6 +5,19 @@ module Eco
       class BaseIO < BaseCase
         @types  = BaseCase.types
 
+        class MissingParameter < StandardError
+          attr_reader :type, :required, :given
+
+          def initialize(msg = nil, type: nil, required:, given:)
+            @type     = type
+            @required = required
+            @given    = given
+            msg += " of type '#{type}'" if type
+            msg += " requires an object '#{required}'. Given: #{given}."
+            super(msg)
+          end
+        end
+
         class << self
           def input_required?(type)
             !valid_type?(type) || [:import, :sync].include?(type)
@@ -80,13 +93,13 @@ module Eco
         def validate_args(input:, people:, session:, options:)
           case
           when !session.is_a?(Eco::API::Session)
-            raise "A UseCase needs a Session object. Given: #{session}"
+            raise MissingParameter.new("UseCase", required: :session, given: session.class)
           when input_required? && !input
-            raise "UseCase of type '#{type}' requires a valid input. None given"
+            raise MissingParameter.new("UseCase", type: type, required: :input, given: input.class)
           when people_required? && !people.is_a?(Eco::API::Organization::People)
-            raise "UseCase of type '#{type}' requires a People object. Given: #{people}"
+            raise MissingParameter.new("UseCase", type: type, required: :people, given: people.class)
           when !options || (options && !options.is_a?(Hash))
-            raise "To inject dependencies via ':options' it should be a Hash object. Given: #{options}"
+            raise MissingParameter.new("Use Case options", required: :Hash, given: options.class)
           end
           true
         end

data/lib/eco/cli/config/default/input.rb

@@ -1,18 +1,71 @@
 ASSETS.cli.config do |cnf|
+  formats = {
+    csv: {
+      option: ["-csv"],
+      extname: [".csv", ".txt"]
+    },
+    xml: {
+      option: ["-xml"],
+      extname: [".xml"]
+    },
+    xls:  {
+      option: ["-xls", "-xlsx", "-excel"],
+      extname: [".xls", ".xlsx", ".xlsm"]
+    },
+    json: {
+      option: ["-json"],
+      extname: [".json"]
+    }
+  }
+
   cnf.input(default_option: "-entries-from") do |session, str_opt, options|
     input = []
     if SCR.get_arg(str_opt)
       file = SCR.get_file(str_opt, required: true)
+
+      # Command line check
+      format = formats.reduce(nil) do |matched, (format, selectors)|
+        used = selectors[:option].reduce(false) {|used, option| SCR.get_arg(option) || used}
+        next matched if matched
+        next format  if used
+      end
+
+      # File/Folder check
+      file = File.expand_path(file)
+      if File.directory?(file)
+        folder = file
+        file   = Dir.glob("#{file}/*").reject {|f| File.directory?(f)}
+        ext    = (format && formats[format][:extname]) || [File.extname(file.first)]
+        file   = file.select {|f| ext.any? {|e| File.extname(f) == e}}.tap do |files|
+          if files.empty?
+            session.logger.error("Could not find any file with extension: #{ext} in folder '#{folder}'")
+            exit(1)
+          end
+        end
+      else
+        ext    = File.extname(file)
+      end
+
+      format ||= formats.reduce(nil) do |matched, (format, selectors)|
+        next matched if matched
+        next format  if selectors[:extname].any? {|e| ext == e}
+      end
+      format ||= :csv
+
       options.deep_merge!(input: {file: {name: file}})
-      if SCR.get_arg("-xml")
-        options.deep_merge!(input: {file: {format: :xml}})
-        session.config.files.validate(:xml, file)
-        input = session.entries(file: file, format: :xml)
-      elsif SCR.get_arg("-json")
-        options.deep_merge!(input: {file: {format: :json}})
-        input = Eco::API::Organization::People.new(JSON.parse(File.read(file)))
+      options.deep_merge!(input: {file: {format: format}})
+
+      case format
+      when :xml
+        [file].flatten.each {|f| session.config.files.validate(:xml, f)}
+        input = session.entries(file: file, format: format)
+      when :xls
+        input = session.entries(file: file, format: format)
+      when :json
+        input = [file].flatten.reduce(Eco::API::Organization::People.new([])) do |people, file|
+          people.merge(JSON.parse(File.read(file)))
+        end
       else
-        options.deep_merge!(input: {file: {format: :csv}})
         input = session.csv_entries(file)
       end
     end

data/lib/eco/cli/config/default/options.rb

@@ -41,7 +41,7 @@ ASSETS.cli.config do |cnf|
       session.schema = sch_id
     end
 
-    desc  = "Used to be used to specify the input file when using -get-partial. "
+    desc  = "Used to be used to specify the input file or folder when using -get-partial."
     desc += "It can also be useful to obtain `-get-partial` of people base on `:export` use cases (i.e. -people-to-csv)"
     options_set.add("-entries-from", desc) do |options, session|
       options.deep_merge!(input: {entries_from: true})

data/lib/eco/cli/config/default/workflow.rb

@@ -103,7 +103,7 @@ ASSETS.cli.config do |config|
         if !io.options[:dry_run] && partial_update
           # get target people afresh
           people = io.session.micro.people_refresh(people: io.people, include_created: true)
-          io     = io.new(people: people)
+          io     = io.base.new(people: people)
         else
           wf_post.skip!
           msg = "Although there are post_launch cases, they will NOT be RUN"

data/lib/eco/cli/scripting/args_helpers.rb

@@ -75,10 +75,10 @@ module Eco
         def get_file(key, required: false, should_exist: true)
           filename = get_arg(key, with_param: true)
           if !filename && required
-            puts "You need to specify a file '#{key} file'"
+            puts "You need to specify a file or folder '#{key} file_or_folder'"
             exit(1)
           elsif !file_exists?(filename) && should_exist && required
-            puts "This file doesn't exist '#{filename}'"
+            puts "This file/folder doesn't exist '#{filename}'"
             exit(1)
           end
 

data/lib/eco/version.rb

@@ -1,3 +1,3 @@
 module Eco
-  VERSION = "2.0.21"
+  VERSION = "2.0.22"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eco-helpers
 version: !ruby/object:Gem::Version
-  version: 2.0.21
+  version: 2.0.22
 platform: ruby
 authors:
 - Oscar Segura
@@ -350,26 +350,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.3'
-- !ruby/object:Gem::Dependency
-  name: creek
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.5.2
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '2.6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.5.2
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '2.6'
 description: 
 email:
 - oscar@ecoportal.co.nz
@@ -412,6 +392,7 @@ files:
 - lib/eco/api/common/people/default_parsers/policy_groups_parser.rb
 - lib/eco/api/common/people/default_parsers/select_parser.rb
 - lib/eco/api/common/people/default_parsers/send_invites_parser.rb
+- lib/eco/api/common/people/default_parsers/xls_parser.rb
 - lib/eco/api/common/people/entries.rb
 - lib/eco/api/common/people/entry_factory.rb
 - lib/eco/api/common/people/person_attribute_parser.rb