eco-helpers 1.0.11 → 1.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3d434c6d214ef71d15aa9c2d334b9e5545c057f8fd9cb702f27080ad7b96a57
-  data.tar.gz: 35d9e89911c0aee5f0550eaabcaa6aecdcfd5a6bf97f6d8a53db416fcc957535
+  metadata.gz: b7816ce8532e8e6f2214b58e5fa94ecb65ec9bd86cc4efa0691001fec78cfb7e
+  data.tar.gz: '0539d3a69ced5ad94f0245e2ccc768818534344b46c6d455af358c5e84ababfd'
 SHA512:
-  metadata.gz: 24eb7ad2d08a96c2a171e1b5bd41d988f561bbafc764f91df3152a36dec887c456600356fab112d7b95f476eccd742f187a9f3adca19fc05464a85cf34e95a74
-  data.tar.gz: 05a8af94fcf4e966868c447d0e907424c9e615ad942126fc50e6ed53abb7025098edbaea922c060b2e87cadde1e0296dc54f51ed6d71a1198fece5e1753c0766
+  metadata.gz: 965d612b80244b63cfae420eaa97a28ec2935d1299dfaec5d9220a4de732442904ea6995e0536f559ab49bb6a52af0d71fee7d1a69209c0a1d92c6317f24c811
+  data.tar.gz: a0ea67d65af9e01a36395da5585b1226ea4bcf70a0b71f7ab0718fd4ae9e31cb2d4cf50f95dec1c5267fd338d1816ed2985a1a1bf2f55265d69339f895531aee

data/eco-helpers.gemspec

@@ -32,6 +32,8 @@ Gem::Specification.new do |s|
   s.add_dependency 'aws-sdk-s3',    '~> 1.30', '>= 1.30.1'
   s.add_dependency 'aws-sdk-ses',   '~> 1.14', '>= 1.14.0'
   s.add_dependency 'dotenv',        '~> 2.6',  '>= 2.6.0'
+  s.add_dependency 'net-sftp',      '~> 2.1',  '>= 2.1.2'
   s.add_dependency 'faker',         '~> 2',    '>= 2.1'
   s.add_dependency 'distribution',  '~> 0.7',  '>= 0.7.3'
+
 end

data/lib/eco/api/common/people/default_parsers.rb

@@ -2,6 +2,8 @@ module Eco
   module API
     module Common
       module People
+
+        # Class to define a set of default attribute parsers
         class DefaultParsers < PersonParser
 
           def initialize(*args)

data/lib/eco/api/common/people/default_parsers/csv_parser.rb

@@ -11,7 +11,8 @@ module Eco
                   arr_hash = []
                   CSV.parse(data, headers: true).each do |row|
                     row_hash = row.headers.uniq.each_with_object({}) do |attr, hash|
-                      hash[attr] = row[attr]
+                      value      = row[attr]
+                      hash[attr] = value.to_s.empty?? nil : value
                     end
                     arr_hash.push(row_hash)
                   end

data/lib/eco/api/common/people/entries.rb

@@ -27,6 +27,8 @@ module Eco
             attr('external_id', *args).first
           end
 
+          # Helper to dump the entries into a CSV
+          # @param filename [String] the destination file
           def export(filename)
             CSV.open(filename, "w") do |csv|
               entry  = self.first
@@ -38,6 +40,7 @@ module Eco
             end
           end
 
+          # Search function to find an `entry` based on one of different options
           def entry(id: nil, external_id: nil, email: nil)
             init_caches
             pers = nil
@@ -48,6 +51,8 @@ module Eco
             pers
           end
 
+          # Search function to find an `entry` based on one of different options
+          # see Eco::API::Common::People::Entries#entry
           def find(object)
             id          = object.respond_to?("id")?          object.send("id") : nil
             external_id = object.respond_to?("external_id")? object.send("external_id") : nil

data/lib/eco/api/common/people/entry_factory.rb

@@ -21,13 +21,17 @@ module Eco
 
             @source_person_parser = person_parser
 
+            # load default parsers
             @person_parser = Eco::API::Common::People::DefaultParsers.new(schema: @schema)
+            # merge the custom parsers
             @person_parser = @person_parser.merge(@source_person_parser)
             @person_parser_patch_version = @source_person_parser.patch_version
 
             @attr_map = attr_map
           end
 
+          # provides with a Eco::API::Common::People::PersonParser object (collection of attribute parsers)
+          # @note if the custom person parser has changed, it updates the copy of this EntryFactory instance
           def person_parser
             if @person_parser_patch_version < @source_person_parser.patch_version
               @person_parser.merge(@source_person_parser)

data/lib/eco/api/common/session.rb

@@ -9,6 +9,7 @@ end
 
 require_relative 'session/logger'
 require_relative 'session/mailer'
+require_relative 'session/sftp'
 require_relative 'session/s3_uploader'
 require_relative 'session/file_manager'
 require_relative 'session/environment'

data/lib/eco/api/common/session/base_session.rb

@@ -40,6 +40,18 @@ module Eco
             enviro.file_manager
           end
 
+          def mailer
+            enviro.mailer
+          end
+
+          def sftp
+            enviro.sftp
+          end
+          
+          def s3uploader
+            enviro.s3uploader
+          end
+
           def logger
             enviro.logger
           end

data/lib/eco/api/common/session/environment.rb

@@ -9,7 +9,7 @@ module Eco
           attr_reader :config, :session
           attr_reader :api #, :host, :version
           attr_reader :file_manager, :logger
-          attr_reader :mailer, :s3uploader
+          attr_reader :mailer, :sftp, :s3uploader
 
           alias_method :fm, :file_manager
 
@@ -31,6 +31,10 @@ module Eco
             @mailer      ||= Eco::API::Common::Session::Mailer.new(enviro: self)
           end
 
+          def sftp
+            @sftp        ||= Eco::API::Common::Session::SFTP.new(enviro: self)
+          end
+          
           def s3upoader
             @s3uploader  ||= Eco::API::Common::Session::S3Uploader.new(enviro: self)
           end

data/lib/eco/api/common/session/sftp.rb

@@ -0,0 +1,108 @@
+require "net/sftp"
+require 'dotenv/load'
+
+module Eco
+  module API
+    module Common
+      module Session
+        class SFTP
+
+          def initialize (enviro:)
+            raise "Required Environment object (enviro:). Given: #{enviro}" if enviro && !enviro.is_a?(Eco::API::Common::Session::Environment)
+            @enviro = enviro
+          end
+
+          # @see Net::SFTP::Session
+          def sftp_session
+            begin
+              @sftp_session ||= Net::SFTP.start(
+                fetch_host,
+                fetch_user,
+                keys:            fetch_key_files,
+                keys_only:       true,
+                non_interactive: true
+              )
+            rescue Exception => e
+              logger.error("Could not open SFTP session. Possible misconfiguration: #{e}")
+              exit(1)
+            end
+            @sftp_session
+          end
+
+          # @see Net::SFTP::Operations::Dir#entries
+          def entries(path)
+            sftp_session.dir.entries(path).sort_by {|rf| rf.name}
+          end
+
+          # **Files** of the remote directory.
+          # @see Net::SFTP::Operations::Dir#entries
+          # @param path [String] remote directory `path`
+          # @param pattern [Regexp] if given, filters by using this pattern
+          # @return [Array<Net::SFTP::Protocol::V01::Name>]
+          def files(path, pattern: nil)
+            entries = entries(path).select {|remote_file| remote_file.file?}
+            return entries unless pattern
+            entries.select {|remote_file| remote_file.name =~ pattern}
+          end
+
+          # **Folders** of the remote directory.
+          # @see Net::SFTP::Operations::Dir#entries
+          # @param path [String] remote directory `path`
+          # @param pattern [Regexp] if given, filters by using this pattern
+          # @return [Array<Net::SFTP::Protocol::V01::Name>]
+          def folders(path, pattern: nil)
+            entries = entries(path).select {|remote_file| remote_file.directory?}
+            return entries unless pattern
+            entries.select {|remote_file| remote_file.name =~ pattern}
+          end
+
+          # @see Net::SFTP::Session#rename
+          def move(fullname_source, fullname_dest, flags=nil, &callback)
+            sftp_session.rename!(fullname_source, fullname_dest, flags, &callback)
+          end
+
+          # Downloads the files specified to a local folder
+          # @see Net::SFTP::Operations::Download
+          # @param files [String, Array<String>] full path to remote file(s) to be downloaded
+          # @param local_folder [String] local destination folder (`"."` if not specified)
+          def download(files, local_folder: nil)
+            [files].flatten.compact.map do |fullname|
+              dest_fullname = File.join(local_folder || ".", File.basename(fullname))
+              sftp_session.download(fullname, dest_fullname)
+            end.each do |dw|
+              # run SSH event loop while dw.active?
+              dw.wait
+            end
+          end
+
+          private
+
+          def logger
+            @enviro&.logger || ::Logger.new(IO::NULL)
+          end
+
+          def config
+            @enviro.config || {}
+          end
+
+          def fetch_host
+            config.sftp.host || ENV['SFTP_HOST']
+          end
+
+          def fetch_user
+            config.sftp.user || ENV['SFTP_USERNAME']
+          end
+
+          def fetch_key_files
+            [config.sftp.key_file || ENV['SFTP_KEY_FILE']]
+          end
+
+          def fetch_base_path
+            config.sftp.base_path || ENV['SFTP_BASE_PATH']
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/session.rb

@@ -42,7 +42,7 @@ module Eco
       def mail_to(**kargs)
         mail.mail(**kargs)
       end
-
+      
       def s3upload(content: nil, file: nil, directory: nil)
         if content && file
           s3uploader.upload(file, content)

data/lib/eco/api/session/config.rb

@@ -28,6 +28,10 @@ module Eco
           self["apis"]       ||= Eco::API::Session::Config::Apis.new(config: self)
         end
 
+        def sftp
+          self["sftp"]       ||= Eco::API::Session::Config::SFTP.new(config: self)
+        end
+
         def logger
           self["logger"]     ||= Eco::API::Session::Config::Logger.new(config: self)
         end
@@ -108,6 +112,10 @@ module Eco
           self
         end
 
+        def active_enviro
+          apis.active_root_name
+        end
+
         def api(logger = ::Logger.new(IO::NULL))
           apis.api(logger)
         end
@@ -294,6 +302,7 @@ require_relative 'config/api'
 require_relative 'config/apis'
 require_relative 'config/logger'
 require_relative 'config/mailer'
+require_relative 'config/sftp'
 require_relative 'config/s3_storage'
 require_relative 'config/files'
 require_relative 'config/people'

data/lib/eco/api/session/config/sftp.rb

@@ -0,0 +1,59 @@
+module Eco
+  module API
+    class Session
+      class Config
+        class SFTP < BaseConfig
+
+          def host=(value)
+            self["host"] = value
+          end
+
+          def host
+            self["host"]
+          end
+
+          def user=(value)
+            self["user"] = value
+          end
+
+          def user
+            self["user"]
+          end
+
+          def key_file=(key)
+            self["key_file"] = key
+          end
+
+          def key_file
+            self["key_file"]
+          end
+
+          def base_path=(path)
+            self["base_path"] = path
+          end
+
+          def base_path
+            self["base_path"]
+          end
+
+          def enviro_subpaths=(hash)
+            self["enviro_subpaths"] = hash
+          end
+
+          def enviro_subpaths
+            self["enviro_subpaths"]
+          end
+
+          def enviro_subpath
+            enviro_subpaths[config.active_enviro]
+          end
+
+          def remote_folder
+            base_path + "/" + enviro_subpath
+          end
+          
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/usecases/base_case.rb

@@ -1,6 +1,7 @@
 module Eco
   module API
     class UseCases
+      # Core class of UseCases. It basically defines and manages allowed `types`
       class BaseCase
 
         class InvalidType < Exception
@@ -11,7 +12,7 @@ module Eco
           end
         end
 
-        @types = [:import, :filter, :transform, :sync, :export]
+        @types = [:import, :filter, :transform, :sync, :export, :other]
 
         class << self
           attr_reader :types

data/lib/eco/api/usecases/base_io.rb

@@ -1,8 +1,9 @@
 module Eco
   module API
     class UseCases
+      # Basic class to manage InputOuput for usecases
       class BaseIO < BaseCase
-        @types  = [:import, :filter, :transform, :sync, :export]
+        @types  = BaseCase.types
 
         class << self
           def input_required?(type)

data/lib/eco/api/usecases/use_case.rb

@@ -2,7 +2,7 @@ module Eco
   module API
     class UseCases
       class UseCase < BaseCase
-        @types = [:import, :filter, :transform, :sync, :export]
+        @types = BaseCase.types
 
         attr_reader :name, :type, :times_launched
         attr_reader :options

data/lib/eco/api/usecases/use_case_chain.rb

@@ -1,12 +1,13 @@
 module Eco
   module API
     class UseCases
+      # Class that enables to chain multiple UseCase
       class UseCaseChain < UseCase
+        @types  = UseCase.types
+
         MAX_CHAINS   = 70
         @@num_chains = 0
 
-        @types  = [:import, :filter, :transform, :sync, :export]
-
         def initialize(name = nil, type: nil, root:, usecase: nil, &block)
           if usecase
             raise "Expected Eco::API::UseCases::UseCase. Given #{usecase.class}" if !usecase.is_a?(Eco::API::UseCases::UseCase)

data/lib/eco/api/usecases/use_case_io.rb

@@ -1,28 +1,38 @@
 module Eco
   module API
     class UseCases
+
+      # InputOutput class for usecases.
+      # @note Same as Eco::API::UseCases::BaseIO but:
+      #  - includes `type` of usecase
+      #  - provides a helper to `chain` InputOutput between usecases
+      # @attr_reader usecase [Eco::API::UseCases::UseCase] the usecase this InputOuput is linked to
       class UseCaseIO < BaseIO
-        @types  = [:import, :filter, :transform, :sync, :export]
+        @types  = BaseIO.types
 
         attr_reader :usecase
 
+        # @see Eco::API::UseCases::BaseIO#initialize
         def initialize(usecase:, **kargs)
           self.usecase = usecase
           super(**kargs)
         end
 
-        # @see Eco::API::UseCases::BaseIO
+        # @see Eco::API::UseCases::BaseIO#new
         # @param usecase [Eco::API::UseCases::UseCase] target usecase
         # @return [Eco::API::UseCases::UseCaseIO]
         def new(usecase:, **kargs)
           super(**kargs.merge(usecase: usecase))
         end
 
+
+        # @param value [Eco::API::UseCases::UseCase] the usecase this InputOuput should be linked to
         def usecase=(value)
           raise "It should be a Eco::API::UseCases::UseCase. Given: #{value}" if !value.is_a?(Eco::API::UseCases::UseCase)
           @usecase = value
         end
 
+        # @return [Symbol] the `type` of the UseCase this InputOuput is linked to
         def type
           @usecase.type
         end
@@ -31,6 +41,8 @@ module Eco
           raise "Can't modify type depends on the usecase linked to this IO object"
         end
 
+        # @see Eco::API::UseCases::BaseIO#params
+        # Same as its superclass but adding `usecase` parameter
         def params(keyed: false)
           super(keyed: keyed).tap do |res|
             if keyed

data/lib/eco/cli/config/default/people.rb

@@ -6,6 +6,8 @@ ASSETS.cli.config do |cnf|
         get: {from: :remote, type: :full}
       })
       people = session.do.file_people
+    elsif SCR.get_arg("-no-people")
+      people = Eco::API::Organization::People.new([])
     elsif SCR.get_arg("-people-from-backup")
       file = SCR.get_arg("-people-from-backup", with_param: true)
       options.deep_merge!(people: {

data/lib/eco/version.rb

@@ -1,3 +1,3 @@
 module Eco
-  VERSION = "1.0.11"
+  VERSION = "1.0.12"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eco-helpers
 version: !ruby/object:Gem::Version
-  version: 1.0.11
+  version: 1.0.12
 platform: ruby
 authors:
 - Oscar Segura
@@ -178,6 +178,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: net-sftp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.2
 - !ruby/object:Gem::Dependency
   name: faker
   requirement: !ruby/object:Gem::Requirement
@@ -261,6 +281,7 @@ files:
 - lib/eco/api/common/session/logger.rb
 - lib/eco/api/common/session/mailer.rb
 - lib/eco/api/common/session/s3_uploader.rb
+- lib/eco/api/common/session/sftp.rb
 - lib/eco/api/common/version_patches.rb
 - lib/eco/api/common/version_patches/ecoportal_api.rb
 - lib/eco/api/common/version_patches/ecoportal_api/account_preferences.rb
@@ -302,6 +323,7 @@ files:
 - lib/eco/api/session/config/people.rb
 - lib/eco/api/session/config/post_launch.rb
 - lib/eco/api/session/config/s3_storage.rb
+- lib/eco/api/session/config/sftp.rb
 - lib/eco/api/session/config/workflow.rb
 - lib/eco/api/session/task.rb
 - lib/eco/api/usecases.rb