eco-helpers 0.9.2 → 0.9.3

data/lib/eco/api/organization/people.rb

@@ -78,11 +78,11 @@ module Eco
           person(id: id, external_id: external_id, email: email, strict: strict)
         end
 
-        def uniq(include_unsearchable: false)
+        def uniq(strict: false, include_unsearchable: false)
           init_caches
           unsearchable = []
           newFrom to_a.each_with_object([]) do |person, people|
-            if found = find(person)
+            if found = find(person, strict: strict)
               people << found
             else
               unsearchable << person
@@ -92,19 +92,19 @@ module Eco
           end
         end
 
-        # TODO: should keep it uniq
-        #def merge(data)
-        #  newFrom to_a + data
-        #end
-
+        def merge(data, strict: false, uniq: true)
+          list = uniq ? exclude_people(data, strict: strict).to_a : to_a
+          data = data.to_a unless data.is_a?(Array)
+          newFrom list + data
+        end
 
-        def exclude(object)
-          exclude_people(into_a(object))
+        def exclude(object, strict: false)
+          exclude_people(into_a(object), strict: strict)
         end
 
-        def exclude_people(list)
+        def exclude_people(list, strict: false)
           discarded = list.map do |person|
-            find(person)
+            find(person, strict: strict)
           end.compact
           newFrom to_a - discarded
         end

data/lib/eco/api/organization/policy_groups.rb

@@ -12,6 +12,14 @@ module Eco
           init_caches
         end
 
+        def ids
+          @by_id.keys
+        end
+
+        def names
+          @by_name.keys
+        end
+        
         def to_id(name)
           case name
           when Enumerable

data/lib/eco/api/policies.rb

@@ -1,10 +1,33 @@
 module Eco
   module API
-    module Policies
+    class Policies < Eco::API::UseCases
+      attr_reader :policies
+
+      def add(policy)
+        raise "Expected Eco::API::Policies::Policy object. Given: #{policy}" unless policy.is_a?(Eco::API::Policies::Policy)
+        super(policy)
+      end
+
+      def define(name, type: :transform, &block)
+        Eco::API::Policies::Policy.new(name, type: type, root: self, &block).tap do |policy|
+          add(policy)
+        end
+      end
+
+      def launch(people:, session:, options: {})
+
+        args = {
+          people:  people,
+          session: session,
+          options: options
+        }
+
+        items.map do |policy|
+          policy.launch(**args)
+        end
+      end
     end
   end
 end
 
-require_relative 'policies/base_policy'
 require_relative 'policies/policy'
-require_relative 'policies/used_policies'

data/lib/eco/api/policies/policy.rb

@@ -1,16 +1,15 @@
 module Eco
   module API
-    module Policies
+    class Policies
       class Policy < Eco::API::UseCases::UseCase
-        TYPES = [:transform]
-
+        @types = [:transform]
+        
         def initialize(name, type: :transform, root:, &block)
-          raise "Undefine usecase type #{type}. Please, use any of #{TYPES}" unless self.class.valid_type?(type)
           super(name, type: type, root: root, &block)
         end
 
         def root=(value)
-          raise "Root should be a UsedPolicies object. Given: #{value}" if !value.is_a?(UsedPolicies)
+          raise "Root should be a Eco::API::Policies object. Given: #{value}" if !value.is_a?(Eco::API::Policies)
           @root = value
         end
 

data/lib/eco/api/session.rb

@@ -1,7 +1,7 @@
 module Eco
   module API
     class Session < Common::Session::BaseSession
-      attr_reader   :batch, :job_groups
+      attr_reader   :batch
       attr_reader   :usecases
       attr_accessor :schema
 
@@ -16,12 +16,8 @@ module Eco
 
         @batch        = Batch.new(enviro)
         @task         = Task.new(enviro)
-        @job_groups   = JobGroups.new(enviro)
-
-        @usecases     = Eco::API::UseCases::DefaultCases.new.merge(config.usecases)
 
         self.schema = config.people.default_schema || schemas.first
-        @policy_groups = config.policy_groups
 
         presets_custom_file = config.people.presets_custom
         presets_map_file    = config.people.presets_map
@@ -112,6 +108,10 @@ module Eco
         end
       end
 
+      def export(*args)
+        @entry_factory.export(*args)
+      end
+
       def new_person(**keyed_args)
         @person_factory.new(**keyed_args)
       end
@@ -126,25 +126,38 @@ module Eco
         end
       end
 
-      def export(*args)
-        @entry_factory.export(*args)
-      end
-
       def csv_entries(file)
         return entries(file: file, format: :csv)
       end
 
       def discarded_entries
+        return @discarded_entries if instance_variable_defined?(:@discarded_entries)
         file = config.people.discarded_file
         file = file_manager.dir.file(file)
-        csv_entries(file)
+        @discarded_entries = csv_entries(file)
       end
 
-      def process_case(name, type: nil, **params)
+      def workflow(io:)
+        config.workflow.tap do |wf|
+          yield(wf, io) if block_given?
+        end
+      end
+
+      def usecases
+        @usecases ||= Eco::API::UseCases::DefaultCases.new.merge(config.usecases)
+      end
+
+      def post_launch
+        @post_launch ||= config.post_launch.select(usecases)
+      end
+
+      def process_case(name, io: nil, type: nil, **params)
         args = { session: self }.merge(params)
-        fatal("Undefined usecase '#{name}' of type '#{type.to_s}'") if !usecases.defined?(name, type: type)
-        logger.debug("Session: going to process '#{name}' defined case")
-        usecases.case(name, type: type).launch(**args)
+        usecases.case(name, type: type).launch(io: io, **args)
+      end
+
+      def job_groups
+        @job_groups ||= Batch::JobsGroups.new(enviro)
       end
 
       def job_group(name, order: :last)
@@ -166,8 +179,4 @@ end
 
 require_relative 'session/config'
 require_relative 'session/batch'
-require_relative 'session/batch_status'
 require_relative 'session/task'
-require_relative 'session/batch_job'
-require_relative 'session/batch_jobs'
-require_relative 'session/job_groups'

data/lib/eco/api/session/batch.rb

@@ -23,9 +23,9 @@ module Eco
         # @option params [String] :per_page the number of people included per each batch api request.
         # @option params [String] :q some text to search. Omit this parameter to target all the people.
         # @return [Array<People>] all the people based on `params`
-        def get_people(people = nil, params: {})
-          return launch(people, method: :get, params: params) if people.is_a?(Enumerable)
-          return get(params: params)
+        def get_people(people = nil, params: {}, silent: false)
+          return launch(people, method: :get, params: params, silent: silent) if people.is_a?(Enumerable)
+          return get(params: params, silent: silent)
         end
 
         # launches a batch of `method` type using `people` and the specified `params`
@@ -36,7 +36,7 @@ module Eco
         # @param method [Symbol] the method to launch the batch api request with.
         # @param params [Hash] api request options.
         # @option params [String] :per_page the number of people included per each batch api request.
-        # @return [BatchStatus] the `status` of this batch launch.
+        # @return [Batch::Status] the `status` of this batch launch.
         def launch(people, method:, params: {} , silent: false)
           batch_from(people, method: method, params: params, silent: silent)
         end
@@ -87,7 +87,7 @@ module Eco
         private
 
         def new_status(queue, method)
-          BatchStatus.new(enviro, queue: queue, method: method)
+          Batch::Status.new(enviro, queue: queue, method: method)
         end
 
         def get(params: {}, silent: false)
@@ -174,7 +174,7 @@ module Eco
             done += slice.length
           end # next slice
 
-          status.print_errors unless silent
+          status.errors.print unless silent
           return status
         end
 
@@ -182,3 +182,9 @@ module Eco
     end
   end
 end
+
+require_relative 'batch/job'
+require_relative 'batch/status'
+require_relative 'batch/errors'
+require_relative 'batch/jobs'
+require_relative 'batch/jobs_groups'

data/lib/eco/api/session/batch/errors.rb

@@ -0,0 +1,134 @@
+module Eco
+  module API
+    class Session
+      class Batch
+        class Errors
+
+          attr_reader :status
+
+          def initialize(status:)
+            "Expected Batch::Status as root. Given: #{status.class}" unless status.is_a?(Batch::Status)
+            @status = status
+          end
+
+          def queue
+            status.queue
+          end
+
+          def method
+            status.method
+          end
+
+          def to_index(*args)
+            status.to_index(*args)
+          end
+
+          def session
+            status.session
+          end
+
+          def logger
+            status.logger
+          end
+
+          def any?
+            queue.any? {|query| !status[query].success?}
+          end
+
+          def entries
+            queue.each_with_index.map do |query, i|
+              unless status[i]
+                msg  = "Error: query with no response. You might have duplicated entries in your queue.\n"
+                msg += "Queue length: #{queue.length}; Queue elements class: #{queue.first.class}\n"
+                msg += "Query with no response. Person: #{person_ref(query)}\n"
+                queue.map do |entry|
+                  if [:id, :external_id, :email].any? {|attr| (v = get_attr(entry, attr)) && v == get_attr(query, attr)}
+                    msg += "It could be this peson entry (idx: #{to_index(entry)}): #{person_ref(entry)}\n"
+                  end
+                end
+                raise msg
+              end
+
+              status[i].success? ? nil : query
+            end.compact
+          end
+
+          def errors
+            entries.each_with_object([]) do |entry, arr|
+              if errs = status[entry].body["errors"]
+                errs.each do |msg|
+                  arr.push({
+                    type:  klass = Eco::API::Error.get_type(msg),
+                    err:   klass.new(err_msg: msg, entry: entry, session: session),
+                    entry: entry
+                  })
+                end
+              end
+            end
+          end
+
+          def by_type
+            errors.group_by do |h|
+              h[:type]
+            end.transform_values do |arr|
+              arr.map {|h| h[:entry]}
+            end
+          end
+
+          def count
+            entries.length
+          end
+
+          def str(key)
+            msg = ""
+            unless status.success?(key)
+              i        = to_index(key)
+              entry    = queue.to_a[i]
+              response = status[i]
+              msg      = "Error #{response.status}: #{response.body}\n"
+              msg     += "-- Failed to batch #{method} (entry #{i+1}). Person: #{person_ref(entry)}"
+            end
+            msg
+          end
+
+          def strs
+            by_type.values.flatten(1).each_with_object([]) do |query, msgs|
+              msgs.push(str(query))
+            end
+          end
+
+          def print_one(key)
+            unless status.success?(key)
+              logger.error(str(key))
+            end
+          end
+
+          def print
+            msgs = strs
+            if msgs.length > 0
+              logger.info()
+              logger.error("There were #{msgs.length} errors:\n" + msgs.join("\n"))
+            else
+              logger.info("There were no errors for the current batch '#{method}'!! ;)")
+            end
+          end
+
+          private
+
+          def person_ref(entry)
+            "(id: '#{get_attr(entry, :id)}') '#{get_attr(entry, :name)}' ('#{get_attr(entry, :external_id)}': '#{get_attr(entry, :email)}')"
+          end
+
+          def get_attr(entry, attr)
+            if entry.respond_to?(attr.to_sym)
+              entry.public_send(attr.to_sym)
+            elsif entry.is_a?(Hash)
+              entry["#{attr}"]
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/session/batch/job.rb

@@ -0,0 +1,213 @@
+module Eco
+  module API
+    class Session
+      class Batch
+        class Job < API::Common::Session::BaseSession
+          @types = [:get, :create, :update, :delete]
+          @sets  = [:core, :details, :account]
+
+          class << self
+            attr_reader :types, :sets
+
+            def valid_type?(value)
+              types.include?(value)
+            end
+
+            def valid_sets?(value)
+              sts = [value].flatten
+              sts.all? { |s| sets.include?(s) }
+            end
+          end
+
+          attr_reader :name, :type, :status
+          attr_reader :usecase
+
+          def initialize(e, name:, type:, sets:, usecase: nil)
+            raise "A name is required to refer a job. Given: #{name}" if !name
+            raise "Type should be one of #{self.class.types}. Given: #{type}"    if !self.class.valid_type?(type)
+            raise "Sets should be some of #{self.class.sets}. Given: #{sets}"    if !self.class.valid_sets?(sets)
+            raise "usecase must be a Eco::API::UseCases::UseCase object. Given: #{usecase.class}" if usecase && !usecase.is_a?(Eco::API::UseCases::UseCase)
+            super(e)
+
+            @name    = name
+            @type    = type
+            @usecase = usecase
+            @sets    = [sets].flatten.compact
+            reset
+          end
+
+          def reset
+            @queue      = []
+            @queue_hash = {}
+            @callbacks  = {}
+            @pending    = true
+            @status     = nil
+          end
+
+          def usecase?
+            !!usecase
+          end
+
+          def options
+            usecase?? usecase.options : {}
+          end
+
+          def signature
+            "Batch job \"#{name}\" ['#{type.to_s.upcase}': #{sets_title}]"
+          end
+
+          def match?(type:, sets:)
+            sets = [sets].flatten
+            type == self.type && (sets.order == @sets.order)
+          end
+
+          def pending?
+            @pending
+          end
+
+          # Adds an entry(ies) to the job queue.
+          # @param entry [Person, Enumberable<Person>] the person(s) we want to update, carrying the changes to be done.
+          # @param unique [Boolean] specifies if repeated entries should be avoided in the queue.
+          # @yield [person] callback before launching the batch job request against the server.
+          # @yeldparam param [Person] current person object that that should be treated by the callback before launching the batch.
+          # @return [Void]
+          def add(entry, unique: true, &block)
+            case entry
+            when Enumerable
+              entry.each {|e| add(e, unique: unique, &block)}
+            else
+              unless !entry
+                unless unique && @queue_hash.key?(entry)
+                  @queue_hash[entry] = true
+                  @queue.push(entry)
+                  @callbacks[entry] = Proc.new if block_given?
+                end
+              end
+            end
+          end
+
+          def people(input = @queue)
+            Eco::API::Organization::People.new(input)
+          end
+
+          def launch(simulate: false)
+            queue = processed_queue
+            launch_feedback(queue, simulate ? 2500 : 800)
+
+            if !simulate
+              if queue.length > 0
+                backup_update(queue)
+                @status = session.batch.launch(queue, method: type)
+                @status.root = self
+              end
+            end
+
+            post_launch(queue: queue, simulate: simulate)
+
+            logger.info("Simulate: this would have launched: '#{type}'") if simulate
+            @pending = false
+            return @status
+          end
+
+          private
+
+          def processed_queue
+            @queue.each {|e| @callbacks[e].call(e) if @callbacks.key?(e) }
+            apply_policies(@queue).select {|e| !as_update(e).empty?}
+          end
+
+          def post_launch(queue: [], simulate: false)
+            if !simulate && @status
+              @status.queue.map do |entry|
+                if @status.success?(entry)
+                  entry.consolidate! if entry.respond_to?(:consolidate!)
+                #else # do not entry.reset! (keep track on changes still)
+                end
+              end
+              # launch_error handlers
+              handlers = session.config.error_handlers
+              if @status.errors.any? && !handlers.empty?
+                err_types = @status.errors.by_type
+                handlers.each do |hd|
+                  if entries = err_types[hd.name]
+                    hd.launch(people: people(entries), session: session, options: options)
+                  end
+                end
+              end
+            elsif simulate
+              queue.map do |entry|
+                entry.consolidate! if entry.respond_to?(:consolidate!)
+              end
+            end
+          end
+
+          def apply_policies(pre_queue)
+            pre_queue.tap do |entries|
+              policies = session.config.policies
+              unless policies.empty?
+                policies.launch(people: people(entries), session: session, options: options)
+              end
+            end
+          end
+
+          def as_update(entry)
+            hash = entry if entry.is_a?(Hash)
+            if only_ids?
+              hash = entry.as_json.slice("id", "external_id", "email")
+            else
+              if entry.is_a?(Ecoportal::API::V1::Person)
+                hash    = entry.as_update
+                if hfields = hash.dig("details", "fields")
+                  hash["details"]["fields"] = hfields.map do |fld|
+                    fld.merge!("alt_id" => entry.details.get_field(fld["id"]).alt_id) if entry.details
+                  end
+                end
+              end
+
+              fields = hash&.dig('details', 'fields')
+              fields&.map! { |fld| fld&.slice("id", "alt_id", "value") }
+            end
+            hash || {}
+          end
+
+          def only_ids?
+            [:delete, :get].include?(type)
+          end
+
+          def sets_title
+            "#{@sets.map {|s| s.to_s}.join(", ")}"
+          end
+
+          def launch_feedback(data, max_chars = 800)
+            if !data || !data.is_a?(Enumerable) || data.empty?
+              logger.warn("#{"*" * 20} Nothing for #{signature} so far :)  #{"*" * 20}")
+              return
+            end
+            header = ("*" * 20) + "  #{signature} - Feedback Sample " + ("*" * 20)
+            logger.info(header)
+
+            sample_length = 1
+            sample = data.slice(0, 20).map do |entry|
+              update         = as_update(entry)
+              max_chars     -= update.pretty_inspect.length
+              sample_length += 1 if max_chars > 0
+              update
+            end
+
+            logger.info("#{sample.slice(0, sample_length).pretty_inspect}")
+            logger.info("#{type.to_s.upcase} length: #{data.length}")
+            logger.info("*" * header.length)
+          end
+
+          def backup_update(data)
+            data_body = data.map { |u| as_update(u) }
+            dir       = config.people.requests_folder
+            file      = File.join(dir, "#{type}_data.json")
+            file_manager.save_json(data_body, file, :timestamp)
+          end
+
+        end
+      end
+    end
+  end
+end