eco-helpers 1.0.13 → 1.0.14

data/lib/eco/api/session/batch/job.rb

@@ -2,7 +2,13 @@ module Eco
   module API
     class Session
       class Batch
-        class Job < API::Common::Session::BaseSession
+        # @attr_reader name [String] the name of this `batch job`
+        # @attr_reader type [Symbol] a valid batch operation
+        # @attr_reader sets [Array<Symbol>] the parts of the person model this batch is supposed to affect
+        # @attr_reader usecase [Eco::API::UseCases::UseCase, nil] when provided: `usecase` that generated this `batch job`
+        # @attr_reader status [Eco::API::Session::Batch::Status] if launched: the `status` of the `batch`
+        # @attr_reader feedback [Eco::API::Session::Batch::Feedback] helper class for feedback and decision making
+        class Job < Eco::API::Common::Session::BaseSession
           @types = [:get, :create, :update, :delete]
           @sets  = [:core, :details, :account]
 
@@ -19,20 +25,26 @@ module Eco
             end
           end
 
-          attr_reader :name, :type, :status
+          attr_reader :name, :type, :sets
           attr_reader :usecase
+          attr_reader :status, :feedback
 
+          # @param e [Eco::API::Common::Session::Environment] requires a session environmen, as any child of `Eco::API::Common::Session::BaseSession`
+          # @param name [String] the name of this `batch job`
+          # @param type [Symbol] a valid batch operation
+          # @param usecase [Eco::API::UseCases::UseCase, nil] when provided: `usecase` that generated this `batch job`
           def initialize(e, name:, type:, sets:, usecase: nil)
             raise "A name is required to refer a job. Given: #{name}" if !name
-            raise "Type should be one of #{self.class.types}. Given: #{type}"    if !self.class.valid_type?(type)
-            raise "Sets should be some of #{self.class.sets}. Given: #{sets}"    if !self.class.valid_sets?(sets)
+            raise "Type should be one of #{self.class.types}. Given: #{type}"    unless self.class.valid_type?(type)
+            raise "Sets should be some of #{self.class.sets}. Given: #{sets}"    unless self.class.valid_sets?(sets)
             raise "usecase must be a Eco::API::UseCases::UseCase object. Given: #{usecase.class}" if usecase && !usecase.is_a?(Eco::API::UseCases::UseCase)
             super(e)
 
-            @name    = name
-            @type    = type
-            @usecase = usecase
-            @sets    = [sets].flatten.compact
+            @name     = name
+            @type     = type
+            @sets     = [sets].flatten.compact
+            @usecase  = usecase
+            @feedback = Eco::API::Session::Batch::Feedback.new(job: self)
             reset
           end
 
@@ -44,23 +56,22 @@ module Eco
             @status     = nil
           end
 
+          # @return [Boolean] was this `batch job` generated by a `usecase`? (`Eco::API::UseCases::UseCase`)
           def usecase?
             !!usecase
           end
 
+          # @return [Hash] options the root `usecase` is run with
           def options
             usecase?? usecase.options : {}
           end
 
-          def signature
-            "Batch job \"#{name}\" ['#{type.to_s.upcase}': #{sets_title}]"
-          end
-
           def match?(type:, sets:)
             sets = [sets].flatten
-            type == self.type && (sets.order == @sets.order)
+            type == self.type && (sets.order == self.sets.order)
           end
 
+          # @return [Boolean] has been this `batch job` launched?
           def pending?
             @pending
           end
@@ -80,23 +91,40 @@ module Eco
                 unless unique && @queue_hash.key?(entry)
                   @queue_hash[entry] = true
                   @queue.push(entry)
-                  @callbacks[entry] = Proc.new if block_given?
+                  @callbacks[entry]  = Proc.new if block_given?
                 end
               end
             end
           end
 
+          # Helper/shortcut to obtain a people object out of `input`
+          # @note if `input` is not provided, it will use `queue`
+          # @return [Eco::API::Organization::People]
           def people(input = @queue)
             Eco::API::Organization::People.new(input)
           end
 
+          # Processes the `queue` and, unless `simulate` is `true`, launches against the server:
+          #   1. if the entries of `queue` got pending _callbacks_ (delayed changes), it processes them
+          #   2. unless type == `:create`: if there's a defined `api_excluded` _callback_ it calls it (see `Eco::API::Session::Config::People#api_excluded`)
+          #   3. transforms the result to a `Eco::API::Organization::People` object
+          #   4. if there are `api policies` defined, it passes the entries through them in order (see `Eco::API::Session::Config#policies`)
+          #   5. at this point all the transformations have taken place...
+          #   6. only include the entries that, after all above, still hold pending changes (`!as_update.empty?`) to be launched as update
+          #   7. if we are **not** in `dry-run` (or `simulate`), launch the batch request against the server (see `Eco::API::Session::Batch#launch`)
+          #   8. next, it links the resulting batch `status` to this `Batch::Job` (see `Eco::API::Session::Batch::Status`)
+          #   9. the post launch kicks in, and for success requests, it consolidates the associated entries (see `Ecoportal::API::V1::Person#consolidate!`)
+          #  10. launches specific error handlers, if there were **errors** from the Server as a result of the `batch.launch`, and there are `Error::Handlers` defined
+          #  11. if we are **not** in `dry-run` (or `simulate`), it backs up the raw queries launched to the Server
           def launch(simulate: false)
-            pqueue = processed_queue
-            launch_feedback(pqueue, simulate ? 2500 : 800)
+            pqueue   = processed_queue
+            requests = pqueue.map {|e| as_update(e)}
+
+            pre_checks(requests, simulate: simulate)
 
             if !simulate
               if pqueue.length > 0
-                backup_update(pqueue)
+                backup_update(requests)
                 @status = session.batch.launch(pqueue, method: type)
                 @status.root = self
               end
@@ -111,36 +139,15 @@ module Eco
 
           private
 
+          def as_update(*args)
+            feedback.as_update(*args)
+          end
+
           def processed_queue
             @queue.each {|e| @callbacks[e].call(e) if @callbacks.key?(e) }
             apply_policies(api_included(@queue)).select {|e| !as_update(e).empty?}
           end
 
-          def post_launch(queue: [], simulate: false)
-            if !simulate && @status
-              @status.queue.map do |entry|
-                if @status.success?(entry)
-                  entry.consolidate! if entry.respond_to?(:consolidate!)
-                #else # do not entry.reset! (keep track on changes still)
-                end
-              end
-              # launch_error handlers
-              handlers = session.config.error_handlers
-              if @status.errors.any? && !handlers.empty?
-                err_types = @status.errors.by_type
-                handlers.each do |handler|
-                  if entries = err_types[handler.name]
-                    handler.launch(people: people(entries), session: session, options: options)
-                  end
-                end
-              end
-            elsif simulate
-              queue.map do |entry|
-                entry.consolidate! if entry.respond_to?(:consolidate!)
-              end
-            end
-          end
-
           # if there is a config definition to exclude entries
           # and the current batch is not a creation batch
           # - filter out excluded entries from the api update
@@ -151,69 +158,65 @@ module Eco
           end
 
           def apply_policies(pre_queue)
-            #pre_queue.tap do |entries|
             people(pre_queue).tap do |entries|
               policies = session.config.policies
-              unless policies.empty?
+              unless policies.empty? || options.dig(:skip, :api_policies)
                 policies.launch(people: entries, session: session, options: options)
               end
             end
           end
 
-          def as_update(entry)
-            hash = entry if entry.is_a?(Hash)
-            if only_ids?
-              hash = entry.as_json.slice("id", "external_id", "email")
-            else
-              if entry.is_a?(Ecoportal::API::V1::Person)
-                hash    = entry.as_update
-                if hfields = hash.dig("details", "fields")
-                  hash["details"]["fields"] = hfields.map do |fld|
-                    fld.merge!("alt_id" => entry.details.get_field(fld["id"]).alt_id) if entry.details
-                  end
-                end
-              end
-
-              fields = hash&.dig('details', 'fields')
-              fields&.map! { |fld| fld&.slice("id", "alt_id", "value") }
+          def batch_policy
+            unless options.dig(:skip, :batch_policy)
+              @batch_policy ||= session.config.batch_policies[self.type]
             end
-            hash || {}
           end
 
-          def only_ids?
-            [:delete, :get].include?(type)
-          end
+          def pre_checks(requests, simulate: false)
+            only_stats = options.dig(:feedback, :only_stats)
+            max_chars  = simulate ? 2500 : 800
+            msg        = feedback.generate(requests, max_chars: max_chars, only_stats: only_stats)
+            logger.info(msg)
 
-          def sets_title
-            "#{@sets.map {|s| s.to_s}.join(", ")}"
+            @request_stats = feedback.request_stats(requests)
+            if simulate && batch_policy && !batch_policy.compliant?(@request_stats)
+              logger.warn("Batch Policy Uncompliance: this and next batches will be aborted!")
+              logger.warn(batch_policy.uncompliance(@request_stats))
+            elsif batch_policy
+              # will throw an Exception if the policy request_stats is not compliant
+              batch_policy.validate!(@request_stats)
+            end
           end
 
-          def launch_feedback(data, max_chars = 800)
-            if !data || !data.is_a?(Enumerable) || data.empty?
-              logger.warn("#{"*" * 20} Nothing for #{signature} so far :)  #{"*" * 20}")
-              return
-            end
-            header = ("*" * 20) + "  #{signature} - Feedback Sample " + ("*" * 20)
-            logger.info(header)
-
-            sample_length = 1
-            sample = data.slice(0, 20).map do |entry|
-              update         = as_update(entry)
-              max_chars     -= update.pretty_inspect.length
-              sample_length += 1 if max_chars > 0
-              update
+          def post_launch(queue: [], simulate: false)
+            if !simulate && @status
+              @status.queue.map do |entry|
+                if @status.success?(entry)
+                  entry.consolidate! if entry.respond_to?(:consolidate!)
+                #else # do not entry.reset! (keep track on changes still)
+                end
+              end
+              # launch_error handlers
+              handlers = session.config.error_handlers
+              if @status.errors.any? && !handlers.empty?
+                err_types = @status.errors.by_type
+                handlers.each do |handler|
+                  if entries = err_types[handler.name]
+                    handler.launch(people: people(entries), session: session, options: options)
+                  end
+                end
+              end
+            elsif simulate
+              queue.map do |entry|
+                entry.consolidate! if entry.respond_to?(:consolidate!)
+              end
             end
-
-            logger.info("#{sample.slice(0, sample_length).pretty_inspect}")
-            logger.info("#{type.to_s.upcase} length: #{data.length}")
-            logger.info("*" * header.length)
           end
 
-          def backup_update(data)
-            data_body = data.map { |u| as_update(u) }
+          def backup_update(requests)
             dir       = config.people.requests_folder
             file      = File.join(dir, "#{type}_data.json")
-            file_manager.save_json(data_body, file, :timestamp)
+            file_manager.save_json(requests, file, :timestamp)
           end
 
         end

data/lib/eco/api/session/batch/policies.rb

@@ -0,0 +1,22 @@
+module Eco
+  module API
+    class Session
+      class Batch
+        class Policies < Eco::API::Session::Batch::BasePolicy
+          CORE_ATTRS    = Eco::API::Session::Batch::RequestStats.core_attrs(stats: true)
+          ACCOUNT_ATTRS = Eco::API::Session::Batch::RequestStats.account_attrs(stats: true)
+          DETAILS_ATTRS = Eco::API::Session::Batch::RequestStats.details_attrs(stats: true)
+
+          core_model    = {core:    CORE_ATTRS}
+          account_model = {account: ACCOUNT_ATTRS}
+          details_model = {details: DETAILS_ATTRS}
+          submodel      = core_model.merge(account_model).merge(details_model)
+          TOP_MODEL     = Eco::API::Session::Batch::Job.types.each_with_object({}) {|t, h| h[t] = submodel}
+
+          self.model = TOP_MODEL
+          policy_attrs *model_attrs
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/session/batch/request_stats.rb

@@ -0,0 +1,195 @@
+module Eco
+  module API
+    class Session
+      class Batch
+        # @attr_reader count [Integer] the total number of requests
+        # @attr_reader stats [Hash] plain `Hash` with the number of requests that include an attribute
+        class RequestStats
+          CORE_ATTRS    = Eco::API::Common::People::PersonParser::CORE_ATTRS
+          ACCOUNT_ATTRS = (Eco::API::Common::People::PersonParser::ACCOUNT_ATTRS + ["permissions_custom"]).uniq
+          DETAILS_ATTRS = ["fields"]
+
+          class << self
+
+            def valid_type?(type)
+              Eco::API::Session::Batch::Job.valid_type?(type.to_sym)
+            end
+
+            def core_attrs(stats: false, all: false)
+              CORE_ATTRS.dup.tap do |attrs|
+                attrs.unshift("core") if stats || all
+              end
+            end
+
+            def account_attrs(stats: false, all: false)
+              ACCOUNT_ATTRS.dup.tap do |attrs|
+                if stats || all
+                  attrs.unshift("account_remove")
+                  attrs.unshift("account") if all
+                end
+              end
+            end
+
+            def details_attrs(stats: false, all: false)
+              DETAILS_ATTRS.dup.tap do |attrs|
+                if stats || all
+                  attrs.unshift("details_remove")
+                  attrs.unshift("details") if all
+                end
+              end
+            end
+
+          end
+
+          attr_reader :type, :count
+
+          def initialize(type:, requests: [])
+            raise "type should be one of #{Eco::API::Session::Batch::Job.types}. Given: #{type}"  unless self.class.valid_type?(type.to_sym)
+            @type  = type.to_sym
+            @count = requests && requests.length
+            @stats = build(requests)
+          end
+
+          def to_h
+            @stats
+          end
+
+          def core_attrs
+            @core_attrs    ||= self.class.core_attrs
+          end
+
+          def account_attrs
+            @account_attrs ||= self.class.account_attrs
+          end
+
+          def details_attrs
+            @details_attrs ||= self.class.details_attrs
+          end
+
+          def attr(attr, percent: false, total: count)
+            i = @stats["#{attr}"]
+            return i unless percent
+            percentage(i, total: total)
+          end
+
+          def core(percent: false)
+            attr("core", percent: percent)
+          end
+
+          def account(percent: false)
+            attr("account", percent: percent)
+          end
+
+          def account_remove(percent: false)
+            attr("account_remove", percent: percent)
+          end
+
+          def details(percent: false)
+            attr("details", percent: percent)
+          end
+
+          def details_remove(percent: false)
+            attr("details_remove", percent: percent)
+          end
+
+          def fields_average
+            if (fields_num = attr("fields")) && (total = details) > 0
+              (fields_num.to_f / total.to_f).round(2)
+            end
+          end
+
+          def message(percent: false)
+            key_val_delimiter = ": "; attr_delimiter = " ++ "
+            pairs_to_line = Proc.new do |pairs|
+              pairs.map do |p|
+                [p.first.to_s, "#{p.last.to_s}" + (percent ? "%" : "")].join(key_val_delimiter)
+              end.join(attr_delimiter)
+            end
+
+            lines = []
+            lines << pairs_to_line.call(core_pairs(percent: percent))
+            lines << pairs_to_line.call(account_pairs(percent: percent))
+            lines << pairs_to_line.call(details_pairs(percent: percent))
+            lines.join("\n")
+          end
+
+          def model
+
+          end
+
+          private
+
+          def percentage(num, total: count)
+            total ||= count
+            if num
+              (num.to_f / total * 100).round(2)
+            end
+          end
+
+          def build(requests)
+            Hash.new(0).tap do |stats|
+              stats[type] = count
+              unless !requests || !requests.is_a?(Enumerable) || requests.empty?
+                requests.each_with_index do |request|
+                  add_core_stats(stats, request || {})
+                  add_account_stats(stats, request || {})
+                  add_details_stats(stats, request || {})
+                end
+              end
+            end
+          end
+
+          def attrs_to_stat(stats, hash, attrs)
+            stats.tap do |st|
+              attrs.each {|attr| st[attr] += 1 if hash.key?(attr)}
+            end
+          end
+
+          def add_core_stats(stats, request)
+            stats["core"] += 1 if (request.keys & core_attrs).length > 0
+            attrs_to_stat(stats, request, core_attrs)
+          end
+
+          def add_account_stats(stats, request)
+            if request.key?("account")
+              stats["account"] += 1
+              stats["account_remove"] += 1 if !request["account"]
+              attrs_to_stat(stats, request["account"] || {}, account_attrs)
+            end
+          end
+
+          def add_details_stats(stats, request)
+            if request.key?("details")
+              stats["details"] += 1
+              stats["details_remove"] += 1 if !request["details"]
+              if fields = request.dig("details", "fields")
+                stats["fields"] += fields.length
+              end
+            end
+          end
+
+          def pairs(attrs, percent: false, total: count)
+            pairs = attrs.map do |a|
+              (v = attr(a, percent: percent, total: count)) > 0 ? [a, v] : nil
+            end.compact
+          end
+
+          def core_pairs(percent: false)
+            [["core",      core(percent: percent)]] + pairs(core_attrs, percent: percent, total: core)
+          end
+
+          def account_pairs(percent: false)
+            aattrs         = ["account_remove"] + account_attrs
+            [["account", account(percent: percent)]] + pairs(aattrs, percent: percent, total: account)
+          end
+
+          def details_pairs(percent: false)
+            details_pairs  = [["details", details(percent: percent)]]
+            details_pairs += [["fields", fields_average]] if attr("fields") && fields_average
+            details_pairs += pairs(["details_remove"], percent: percent, total: details)
+          end
+        end
+      end
+    end
+  end
+end