eco-helpers 2.0.42 → 2.0.46

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c14b952724b755aa88b633094baf947b8c32b3a8699f77e8a4d63cdc94e10765
-  data.tar.gz: 015fd112ec97cb490c1bae82cdbd744256ca8879b98578f4c2442a0730652bf5
+  metadata.gz: fd9b2a6564e5ce38fa3ed762ab04f5e4e79b1a6580738d9091b846233c6fb66e
+  data.tar.gz: ac0200ad60b30fc0d40997b4975f016daee8cd48c881bfecae106ca0867ff123
 SHA512:
-  metadata.gz: cbdf02bb1b3323069682c6b9dca370b598d846fc185e8db6dba7e337f1939a41f00b17d60bcb599c4f8abf9c0c6a713495e88258eb5a521b061ffa01c63d552d
-  data.tar.gz: b656b2a52dc65a7e2ae582ec68999de0d7551a6a17c5b83a3763ead94d161a8c969dc76add52ec6a93ebbfd68b5b47293c06fe3e33c8cfaacc4ad6273bf5b17f
+  metadata.gz: f72270169fd60eaee60b27d333391bc6d2235fe856752d9f7dcd5d5aca18fd129e148cd0830bea37cd86833cabc64f10e4fb09e71c4ba2ba9b6912313da82b4a
+  data.tar.gz: 690b338d062020a6d80794d794f00f8ce9c263bb8b12a84f774f50193bd49ff474b6a49f670a1e05df9a2db88f70d3639c54582fc8706617c44123753c0a7b9e

data/CHANGELOG.md

@@ -1,6 +1,54 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.0.46] - 2021-12-24
+
+### Added
+  - `Eco::API::UseCases::OozeSamples::RegisterUpdateCase`
+    - added counter of retrieved entries (to be able to compare with matched initially scoped results)
+    - added also counter of failed to retrieve oozes
+    - added also counter of double ups in search pagination
+    - counter for updated pages and pages that failed update
+  - `Eco::API::Session#enviro_name`: to identify in whic environment we are running
+
+### Changed
+  - upgraded `ecoportal-api-v2` dependency
+  - `Eco::API::UseCases::OozeSamples::OozeBaseCase#update_ooze`
+    - To return `false` if there was request made to the server
+    - To return `Reponse` if there was request made to the server
+
+### Fixed
+  - Changed the `workflow` in
+    - `on(:end)` to do not recache people if there were no `:sync` or `:transform` cases
+  - `Eco::API::UseCases::OozeSamples::OozeRunBaseCase#update_ooze`
+    - It's parent had parameter (it as a child should so)
+  - `Eco::API::UseCases::OozeSamples::OozeBaseCase#process_ooze`
+    - It was not retrieving the `ooze` unless explicitly passed as parameter
+    - This did not play well with child implementations that expect it to retrieve it based on `ooze_id` cli argument
+
+## [2.0.45] - 2021-12-15
+
+### Added
+  - `Eco::API::UseCases::OozeSamples::RegisterExportCase`
+
+### Changed
+  - Upgrade `ecoportal-api` dependency
+  - Upgrade `ecoportal-api-v2` dependency
+
+## [2.0.44] - 2021-11-25
+
+### Changed
+  - Upgrade `ecoportal-api` dependency
+
+## [2.0.43] - 2021-11-25
+
+### Added
+  - `Eco::API::Session::Batch::Job` added better **logging**
+
+### Changed
+  - `Eco::API::Session::Batch` added **resilience** and **recovery** to connection errors
+  - `Eco::API::Policies::DefaultPolicies::UserAccess` changed logging from `warn` to `info`.
+
 ## [2.0.42] - 2021-10-30
 
 ### Added

data/eco-helpers.gemspec

@@ -30,8 +30,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "yard",      ">= 0.9.26", "< 0.10"
   spec.add_development_dependency "redcarpet", ">= 3.5.1",  "< 3.6"
 
-  spec.add_dependency 'ecoportal-api', '>= 0.8.3', '< 0.9'
-  spec.add_dependency 'ecoportal-api-v2', '>= 0.8.19', '< 0.9'
+  spec.add_dependency 'ecoportal-api', '>= 0.8.4', '< 0.9'
+  spec.add_dependency 'ecoportal-api-v2', '>= 0.8.23', '< 0.9'
   spec.add_dependency 'aws-sdk-s3',    '>= 1.83.0', '< 2'
   spec.add_dependency 'aws-sdk-ses',   '>= 1.36.0', '< 2'
   spec.add_dependency 'dotenv',        '>= 2.7.6', '< 2.8'

data/lib/eco/api/microcases/people_load.rb

@@ -30,7 +30,7 @@ module Eco
             people_load(modifier: modifier - [:newest, :file])
           end
         when modifier.include?(:api)
-          logger.info("Going to get all the people via API")
+          logger.info("Going to get all the people via API (load)")
 
           start = Time.now
           session.batch.get_people.tap do |people|

data/lib/eco/api/policies/default_policies/99_user_access_policy.rb

@@ -22,7 +22,7 @@ class Eco::API::Policies::DefaultPolicies::UserAccess < Eco::API::Common::Loader
   def warn_account_removal!
     if account_removed_count > 0
       msg = "(DefaultPolicy on job '#{job.name}') Removed account to #{account_removed_count} people"
-      session.logger.warn(msg)
+      session.logger.info(msg)
     end
   end
 

data/lib/eco/api/session/batch/job.rb

@@ -176,6 +176,7 @@ module Eco
               if pqueue.length > 0
                 req_backup = as_update(pqueue, add_feedback: false)
                 backup_update(req_backup)
+                logger.debug("Job ('#{name}':#{type}): going to launch batch against #{pqueue.count} entries")
                 session.batch.launch(pqueue, method: type).tap do |job_status|
                   @status     = job_status
                   status.root = self
@@ -334,7 +335,7 @@ module Eco
                 handlers.each do |handler|
                   if entries = err_types[handler.name]
                     handler_job = subjobs_add("#{self.name} => #{handler.name}", usecase: handler)
-                    logger.debug("Running error handler #{handler.name}")
+                    logger.debug("Running error handler #{handler.name} (against #{entries.count} entries)")
                     handler.launch(people: people(entries), session: session, options: options, job: handler_job)
                     logger.debug("Launching job of error handler: #{handler_job.name}")
                     handler_job.launch(simulate: simulate)

data/lib/eco/api/session/batch.rb

@@ -92,6 +92,7 @@ module Eco
           return people_api.get_all(params: params, silent: silent)
         end
 
+
         def batch_from(data, method:, params: {}, silent: false)
           fatal "Invalid batch method: #{method}." if !self.class.valid_method?(method)
           return nil if !data || !data.is_a?(Enumerable)
@@ -101,12 +102,23 @@ module Eco
           params   = {per_page: DEFAULT_BATCH_BLOCK}.merge(params)
           per_page = params[:per_page] || DEFAULT_BATCH_BLOCK
 
+          launch_batch(data,
+            method: method,
+            per_page: per_page,
+            people_api: people_api,
+            silent: silent
+          )
+        end
+
+        def launch_batch(data, method:, status: nil, job_mode: true, per_page: DEFAULT_BATCH_BLOCK, people_api: api&.people, silent: false)
           iteration  = 1; done = 0
           iterations = (data.length.to_f / per_page).ceil
 
-          Eco::API::Session::Batch::Status.new(enviro, queue: data, method: method).tap do |status|
+          status ||= Eco::API::Session::Batch::Status.new(enviro, queue: data, method: method)
+          status.tap do |status|
             start_time  = Time.now
             start_slice = Time.now; slice = []
+            pending_for_server_error = data.to_a[0..-1]
             data.each_slice(per_page) do |slice|
               msg  = "starting batch '#{method}' iteration #{iteration}/#{iterations},"
               msg += " with #{slice.length} entries of #{data.length} -- #{done} done"
@@ -115,11 +127,14 @@ module Eco
 
               start_slice = Time.now
               offer_retry_on(Ecoportal::API::Errors::TimeOut) do
-                people_api.batch do |batch|
+                people_api.batch(job_mode: false) do |batch|
                   slice.each do |person|
                     batch.public_send(method, person) do |response|
                       faltal("Request with no response") unless !!response
-                      status[person] = response
+                      unless server_error?(response)
+                        pending_for_server_error.delete(person)
+                        status[person] = response
+                      end
                     end
                   end
                 end # end batch
@@ -128,9 +143,31 @@ module Eco
               iteration  += 1
               done       += slice.length
             end # next slice
+
+            # temporary working around (due to back-end problems with batch/jobs)
+            unless pending_for_server_error.empty?
+              msg = "Going to re-try #{pending_for_server_error.count} due to server errors"
+              logger.info(msg) unless silent
+              launch_batch(pending_for_server_error,
+                status: status,
+                method: method,
+                job_mode: false,
+                per_page: per_page,
+                people_api: people_api,
+                silent: silent
+              )
+            end
           end
         end
 
+        def server_error?(response)
+          res_status   = response.status
+          server_error = !res_status || res_status.server_error?
+          other_error  = !server_error && (!res_status.code || res_status.code < 100)
+          no_body      = !server_error && !other_error  && !response.body
+          server_error || other_error || no_body
+        end
+
         def offer_retry_on(error_type, retries_left = 3, &block)
           begin
             block.call

data/lib/eco/api/session.rb

@@ -17,6 +17,11 @@ module Eco
         logger.debug("LINE COMMAND: #{$0} #{ARGV.join(" ")}")
       end
 
+      # @retrun [String] the name of the current environment
+      def enviro_name
+        config.active_enviro
+      end
+
       # @return [Eco::API::Session::Batch] provides helper to launch batch operations.
       def batch
         @batch  ||= Batch.new(enviro)

data/lib/eco/api/usecases/ooze_samples/ooze_base_case.rb

@@ -20,10 +20,11 @@ class Eco::API::UseCases::OozeSamples::OozeBaseCase < Eco::API::Common::Loaders:
   end
 
   # Write here your script
-  def process_ooze(ooze = target)
+  def process_ooze(ooz = target)
+    ooz ||= ooze
     raise "You need to inherit from this class ('#{self.class}') and call super with a block" unless block_given?
-    yield(ooze)
-    update_ooze(ooze)
+    yield(ooz)
+    update_ooze(ooz)
   end
 
   protected
@@ -109,10 +110,12 @@ class Eco::API::UseCases::OozeSamples::OozeBaseCase < Eco::API::Common::Loaders:
     exit_error "Stage '#{id_name}' doesn't exist in ooze '#{ooze_id}'"
   end
 
+  # It fill update the ooze only if it's dirty (it carries changes)
+  # @return [Boolean, Response] `false` if there was not request against the server, `Response` otherwise
   def update_ooze(ooze = target)
     if !options[:simulate]
-      return unless dirty?(ooze)
-      
+      return false unless dirty?(ooze)
+
       ooze.validate.tap do |validation|
         raise validation if validation.is_a?(String)
       end
@@ -133,6 +136,7 @@ class Eco::API::UseCases::OozeSamples::OozeBaseCase < Eco::API::Common::Loaders:
       end
       backup_patch!(ooze)
       exit(0) if dirty?(ooze) && dry_count > DRY_COUNT
+      false
     end
   end
 

data/lib/eco/api/usecases/ooze_samples/ooze_run_base_case.rb

@@ -39,9 +39,9 @@ class Eco::API::UseCases::OozeSamples::OozeRunBaseCase < Eco::API::UseCases::Ooz
     options.dig(:source, :stage_id)
   end
 
-  def update_ooze
+  def update_ooze(ooz = target)
     prompt_to_confirm!
-    super
+    super(ooz)
   end
 
   def exit_if_no_changes!

data/lib/eco/api/usecases/ooze_samples/register_export_case.rb

@@ -0,0 +1,184 @@
+# Use case to update a register
+# @note
+#   - You can define methods `filters` and `search` to change the target entries of the register
+#   - You need to define the `process_ooze` method
+# This case expects `options[:source][:register_id]`
+class Eco::API::UseCases::OozeSamples::RegisterExportCase < Eco::API::Common::Loaders::UseCase
+
+  class << self
+    # @return [Integer] the number of pages to be processed in each batch
+    def batch_size(size = nil)
+      @batch_size ||= 25
+      return @batch_size unless size
+      @batch_size = size
+    end
+  end
+
+  include Eco::API::UseCases::OozeSamples::Helpers
+
+  name "register-export-case"
+  type :other
+
+  attr_reader :session, :options, :usecase
+  attr_reader :target
+
+  def main(session, options, usecase)
+    options[:end_get] = false
+    @session = session; @options = options; @usecase = usecase
+    @target  = nil
+    raise "You need to inherit from this class ('#{self.class}') and call super with a block" unless block_given?
+    with_each_entry do |ooze|
+      process_ooze(ooze)
+    end
+    yield
+  end
+
+  # Write here your script
+  def process_ooze(ooze = target)
+    raise "You need to inherit from this class ('#{self.class}') and call super with a block" unless block_given?
+    yield(ooze)
+  end
+
+  private
+
+  def new_target(object)
+    @target = object
+  end
+
+  def with_each_entry
+    batched_search_results do |page_results|
+      page_results.each do |page_result|
+        if ooz = build_full_ooze(page_result.id)
+          yield(ooz)
+        end
+      end
+    end
+  end
+
+  # It builds a full page model (not updatable)
+  # @note If it's a page instance with stages, there's where it will be handy
+  # @param ooze_id [String]
+  # @return [Ecoportal::API::V2::Page]
+  def build_full_ooze(ooze_id)
+    if page = ooze(ooze_id)
+      return page unless page.is_a?(Ecoportal::API::V2::Pages::PageStage)
+      secs_doc = page.sections.doc
+      flds_doc = page.components.doc
+      pending_stage_ids = page.stages.map(&:id) - [page.current_stage_id]
+      pending_stage_ids.each do |id|
+        if page = stage(id, ooze: page)
+          page.sections.doc.each do |sec_doc|
+            unless secs_doc.find {|sec| sec["id"] == sec_doc["id"]}
+              secs_doc << sec_doc
+            end
+          end
+          page.components.doc.each do |comp_doc|
+            unless flds_doc.find {|fld| fld["id"] == comp_doc["id"]}
+              flds_doc << comp_doc
+            end
+          end
+        end
+      end
+      Ecoportal::API::V2::Page.new(page.doc)
+    end
+  end
+
+  #def update_oozes(batched_oozes = batch_queue)
+  #  batched_oozes.each do |ooze|
+  #    update_ooze(ooze)
+  #  end
+  #  batched_oozes.clear
+  #end
+
+  def batched_search_results
+    raise "Missing block. It yields in slices of #{self.class.batch_size} results" unless block_given?
+    results_preview
+    results = []
+    apiv2.registers.search(register_id, search_options) do |page_result|
+      results << page_result
+      if results.length >= self.class.batch_size
+        yield(results)
+        results = []
+      end
+    end
+    yield(results) unless results.empty?
+  end
+
+  def ooze(ooze_id = nil, stage_id: nil)
+    return target unless ooze_id
+    apiv2.pages.get(ooze_id, stage_id: stage_id).tap do |ooze|
+      if ooze
+        new_target(ooze)
+        logger.info("Got #{object_reference(ooze)}")
+      else
+        exit_error "Could not get ooze '#{ooze_id}'"
+      end
+    end
+  end
+
+  def stage(id_name = nil, ooze: target)
+    if ooze_id = ooze && ooze.id
+      exit_error "#{object_reference(ooze)} does not have stages!" unless ooze.stages?
+    else
+      exit_error "There's no target ooze to get retrieve stages from"
+    end
+
+    if stg = ooze.stages[id_name] || ooze.stages.get_by_name(id_name)
+      return ooze if ooze.respond_to?(:current_stage_id) && (ooze.current_stage_id == stg.id)
+      return apiv2.pages.get(ooze_id, stage_id: stg.id).tap do |stage|
+        if stage
+          new_target(stage)
+          logger.info("Got #{object_reference(stage)} from #{object_reference(ooze)}")
+        else
+          exit_error "Could not get stage '#{id_name}' in ooze '#{ooze_id}'"
+        end
+      end
+    end
+    exit_error "Stage '#{id_name}' doesn't exist in ooze '#{ooze_id}'"
+  end
+
+  def results_preview
+    apiv2.registers.search(register_id, search_options.merge(only_first: true)).tap do |search_results|
+      str_results = "Total target entries: #{search_results.total} (out of #{search_results.total_before_filtering})"
+      session.prompt_user("Do you want to proceed (y/N):", explanation: str_results, default: "N", timeout: 10) do |res|
+        unless res.upcase.start_with?("Y")
+          puts "..."
+          logger.info "Aborting script..."
+          exit(0)
+        end
+      end
+    end
+  end
+
+  def search_options
+    @search_options ||= {}.tap do |opts|
+      opts.merge!(sort: "created_at")
+      opts.merge!(dir:  "asc")
+      opts.merge!(query:   conf_search) if conf_search
+      opts.merge!(filters: conf_filters)
+    end
+  end
+
+  def conf_filters
+    return filters if self.respond_to?(:filters)
+    []
+  end
+
+  def conf_search
+    return search if self.respond_to?(:search)
+  end
+
+  def register_id
+    options.dig(:source, :register_id)
+  end
+
+  def apiv2
+   @apiv2 ||= session.api(version: :oozes)
+  end
+
+  def exit_error(msg)
+    logger.error(msg)
+    exit(1)
+  end
+
+end

data/lib/eco/api/usecases/ooze_samples/register_update_case.rb

@@ -17,12 +17,30 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
   name "register-update-case"
   type :other
 
+  attr_reader :retrieved_oozes, :non_retrieved_oozes
+  attr_reader :total_search_oozes, :dupped_search_oozes
+  attr_reader :ooze_result_ids, :updated_oozes, :failed_update_oozes
+
   def main(session, options, usecase, &block)
+    @retrieved_oozes = 0
+    @non_retrieved_oozes = 0
+    @dupped_search_oozes = 0
+    @ooze_result_ids     = {}
+    @updated_oozes       = 0
+    @failed_update_oozes = 0
+
     super(session, options, usecase) do
       with_each_entry do |ooze|
         process_ooze(&block)
       end
     end
+    msg  = "Run end:\n"
+    msg += "  • Search results: #{total_search_oozes}\n"
+    msg += "  • Duplicated search results #{dupped_search_oozes}\n"
+    msg += "  • Retrieved a total of #{retrieved_oozes}\n"
+    msg += "  • Could not get #{non_retrieved_oozes} oozes.\n"
+    msg += "  • Updated #{updated_oozes} oozes (failed update on #{failed_update_oozes} oozes).\n"
+    logger.info(msg)
   end
 
   def process_ooze(ooze = target)
@@ -47,7 +65,14 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
   def with_each_entry
     batched_search_results do |page_results|
       page_results.each do |page_result|
+        if ooze_result_ids[page_result.id]
+          @dupped_search_oozes += 1
+        else
+          ooze_result_ids[page_result.id] = true
+        end
+
         if pending = queue_shift(page_result.id)
+
           if dirty?(pending)
             msg  = "Inconsistent search results. "
             msg << "Launching update on '#{object_reference(pending)}' to be able to queue it back"
@@ -56,7 +81,11 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
           end
         end
         if ooz = ooze(page_result.id)
+          @retrieved_oozes += 1
           yield(ooz)
+        else
+          @non_retrieved_oozes += 1
+          console.warn "Could not get page #{page_result.id}"
         end
       end
       update_oozes
@@ -113,7 +142,15 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
 
   def update_oozes(batched_oozes = batch_queue)
     batched_oozes.each do |ooze|
-      update_ooze(ooze)
+      update_ooze(ooze).tap do |result|
+        if result.is_a?(Ecoportal::API::Common::Response)
+          if result.success?
+            @updated_oozes += 1
+          else
+            @failed_update_oozes +=1
+          end
+        end
+      end
     end
     batched_oozes.clear
   end
@@ -127,6 +164,7 @@ class Eco::API::UseCases::OozeSamples::RegisterUpdateCase < Eco::API::UseCases::
 
   def results_preview
     apiv2.registers.search(register_id, search_options.merge(only_first: true)).tap do |search_results|
+      @total_search_oozes = search_results.total
       str_results = "Total target entries: #{search_results.total} (out of #{search_results.total_before_filtering})"
       session.prompt_user("Do you want to proceed (y/N):", explanation: str_results, default: "N", timeout: 10) do |res|
         unless res.upcase.start_with?("Y")

data/lib/eco/api/usecases/ooze_samples.rb

@@ -10,6 +10,7 @@ end
 
 require_relative 'ooze_samples/helpers'
 require_relative 'ooze_samples/ooze_base_case'
+require_relative 'ooze_samples/register_export_case'
 require_relative 'ooze_samples/ooze_run_base_case'
 require_relative 'ooze_samples/ooze_update_case'
 require_relative 'ooze_samples/ooze_from_doc_case'

data/lib/eco/cli/config/default/workflow.rb

@@ -3,6 +3,8 @@ ASSETS.cli.config do |config|
 
     io = nil
     rescued = false
+    cases_with_input  = nil
+    cases_with_output = nil
 
     # default rescue
     wf.rescue do |exception, io|
@@ -165,9 +167,16 @@ ASSETS.cli.config do |config|
     wf.on(:end) do |wf_end, io|
       get_people     = io.options.dig(:people, :get)
       partial_update = get_people && get_people.dig(:type) == :partial
+
       unless !io.options[:end_get] || io.options[:dry_run] || partial_update
-        # Prevent getting people when there were no use cases that used them
-        unless io.people
+        people_update_cases = config.usecases.active(io: io).any? do |usecase, data|
+          [:transform, :sync].any? {|type| usecase.type == type}
+        end
+
+        if !people_update_cases
+          # Prevent getting people when there were no use cases that used them
+          io.session.logger.info("Won't be recaching people, as there haven't been any targetted updates")
+        elsif !io.people
           people = io.session.micro.people_cache
           io     = io.new(people: people)
         end

data/lib/eco/version.rb

@@ -1,3 +1,3 @@
 module Eco
-  VERSION = "2.0.42"
+  VERSION = "2.0.46"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eco-helpers
 version: !ruby/object:Gem::Version
-  version: 2.0.42
+  version: 2.0.46
 platform: ruby
 authors:
 - Oscar Segura
@@ -116,7 +116,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.3
+        version: 0.8.4
     - - "<"
       - !ruby/object:Gem::Version
         version: '0.9'
@@ -126,7 +126,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.3
+        version: 0.8.4
     - - "<"
       - !ruby/object:Gem::Version
         version: '0.9'
@@ -136,7 +136,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.19
+        version: 0.8.23
     - - "<"
       - !ruby/object:Gem::Version
         version: '0.9'
@@ -146,7 +146,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.19
+        version: 0.8.23
     - - "<"
       - !ruby/object:Gem::Version
         version: '0.9'
@@ -564,6 +564,7 @@ files:
 - lib/eco/api/usecases/ooze_samples/ooze_from_doc_case.rb
 - lib/eco/api/usecases/ooze_samples/ooze_run_base_case.rb
 - lib/eco/api/usecases/ooze_samples/ooze_update_case.rb
+- lib/eco/api/usecases/ooze_samples/register_export_case.rb
 - lib/eco/api/usecases/ooze_samples/register_update_case.rb
 - lib/eco/api/usecases/ooze_samples/target_oozes_update_case.rb
 - lib/eco/api/usecases/use_case.rb