ecoportal-api-v2 1.1.7 → 2.0.0

data/lib/ecoportal/api/v2/registers.rb

@@ -2,26 +2,29 @@ require 'base64'
 module Ecoportal
   module API
     class V2
-      # @attr_reader client [Common::Client] a `Common::Client` object that holds the configuration of the api connection.
+      # @attr_reader client [Common::Client] a `Common::Client` object that
+      #   holds the configuration of the api connection.
       class Registers
         extend Common::BaseClass
         include Enumerable
         include Common::Content::DocHelpers
 
-        class_resolver :register_class,  "Ecoportal::API::V2::Registers::Register"
+        class_resolver :register_class, "Ecoportal::API::V2::Registers::Register"
         class_resolver :register_search_result_class, "Ecoportal::API::V2::Registers::PageResult"
         class_resolver :register_search_results, "Ecoportal::API::V2::Registers::SearchResults"
 
         attr_reader :client
 
-        # @param client [Common::Client] a `Common::Client` object that holds the configuration of the api connection.
+        # @param client [Common::Client] a `Common::Client` object that
+        #   holds the configuration of the api connection.
         # @return [Registers] an instance object ready to make registers api requests.
         def initialize(client)
           @client = client
         end
 
         def each(params: {}, &block)
-          return to_enum(:each) unless block
+          return to_enum(:each, params: params) unless block
+
           get.each(&block)
         end
 
@@ -29,7 +32,11 @@ module Ecoportal
         # @return [Enumerable<Register>] an `Enumerable` with all schemas already wrapped as `Register` objects.
         def get
           response = client.get("/templates")
-          Common::Content::WrappedResponse.new(response, register_class, key: "registers")
+          Common::Content::WrappedResponse.new(
+            response,
+            register_class,
+            key: "registers"
+          )
         end
 
         # Gets all the oozes/pages of `register_id` matching the `options`
@@ -41,18 +48,21 @@ module Ecoportal
         # @yield [result]  something to do with search page-result.
         # @yieldparam result [Ecoportal::V2::Registers::PageResult] a page result.
         # @return [Ecoportal::API::V2::Registers, Ecoportal::API::V2::Registers::SearchResults]
-        def search(register_id, options = {})
+        def search(register_id, options = {}) # rubocop:disable Metrics/AbcSize
           only_first = options.delete(:only_first)
           options    = build_options(options)
 
           if only_first
             response = client.get("/registers/#{register_id}/search", params: options)
             raise "Request failed - Status #{response.status}: #{response.body}" unless response.success?
+
             return register_search_results.new(response.body["data"])
           end
 
           cursor_id = nil
-          results = 0; total = nil
+          results   = 0
+          total     = nil
+
           loop do
             options.update(cursor_id: cursor_id) if cursor_id
             response = client.get("/registers/#{register_id}/search", params: options)
@@ -61,12 +71,14 @@ module Ecoportal
             data    = response.body["data"]
             total ||= data["total"]
             if total != data["total"]
-              msg  = "Change of total in search results. Probably due to changes that affect the filter (register: #{register_id}):"
+              msg  = "Change of total in search results. "
+              msg << "Probably due to changes that affect the filter"
+              msg << "(register: #{register_id}):"
               print_search_status(msg, total, results, cursor_id, data, options)
               #total = data["total"]
             end
 
-            unless total == 0
+            unless total&.zero?
               results += data["results"].length
               print_progress(results, total)
             end
@@ -77,12 +89,15 @@ module Ecoportal
             end
 
             break if total <= results
+
             unless data["cursor_id"]
-              msg  = "Possible error... finishing search for lack of cursor_id in response:"
+              msg = "Possible error... finishing search for lack of cursor_id in response:"
               print_search_status(msg, total, results, cursor_id, data, options)
             end
+
             break unless (cursor_id = data["cursor_id"])
           end
+
           self
         end
 
@@ -96,14 +111,14 @@ module Ecoportal
             options.each do |key, value|
               if key == :filters && value.any?
                 ret[key] = {filters: value}.to_json
-              else
-                ret[key] = value if key
+              elsif key
+                ret[key] = value
               end
             end
           end
         end
 
-        def print_search_status(msg, total, results, cursor_id, data, options)
+        def print_search_status(msg, total, results, cursor_id, data, options) # rubocop:disable Metrics/ParameterLists
           msg += "\n"
           msg += "  • Original total: #{total}\n"
           msg += "    • Current total: #{data&.dig("total")}\n"

data/lib/ecoportal/api/v2/s3/data.rb

@@ -0,0 +1,27 @@
+module Ecoportal
+  module API
+    class V2
+      class S3
+        class Data < Common::Content::DoubleModel
+          passthrough :s3_endpoint, :AWSAccessKeyId
+          passthrough :policy, :signature
+          passthrough :user_tmpdir
+
+          def x_amz_server_side_encryption
+            doc['x-amz-server-side-encryption']
+          end
+
+          def [](key)
+            doc[key.to_s]
+          end
+
+          def user_id
+            return unless user_tmpdir
+
+            user_tmpdir.split('uploads/').last
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v2/s3/files/batch_upload.rb

@@ -0,0 +1,110 @@
+module Ecoportal
+  module API
+    class V2
+      class S3
+        class Files
+          # Class service to upload multiple files in one go
+          class BatchUpload
+            include Ecoportal::API::Common::Concerns::Benchmarkable
+            include Ecoportal::API::Common::Concerns::Threadable
+
+            SIZE_UNITS = 1_024 # KB
+            MIN_UNIT   = 1
+
+            attr_reader :files, :files_api
+
+            FileResult = Struct.new(:file) do
+              attr_accessor :poll, :s3_file_reference, :error
+
+              def error?
+                !error.nil?
+              end
+
+              def success?
+                poll&.success?
+              end
+
+              def container_id
+                return unless success?
+                poll.container_id
+              end
+            end
+
+            # @param files [Array<String>] the files to be uploaded
+            # @param files_api [API::S3::Files] the api object.
+            def initialize(files, files_api:)
+              @files     = [files].flatten.compact
+              @files_api = files_api
+            end
+
+            # Do the actual upload of the file
+            def upload!(benchmarking: false, threads: 1, **kargs) # rubocop:disable Metrics/AbcSize
+              bench        = benchmarking && benchmark_enabled?
+              bench_mth    = "#{self.class}##{__method__}"
+              thr_children = []
+              benchmarking("#{bench_mth}.#{files.count}_files", print: bench) do
+                with_preserved_thread_globals(report: false) do
+                  files.each do |file|
+                    new_thread(thr_children, max: threads) do
+                      file_results << (result = FileResult.new(file))
+
+                      s3_ref = nil
+
+                      kbytes = bench ? file_size(file) : 1
+
+                      benchmarking("#{bench_mth}.s3_upload (KB)", units: kbytes, print: bench) do
+                        s3_ref = result.s3_file_reference = s3_api.upload_file(file)
+                      end
+
+                      benchmarking("##{bench_mth}.ep_poll (KB)", units: kbytes, print: bench) do
+                        files_api.poll!(s3_ref, **kargs) do |poll|
+                          result.poll = poll
+                        end
+                      end
+                      yield(result) if block_given?
+                    rescue *rescued_errors => err # each_file
+                      result.error = err
+                      yield(result) if block_given?
+                    end
+                  end
+                end
+              end
+
+              thr_children.each(&:join)
+              file_results
+            ensure
+              puts benchmark_summary(:all) if bench
+            end
+
+            private
+
+            def file_size(file, bytes: SIZE_UNITS)
+              return 1 unless File.exist?(file)
+
+              units = (File.size(file) / bytes).round(10)
+              [MIN_UNIT, units].max
+            end
+
+            def s3_api
+              files_api.s3_api
+            end
+
+            def file_results
+              @file_results ||= []
+            end
+
+            def rescued_errors
+              @rescued_errors ||= [
+                Ecoportal::API::V2::S3::MissingLocalFile,
+                Ecoportal::API::V2::S3::CredentialsGetFailed,
+                Ecoportal::API::V2::S3::Files::FailedPollRequest,
+                Ecoportal::API::V2::S3::Files::CantCheckStatus,
+                Ecoportal::API::Errors::TimeOut
+              ].freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v2/s3/files/poll.rb

@@ -0,0 +1,82 @@
+module Ecoportal
+  module API
+    class V2
+      class S3
+        class Files
+          class Poll < Common::Content::DoubleModel
+            passthrough :poll_url
+            embeds_one :status, nullable: true, klass: "Ecoportal::API::V2::S3::Files::PollStatus"
+
+            def poll_id
+              return @poll_id       if instance_variable_defined?(:@poll_id)
+              return @poll_id = nil unless (uri = parsed_poll_url)
+
+              @poll_id = uri.path.split('/poll/').last
+            end
+            alias_method :id, :poll_id
+
+            # The final File eP container id
+            def container_id
+              return unless status?
+
+              status.container_id
+            end
+
+            def status?
+              !doc['status'].nil?
+            end
+
+            def status=(value)
+              case value
+              when NilClass
+                doc["status"] = nil
+              when Ecoportal::API::V2::S3::Files::PollStatus
+                doc["status"] = JSON.parse(value.to_json)
+              when Hash
+                doc["status"] = value
+              else
+                # TODO
+                raise "Invalid set on status: Need nil, PollStatus or Hash; got #{value.class}"
+              end
+
+              remove_instance_variable("@status") if defined?(@status)
+            end
+
+            def pending?
+              !complete?
+            end
+
+            def complete?
+              status&.complete?
+            end
+
+            def success?
+              status&.success?
+            end
+
+            def failed?
+              status&.failed?
+            end
+
+            def timeout?
+              status&.timeout?
+            end
+
+            private
+
+            def parsed_poll_url
+              return nil unless poll_url
+
+              require 'uri'
+              URI.parse(poll_url)
+            end
+
+            def callbacks
+              @callbacks ||= []
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v2/s3/files/poll_status.rb

@@ -0,0 +1,52 @@
+module Ecoportal
+  module API
+    class V2
+      class S3
+        class Files
+          class PollStatus < Common::Content::DoubleModel
+            class FileContainer < Common::Content::DoubleModel
+              passkey :id
+              passthrough :file_container_id # same as :id :)
+              passthrough :name, :label
+              passthrough :file_size, :content_type
+              passthrough :tags
+              passboolean :archived
+              passthrough :active_person, :user_name
+              passdate :created_at, :updated_at, :file_update_at
+            end
+
+            # PollStatus
+            passthrough :status
+            embeds_one :file, klass: FileContainer
+
+            def container_id
+              return unless success?
+
+              file&.id
+            end
+
+            def complete?
+              success? || failed?
+            end
+
+            def timeout?
+              statut == 'timeout'
+            end
+
+            def pending?
+              status == "pending"
+            end
+
+            def success?
+              status == "success"
+            end
+
+            def failed?
+              status == "failed"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v2/s3/files.rb

@@ -0,0 +1,132 @@
+module Ecoportal
+  module API
+    class V2
+      # @attr_reader client [Common::Client] a `Common::Client` object that
+      #   holds the configuration of the api connection.
+      class S3
+        class Files
+          class FailedPollRequest < StandardError; end
+          class CantCheckStatus < StandardError; end
+
+          extend Common::BaseClass
+
+          POLL_TIMEOUT       = 240
+          DELAY_STATUS_CHECK = 5
+
+          class_resolver :poll_class, "Ecoportal::API::V2::S3::Files::Poll"
+          class_resolver :poll_status_class, "Ecoportal::API::V2::S3::Files::PollStatus"
+
+          attr_reader :client, :s3_api
+
+          # @param client [Common::Client] a `Common::Client` object that holds the configuration of the api connection.
+          # @param s3_api [V2::S3] the parent S3 api object.
+          # @return [Schemas] an instance object ready to make schema api requests.
+          def initialize(client, s3_api:)
+            @client = client
+            @s3_api = s3_api
+          end
+
+          # Helper to upload multiple files at once
+          def upload!(files, **kargs, &block)
+            BatchUpload.new(files, files_api: self).upload!(**kargs, &block)
+          end
+
+          # Tell ecoPortal to register the file in an actual File Container
+          # @note this is essential to do for the file to be attachable.
+          #   - eP will scan the file and give create the proper data model that
+          #     refers to the file (FileContainer).
+          #   - FileContainers also have a file versioning sytem.
+          #   - A file container belongs only to one single organization.
+          # @yield [poll] block early, when the poll is created
+          # @yieldparam poll [Ecoportal::API::V2::S3::Files::Poll, NilClass] the creatd poll
+          # @param s3_file_reference [Hash]
+          # @return [Ecoportal::API::V2::S3::Files::PollStatus, NilClass]
+          #    when finalized (success, failed or timeout)
+          def poll!(
+            s3_file_reference,
+            check_delay:   DELAY_STATUS_CHECK,
+            raise_timeout: false
+          )
+            raise ArgumentError, "Block required. None given" unless block_given?
+
+            poll = create_poll(s3_file_reference)
+            yield(poll) if block_given?
+
+            wait_for_poll_completion(poll.id, check_delay: check_delay).tap do |status|
+              poll.status = status
+
+              next unless raise_timeout
+              next if status&.complete?
+
+              msg  = "File poll `#{poll.id}` not complete. "
+              msg << "Probably timeout after #{POLL_TIMEOUT} seconds. "
+              msg << "Current status: #{poll.status.status}"
+
+              raise Ecoportal::API::Errors::TimeOut, msg
+            end
+          end
+
+          # Create Poll Request to ecoPortal so it registers the file
+          # in an actual File Container
+          # @param s3_file_reference [Hash]
+          # @return [Ecoportal::API::V2::S3::Files::Poll, NilClass]
+          def create_poll(s3_file_reference)
+            response = client.post("/s3/files", data: s3_file_reference)
+            body     = body_data(response.body)
+
+            return poll_class.new(body) if response.success?
+
+            msg = "Could not create poll request for\n"
+            msg << JSON.pretty_generate(s3_file_reference)
+            msg << "\n - Error #{response.status}: #{body}"
+            raise FailedPollRequest, msg
+          end
+
+          # Check progress on the File Poll status
+          # @param poll_id [String]
+          def poll_status(poll_id)
+            response = client.get("/s3/files/poll/#{poll_id}")
+            body     = body_data(response.body)
+
+            return poll_status_class.new(body) if response.success?
+
+            msg = "Could not get status for poll '#{poll_id}' "
+            msg << "- Error #{response.status}: #{body}"
+            raise CantCheckStatus, msg
+          end
+
+          private
+
+          def body_data(body)
+            return body unless body.is_a?(Hash)
+            return body unless body.key?("data")
+
+            body["data"]
+          end
+
+          # @note timeout library is evil. So we make poor-man timeout.
+          #   https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
+          def wait_for_poll_completion(poll_id, check_delay: DELAY_STATUS_CHECK)
+            before = Time.now
+
+            loop do
+              status = poll_status(poll_id)
+              break status if status&.complete?
+
+              if Time.now >= before + POLL_TIMEOUT
+                status&.status = 'timeout'
+                break status
+              end
+
+              sleep(check_delay)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+require 'ecoportal/api/v2/s3/files/poll'
+require 'ecoportal/api/v2/s3/files/poll_status'
+require 'ecoportal/api/v2/s3/files/batch_upload'

data/lib/ecoportal/api/v2/s3/upload.rb

@@ -0,0 +1,154 @@
+module Ecoportal
+  module API
+    class V2
+      class S3
+        class Upload
+          class MissingFile < ArgumentError; end
+
+          DEFAULT_MIME_TYPE = 'application/octet-stream'.freeze
+          EXPECTED_CODE     = "204".freeze # No Content
+          MAX_RETRIES       = 5
+
+          attr_reader :filename
+
+          def initialize(credentials, file:)
+            msg = "Expecting #{credentials_class}. Given: #{credentials.class}"
+            raise ArgumentError, msg unless credentials.is_a?(credentials_class)
+
+            @credentials = credentials
+            @filename    = file
+          end
+
+          def upload!
+            require_deps!
+
+            @response = Net::HTTP.start(
+              uri.hostname,
+              uri.port,
+              use_ssl: true
+            ) do |https|
+              https.max_retries = self.class::MAX_RETRIES
+              https.request(request)
+            end
+
+            return unless success?
+
+            yield(s3_file_reference) if block_given?
+            s3_file_reference
+          end
+
+          def failed?
+            return false unless response
+
+            response.code != EXPECTED_CODE
+          end
+
+          def success?
+            return false unless response
+
+            response.code == EXPECTED_CODE
+          end
+
+          def s3_file_reference
+            return unless success?
+
+            @s3_file_reference ||= {
+              'filename' => file_basename,
+              'filetype' => mime_type,
+              'filesize' => file_size,
+              'path'     => s3_file_path
+            }
+          end
+
+          private
+
+          attr_reader :credentials
+          attr_reader :response
+
+          def request
+            Net::HTTP::Post.new(uri).tap do |req|
+              req.set_form form_data, 'multipart/form-data'
+            end
+          end
+
+          def uri
+            @uri ||= URI(credentials[:s3_endpoint])
+          end
+
+          def form_data
+            [
+              ['key', s3_file_path],
+              *aws_params,
+              *file_params
+            ]
+          end
+
+          def s3_file_path
+            "#{user_tmpdir}/#{file_basename}"
+          end
+
+          def user_tmpdir
+            credentials[:user_tmpdir]
+          end
+
+          def aws_params
+            params = %i[AWSAccessKeyId policy signature x-amz-server-side-encryption]
+            params.map do |key|
+              [key.to_s, credentials[key]]
+            end
+          end
+
+          def file_params
+            [
+              ['content-type', mime_type],
+              ['file',         io_stream]
+            ]
+          end
+
+          def mime_type
+            @mime_type ||= lib_mime_type || DEFAULT_MIME_TYPE
+          end
+
+          def lib_mime_type
+            MIME::Types.type_for(file_extension).first&.content_type
+          end
+
+          def io_stream
+            require_file!
+            File.open(file_fullname)
+          end
+
+          def credentials_class
+            Ecoportal::API::V2::S3::Data
+          end
+
+          def file_size
+            @file_size ||= File.size(file_fullname)
+          end
+
+          def file_extension
+            @file_extension ||= File.extname(filename)[1..]
+          end
+
+          def file_basename
+            @file_basename ||= File.basename(filename)
+          end
+
+          def file_fullname
+            @file_fullname ||= File.expand_path(filename)
+          end
+
+          def require_file!
+            msg = "File '#{file_basename}' doesn't exist"
+            raise MissingFile, msg unless File.exist?(file_fullname)
+          end
+
+          def require_deps!
+            require 'net/http'
+            require 'mime/types'
+          end
+        end
+      end
+    end
+  end
+end