ecoportal-api-v2 1.1.8 → 2.0.5

data/lib/ecoportal/api/v2/s3/files/batch_upload.rb

@@ -0,0 +1,110 @@
+module Ecoportal
+  module API
+    class V2
+      class S3
+        class Files
+          # Class service to upload multiple files in one go
+          class BatchUpload
+            include Ecoportal::API::Common::Concerns::Benchmarkable
+            include Ecoportal::API::Common::Concerns::Threadable
+
+            SIZE_UNITS = 1_024 # KB
+            MIN_UNIT   = 1
+
+            attr_reader :files, :files_api
+
+            FileResult = Struct.new(:file) do
+              attr_accessor :poll, :s3_file_reference, :error
+
+              def error?
+                !error.nil?
+              end
+
+              def success?
+                poll&.success?
+              end
+
+              def container_id
+                return unless success?
+                poll.container_id
+              end
+            end
+
+            # @param files [Array<String>] the files to be uploaded
+            # @param files_api [API::S3::Files] the api object.
+            def initialize(files, files_api:)
+              @files     = [files].flatten.compact
+              @files_api = files_api
+            end
+
+            # Do the actual upload of the file
+            def upload!(benchmarking: false, threads: 1, **kargs) # rubocop:disable Metrics/AbcSize
+              bench        = benchmarking && benchmark_enabled?
+              bench_mth    = "#{self.class}##{__method__}"
+              thr_children = []
+              benchmarking("#{bench_mth}.#{files.count}_files", print: bench) do
+                with_preserved_thread_globals(report: false) do
+                  files.each do |file|
+                    new_thread(thr_children, max: threads) do
+                      file_results << (result = FileResult.new(file))
+
+                      s3_ref = nil
+
+                      kbytes = bench ? file_size(file) : 1
+
+                      benchmarking("#{bench_mth}.s3_upload (KB)", units: kbytes, print: bench) do
+                        s3_ref = result.s3_file_reference = s3_api.upload_file(file)
+                      end
+
+                      benchmarking("##{bench_mth}.ep_poll (KB)", units: kbytes, print: bench) do
+                        files_api.poll!(s3_ref, **kargs) do |poll|
+                          result.poll = poll
+                        end
+                      end
+                      yield(result) if block_given?
+                    rescue *rescued_errors => err # each_file
+                      result.error = err
+                      yield(result) if block_given?
+                    end
+                  end
+                end
+              end
+
+              thr_children.each(&:join)
+              file_results
+            ensure
+              puts benchmark_summary(:all) if bench
+            end
+
+            private
+
+            def file_size(file, bytes: SIZE_UNITS)
+              return 1 unless File.exist?(file)
+
+              units = (File.size(file) / bytes).round(10)
+              [MIN_UNIT, units].max
+            end
+
+            def s3_api
+              files_api.s3_api
+            end
+
+            def file_results
+              @file_results ||= []
+            end
+
+            def rescued_errors
+              @rescued_errors ||= [
+                Ecoportal::API::V2::S3::MissingLocalFile,
+                Ecoportal::API::V2::S3::CredentialsGetFailed,
+                Ecoportal::API::V2::S3::Files::FailedPollRequest,
+                Ecoportal::API::V2::S3::Files::CantCheckStatus,
+                Ecoportal::API::Errors::TimeOut
+              ].freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v2/s3/files/poll.rb

@@ -0,0 +1,82 @@
+module Ecoportal
+  module API
+    class V2
+      class S3
+        class Files
+          class Poll < Common::Content::DoubleModel
+            passthrough :poll_url
+            embeds_one :status, nullable: true, klass: "Ecoportal::API::V2::S3::Files::PollStatus"
+
+            def poll_id
+              return @poll_id       if instance_variable_defined?(:@poll_id)
+              return @poll_id = nil unless (uri = parsed_poll_url)
+
+              @poll_id = uri.path.split('/poll/').last
+            end
+            alias_method :id, :poll_id
+
+            # The final File eP container id
+            def container_id
+              return unless status?
+
+              status.container_id
+            end
+
+            def status?
+              !doc['status'].nil?
+            end
+
+            def status=(value)
+              case value
+              when NilClass
+                doc["status"] = nil
+              when Ecoportal::API::V2::S3::Files::PollStatus
+                doc["status"] = JSON.parse(value.to_json)
+              when Hash
+                doc["status"] = value
+              else
+                # TODO
+                raise "Invalid set on status: Need nil, PollStatus or Hash; got #{value.class}"
+              end
+
+              remove_instance_variable("@status") if defined?(@status)
+            end
+
+            def pending?
+              !complete?
+            end
+
+            def complete?
+              status&.complete?
+            end
+
+            def success?
+              status&.success?
+            end
+
+            def failed?
+              status&.failed?
+            end
+
+            def timeout?
+              status&.timeout?
+            end
+
+            private
+
+            def parsed_poll_url
+              return nil unless poll_url
+
+              require 'uri'
+              URI.parse(poll_url)
+            end
+
+            def callbacks
+              @callbacks ||= []
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v2/s3/files/poll_status.rb

@@ -0,0 +1,52 @@
+module Ecoportal
+  module API
+    class V2
+      class S3
+        class Files
+          class PollStatus < Common::Content::DoubleModel
+            class FileContainer < Common::Content::DoubleModel
+              passkey :id
+              passthrough :file_container_id # same as :id :)
+              passthrough :name, :label
+              passthrough :file_size, :content_type
+              passthrough :tags
+              passboolean :archived
+              passthrough :active_person, :user_name
+              passdate :created_at, :updated_at, :file_update_at
+            end
+
+            # PollStatus
+            passthrough :status
+            embeds_one :file, klass: FileContainer
+
+            def container_id
+              return unless success?
+
+              file&.id
+            end
+
+            def complete?
+              success? || failed?
+            end
+
+            def timeout?
+              statut == 'timeout'
+            end
+
+            def pending?
+              status == "pending"
+            end
+
+            def success?
+              status == "success"
+            end
+
+            def failed?
+              status == "failed"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v2/s3/files.rb

@@ -0,0 +1,132 @@
+module Ecoportal
+  module API
+    class V2
+      # @attr_reader client [Common::Client] a `Common::Client` object that
+      #   holds the configuration of the api connection.
+      class S3
+        class Files
+          class FailedPollRequest < StandardError; end
+          class CantCheckStatus < StandardError; end
+
+          extend Common::BaseClass
+
+          POLL_TIMEOUT       = 240
+          DELAY_STATUS_CHECK = 5
+
+          class_resolver :poll_class, "Ecoportal::API::V2::S3::Files::Poll"
+          class_resolver :poll_status_class, "Ecoportal::API::V2::S3::Files::PollStatus"
+
+          attr_reader :client, :s3_api
+
+          # @param client [Common::Client] a `Common::Client` object that holds the configuration of the api connection.
+          # @param s3_api [V2::S3] the parent S3 api object.
+          # @return [Schemas] an instance object ready to make schema api requests.
+          def initialize(client, s3_api:)
+            @client = client
+            @s3_api = s3_api
+          end
+
+          # Helper to upload multiple files at once
+          def upload!(files, **kargs, &block)
+            BatchUpload.new(files, files_api: self).upload!(**kargs, &block)
+          end
+
+          # Tell ecoPortal to register the file in an actual File Container
+          # @note this is essential to do for the file to be attachable.
+          #   - eP will scan the file and give create the proper data model that
+          #     refers to the file (FileContainer).
+          #   - FileContainers also have a file versioning sytem.
+          #   - A file container belongs only to one single organization.
+          # @yield [poll] block early, when the poll is created
+          # @yieldparam poll [Ecoportal::API::V2::S3::Files::Poll, NilClass] the creatd poll
+          # @param s3_file_reference [Hash]
+          # @return [Ecoportal::API::V2::S3::Files::PollStatus, NilClass]
+          #    when finalized (success, failed or timeout)
+          def poll!(
+            s3_file_reference,
+            check_delay:   DELAY_STATUS_CHECK,
+            raise_timeout: false
+          )
+            raise ArgumentError, "Block required. None given" unless block_given?
+
+            poll = create_poll(s3_file_reference)
+            yield(poll) if block_given?
+
+            wait_for_poll_completion(poll.id, check_delay: check_delay).tap do |status|
+              poll.status = status
+
+              next unless raise_timeout
+              next if status&.complete?
+
+              msg  = "File poll `#{poll.id}` not complete. "
+              msg << "Probably timeout after #{POLL_TIMEOUT} seconds. "
+              msg << "Current status: #{poll.status.status}"
+
+              raise Ecoportal::API::Errors::TimeOut, msg
+            end
+          end
+
+          # Create Poll Request to ecoPortal so it registers the file
+          # in an actual File Container
+          # @param s3_file_reference [Hash]
+          # @return [Ecoportal::API::V2::S3::Files::Poll, NilClass]
+          def create_poll(s3_file_reference)
+            response = client.post("/s3/files", data: s3_file_reference)
+            body     = body_data(response.body)
+
+            return poll_class.new(body) if response.success?
+
+            msg = "Could not create poll request for\n"
+            msg << JSON.pretty_generate(s3_file_reference)
+            msg << "\n - Error #{response.status}: #{body}"
+            raise FailedPollRequest, msg
+          end
+
+          # Check progress on the File Poll status
+          # @param poll_id [String]
+          def poll_status(poll_id)
+            response = client.get("/s3/files/poll/#{poll_id}")
+            body     = body_data(response.body)
+
+            return poll_status_class.new(body) if response.success?
+
+            msg = "Could not get status for poll '#{poll_id}' "
+            msg << "- Error #{response.status}: #{body}"
+            raise CantCheckStatus, msg
+          end
+
+          private
+
+          def body_data(body)
+            return body unless body.is_a?(Hash)
+            return body unless body.key?("data")
+
+            body["data"]
+          end
+
+          # @note timeout library is evil. So we make poor-man timeout.
+          #   https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
+          def wait_for_poll_completion(poll_id, check_delay: DELAY_STATUS_CHECK)
+            before = Time.now
+
+            loop do
+              status = poll_status(poll_id)
+              break status if status&.complete?
+
+              if Time.now >= before + POLL_TIMEOUT
+                status&.status = 'timeout'
+                break status
+              end
+
+              sleep(check_delay)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+require 'ecoportal/api/v2/s3/files/poll'
+require 'ecoportal/api/v2/s3/files/poll_status'
+require 'ecoportal/api/v2/s3/files/batch_upload'

data/lib/ecoportal/api/v2/s3/upload.rb

@@ -0,0 +1,154 @@
+module Ecoportal
+  module API
+    class V2
+      class S3
+        class Upload
+          class MissingFile < ArgumentError; end
+
+          DEFAULT_MIME_TYPE = 'application/octet-stream'.freeze
+          EXPECTED_CODE     = "204".freeze # No Content
+          MAX_RETRIES       = 5
+
+          attr_reader :filename
+
+          def initialize(credentials, file:)
+            msg = "Expecting #{credentials_class}. Given: #{credentials.class}"
+            raise ArgumentError, msg unless credentials.is_a?(credentials_class)
+
+            @credentials = credentials
+            @filename    = file
+          end
+
+          def upload!
+            require_deps!
+
+            @response = Net::HTTP.start(
+              uri.hostname,
+              uri.port,
+              use_ssl: true
+            ) do |https|
+              https.max_retries = self.class::MAX_RETRIES
+              https.request(request)
+            end
+
+            return unless success?
+
+            yield(s3_file_reference) if block_given?
+            s3_file_reference
+          end
+
+          def failed?
+            return false unless response
+
+            response.code != EXPECTED_CODE
+          end
+
+          def success?
+            return false unless response
+
+            response.code == EXPECTED_CODE
+          end
+
+          def s3_file_reference
+            return unless success?
+
+            @s3_file_reference ||= {
+              'filename' => file_basename,
+              'filetype' => mime_type,
+              'filesize' => file_size,
+              'path'     => s3_file_path
+            }
+          end
+
+          private
+
+          attr_reader :credentials
+          attr_reader :response
+
+          def request
+            Net::HTTP::Post.new(uri).tap do |req|
+              req.set_form form_data, 'multipart/form-data'
+            end
+          end
+
+          def uri
+            @uri ||= URI(credentials[:s3_endpoint])
+          end
+
+          def form_data
+            [
+              ['key', s3_file_path],
+              *aws_params,
+              *file_params
+            ]
+          end
+
+          def s3_file_path
+            "#{user_tmpdir}/#{file_basename}"
+          end
+
+          def user_tmpdir
+            credentials[:user_tmpdir]
+          end
+
+          def aws_params
+            params = %i[AWSAccessKeyId policy signature x-amz-server-side-encryption]
+            params.map do |key|
+              [key.to_s, credentials[key]]
+            end
+          end
+
+          def file_params
+            [
+              ['content-type', mime_type],
+              ['file',         io_stream]
+            ]
+          end
+
+          def mime_type
+            @mime_type ||= lib_mime_type || DEFAULT_MIME_TYPE
+          end
+
+          def lib_mime_type
+            MIME::Types.type_for(file_extension).first&.content_type
+          end
+
+          def io_stream
+            require_file!
+            File.open(file_fullname)
+          end
+
+          def credentials_class
+            Ecoportal::API::V2::S3::Data
+          end
+
+          def file_size
+            @file_size ||= File.size(file_fullname)
+          end
+
+          def file_extension
+            @file_extension ||= File.extname(filename)[1..]
+          end
+
+          def file_basename
+            @file_basename ||= File.basename(filename)
+          end
+
+          def file_fullname
+            @file_fullname ||= File.expand_path(filename)
+          end
+
+          def require_file!
+            msg = "File '#{file_basename}' doesn't exist"
+            raise MissingFile, msg unless File.exist?(file_fullname)
+          end
+
+          def require_deps!
+            require 'net/http'
+            require 'mime/types'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v2/s3.rb

@@ -0,0 +1,66 @@
+module Ecoportal
+  module API
+    class V2
+      # @attr_reader client [Common::Client] a `Common::Client` object that holds the configuration of the api connection.
+      class S3
+        class MissingLocalFile < ArgumentError; end
+        class CredentialsGetFailed < StandardError; end
+
+        extend Common::BaseClass
+
+        class_resolver :data_class, "Ecoportal::API::V2::S3::Data"
+        class_resolver :files_class, "Ecoportal::API::V2::S3::Files"
+
+        attr_reader :client
+
+        # @param client [Common::Client] a `Common::Client` object that holds the configuration of the api connection.
+        # @return [Schemas] an instance object ready to make schema api requests.
+        def initialize(client)
+          @client = client
+        end
+
+        # Gets the S3 credentials to upload files
+        # @return [Ecoportal::API::V2::S3::Data]
+        def data
+          response = client.get("/s3/data")
+          body     = body_data(response.body)
+
+          return data_class.new(body) if response.success?
+
+          msg = "Could not get S3 credentials - Error #{response.status}: #{body}"
+          raise CredentialsGetFailed, msg
+        end
+
+        # @param file [String] full path to existing local file
+        # @param credentials [Ecoportal::API::V2::S3::Data, NilClass]
+        # @return [Hash, NilClass] with the s3_file_reference on success, and `nil` otherwise
+        def upload_file(file, credentials: data, &block)
+          msg = "The file '#{file}' does not exist"
+          raise MissingLocalFile, msg unless File.exist?(file)
+
+          credentials ||= data
+          Upload.new(credentials, file: file).upload!(&block)
+        end
+
+        # Obtain specific object for eP file api requests.
+        # @return [Files] an instance object ready to make eP file api requests.
+        def files
+          files_class.new(client, s3_api: self)
+        end
+
+        private
+
+        def body_data(body)
+          return body unless body.is_a?(Hash)
+          return body unless body.key?("data")
+
+          body["data"]
+        end
+      end
+    end
+  end
+end
+
+require 'ecoportal/api/v2/s3/data'
+require 'ecoportal/api/v2/s3/upload'
+require 'ecoportal/api/v2/s3/files'

data/lib/ecoportal/api/v2.rb

@@ -8,10 +8,10 @@ module Ecoportal
       extend Common::BaseClass
       include Common::Logging
 
-      VERSION = "v2"
+      VERSION = "v2".freeze
 
       class << self
-        def v2key (ukey, gkey)
+        def v2key(ukey, gkey)
           Base64.urlsafe_encode64({
             organization: gkey,
             user:         ukey
@@ -22,6 +22,7 @@ module Ecoportal
       class_resolver :people_class,    "Ecoportal::API::V2::People"
       class_resolver :registers_class, "Ecoportal::API::V2::Registers"
       class_resolver :pages_class,     "Ecoportal::API::V2::Pages"
+      class_resolver :s3_class,        "Ecoportal::API::V2::S3"
 
       attr_reader :client, :logger
 
@@ -64,6 +65,12 @@ module Ecoportal
         pages_class.new(client)
       end
 
+      # Obtain specific object for file api requests.
+      # @return [S3] an instance object ready to make files api requests.
+      def s3 # rubocop:disable Naming/VariableNumber
+        s3_class.new(client)
+      end
+
       private
 
       def get_key(api_key: nil, user_key: nil, org_key: nil)
@@ -72,7 +79,6 @@ module Ecoportal
         raise "You need to provide either an api_key or user_key" unless user_key
         raise "You need to provide an org_key as well (not just a user_key)" unless org_key
       end
-
     end
   end
 end
@@ -80,3 +86,4 @@ end
 require 'ecoportal/api/v2/people'
 require 'ecoportal/api/v2/pages'
 require 'ecoportal/api/v2/registers'
+require 'ecoportal/api/v2/s3'

data/lib/ecoportal/api/v2_version.rb

@@ -1,5 +1,5 @@
 module Ecoportal
   module API
-    GEM2_VERSION = "1.1.8".freeze
+    GEM2_VERSION = '2.0.5'.freeze
   end
 end