s3-light 1.0.0

data/lib/s3-light/concurrent_result.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module S3Light
+  class ConcurrentResult
+    def initialize
+      @hash = Concurrent::Hash.new
+    end
+
+    def add(key, value)
+      @hash[key] = value
+    end
+
+    def to_h
+      @hash
+    end
+
+    def inspect
+      "#<#{self.class.name} @hash=#{@hash}>"
+    end
+  end
+end

data/lib/s3-light/configuration.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module S3Light
+  class Configuration
+    attr_accessor :io_buffer_size
+
+    def initialize
+      @io_buffer_size = 16 * (1024 * 1024)
+    end
+  end
+end

data/lib/s3-light/connection/body.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module S3Light
+  class Connection
+    class Body
+      def initialize(body)
+        @body = body
+        validate_body_type!
+        @size = calculate_size
+        @io = to_io
+      end
+
+      attr_reader :size
+
+      def read(length = nil, outbuf = nil)
+        @io.read(length, outbuf)
+      end
+
+      def rewind
+        @io.rewind
+      end
+
+      private
+        def to_io
+          case @body
+          when String
+            StringIO.new(@body)
+          when StringIO, IO
+            @body
+          when Enumerable
+            StringIO.new(@body.to_a.join)
+          when nil
+            StringIO.new('')
+          else
+            raise RequestError, "body of wrong type: #{@body.class}"
+          end
+        end
+
+        def calculate_size
+          case @body
+          when String
+            @body.bytesize
+          when StringIO, IO
+            @body.size
+          when Enumerable
+            @body.to_a.join.bytesize
+          when nil
+            0
+          else
+            raise RequestError, 'cannot determine size of body'
+          end
+        end
+
+        def validate_body_type!
+          return if @body.is_a?(String)
+          return if @body.respond_to?(:read)
+          return if @body.is_a?(Enumerable)
+          return if @body.nil?
+
+          raise RequestError, "body of wrong type: #{@body.class}"
+        end
+    end
+  end
+end

data/lib/s3-light/connection/response.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module S3Light
+  class Connection
+    class Response
+      def initialize(http_response)
+        @http_response = http_response
+      end
+
+      def xml
+        return @xml if defined? @xml
+
+        @xml = Nokogiri::XML(@http_response.body.to_s)
+      end
+
+      def code
+        @http_response.code
+      end
+
+      def body
+        @http_response.body
+      end
+    end
+  end
+end

data/lib/s3-light/connection.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+module S3Light
+  class Connection
+    class HttpError < StandardError
+      attr_reader :response
+
+      def initialize(response)
+        @response = response
+        super("HTTP Error: #{response.code} - #{response.body}")
+      end
+
+      def code
+        @response.code
+      end
+    end
+    attr_reader :endpoint
+
+    def initialize(endpoint:, access_key_id:, secret_access_key:, ssl_context:)
+      @endpoint = URI(endpoint)
+      @access_key_id = access_key_id
+      @secret_access_key = secret_access_key
+      @ssl_context = ssl_context
+      @opened = false
+    end
+
+    def persistent_connection
+      return @persistent_connection if @persistent_connection
+
+      @persistent_connection = HTTP.persistent(@endpoint).headers(
+        'User-Agent' => "S3Light/#{S3Light::VERSION}",
+        'Host' => @endpoint.host
+      )
+      ObjectSpace.define_finalizer(self, self.class.close_connection(@persistent_connection))
+
+      @persistent_connection
+    end
+
+    def make_request(method, path, headers: {}, body: nil)
+      @opened = true
+      full_path = URI.join(@endpoint, path).to_s
+      request_time = Time.now.utc
+      body = body.is_a?(Body) ? body : Body.new(body)
+
+      headers = build_headers(method, full_path, headers, body, request_time)
+
+      response = stream_request(method, full_path, headers, body)
+
+      handle_response(response)
+    end
+
+    def download_file(path, output_path)
+      @opened = true
+      full_path = URI.join(@endpoint, path).to_s
+      request_time = Time.now.utc
+
+      headers = build_headers('GET', full_path, {}, Body.new(nil), request_time)
+
+      File.open(output_path, 'wb') do |file|
+        response = persistent_connection.headers(headers).get(full_path, ssl_context: @ssl_context)
+        raise HttpError.new(response) if response.code > 399
+
+        response.body.each do |chunk|
+          file.write(chunk)
+        end
+      end
+
+      output_path
+    end
+
+    def close
+      @persistent_connection&.close
+      @opened = false
+    end
+
+    def ==(other_connection)
+      @endpoint == other_connection.endpoint
+    end
+
+    def inspect
+      "#<#{self.class.name} @endpoint=#{@endpoint} @opened=#{@opened}>"
+    end
+
+    def self.close_connection(connection)
+      proc do
+        connection.close
+      end
+    end
+
+    private
+      def stream_request(method, full_path, headers, body)
+        if body.size > 0
+          headers['Content-Length'] = body.size.to_s
+        end
+
+        persistent_connection.headers(headers).request(
+          method,
+          full_path,
+          ssl_context: @ssl_context,
+          body: body
+        )
+      end
+
+      def build_headers(method, path, headers, body, request_time)
+        canonical_headers = {
+          'host' => @endpoint.host,
+          'x-amz-date' => request_time.strftime('%Y%m%dT%H%M%SZ'),
+          'x-amz-content-sha256' => 'UNSIGNED-PAYLOAD'
+        }
+
+        canonical_headers['content-length'] = body.size.to_s if body.size > 0
+
+        signed_headers = canonical_headers.keys.sort.join(';')
+
+        auth_header = authorization_header(
+          method,
+          path,
+          canonical_headers,
+          signed_headers,
+          body,
+          request_time
+        )
+
+        headers.merge(canonical_headers).merge('Authorization' => auth_header)
+      end
+
+      def authorization_header(method, path, canonical_headers, signed_headers, body, request_time)
+        algorithm = 'AWS4-HMAC-SHA256'
+        credential_scope = credential_scope(request_time)
+        string_to_sign = string_to_sign(method, path, canonical_headers, signed_headers, body, request_time)
+        signature = signature(string_to_sign, request_time)
+
+        "#{algorithm} Credential=#{@access_key_id}/#{credential_scope}, SignedHeaders=#{signed_headers}, Signature=#{signature}"
+      end
+
+      def credential_scope(request_time)
+        date = request_time.strftime('%Y%m%d')
+        region = @endpoint.host.split('.')[1] || 'us-east-1'
+        "#{date}/#{region}/s3/aws4_request"
+      end
+
+      def string_to_sign(method, path, canonical_headers, signed_headers, body, request_time)
+        uri = URI(path)
+        canonical_request = [
+          method.to_s.upcase,
+          uri.path,
+          uri.query || '',
+          canonical_headers.sort.map { |k, v| "#{k}:#{v.strip}" }.join("\n") + "\n",
+          signed_headers,
+          'UNSIGNED-PAYLOAD'
+        ].join("\n")
+
+        [
+          'AWS4-HMAC-SHA256',
+          request_time.strftime('%Y%m%dT%H%M%SZ'),
+          credential_scope(request_time),
+          Digest::SHA256.hexdigest(canonical_request)
+        ].join("\n")
+      end
+
+      def signature(string_to_sign, request_time)
+        date = request_time.strftime('%Y%m%d')
+        region = @endpoint.host.split('.')[1] || 'us-east-1'
+        service = 's3'
+
+        k_date = hmac("AWS4#{@secret_access_key}", date)
+        k_region = hmac(k_date, region)
+        k_service = hmac(k_region, service)
+        k_signing = hmac(k_service, 'aws4_request')
+
+        OpenSSL::HMAC.hexdigest('sha256', k_signing, string_to_sign)
+      end
+
+      def hmac(key, value)
+        OpenSSL::HMAC.digest('sha256', key, value)
+      end
+
+      def handle_response(response)
+        case response.code
+        when 200..299
+          Connection::Response.new(response)
+        else
+          raise HttpError.new(response)
+        end
+      end
+
+      def chunked?(headers)
+        headers['Transfer-Encoding'] == 'chunked'
+      end
+  end
+end

data/lib/s3-light/connections_manager/main.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module S3Light
+  module ConnectionsManager
+    class Main
+      def initialize
+        @connections = {}
+      end
+
+      def yield_connection(client, &block)
+        client_connection = nil
+        @connections.each do |existing_client, connection|
+          if client.endpoint == existing_client.endpoint
+            client_connection = connection
+            break
+          end
+        end
+
+        client_connection ||= create_new_connection(client)
+        @connections[client] = client_connection
+
+        block.call(client_connection)
+      end
+
+      def close_all_connections
+        return if @connections.empty?
+        @connections.each do |_, connection|
+          connection.close
+        end
+
+        @connections = {}
+      end
+
+      private
+        def create_new_connection(client)
+          Connection.new(
+            endpoint: client.endpoint, access_key_id: client.access_key_id, secret_access_key: client.secret_access_key,
+            ssl_context: client.ssl_context
+          )
+        end
+    end
+  end
+end

data/lib/s3-light/connections_manager/threaded.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module S3Light
+  module ConnectionsManager
+    class Threaded
+      def initialize(client, concurrency)
+        @client = client
+        @thread_pool = Concurrent::FixedThreadPool.new(concurrency)
+        @connections = Concurrent::Array.new
+      end
+
+      def with_connection
+        @thread_pool.post do
+          existing_connection = Thread.current[:connection]
+          yield existing_connection if existing_connection
+
+          new_connection = create_new_connection
+          @connections << new_connection
+          Thread.current[:connection] = new_connection
+
+          yield new_connection
+        end
+      end
+
+      def close
+        @connections.each(&:close)
+        @connections = Concurrent::Array.new
+      end
+
+      def wait_to_finish
+        @thread_pool.shutdown
+        @thread_pool.wait_for_termination
+      end
+
+      private
+        def create_new_connection
+          Connection.new(
+            endpoint: @client.endpoint, access_key_id: @client.access_key_id, secret_access_key: @client.secret_access_key,
+            ssl_context: @client.ssl_context
+          )
+        end
+    end
+  end
+end

data/lib/s3-light/md5_calculator.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module S3Light
+  class Md5Calculator
+    def initialize(input)
+      @input = input
+    end
+
+    def md5
+      @md5 ||=
+        case @input
+        when String
+          Digest::MD5.hexdigest(@input)
+        when IO, StringIO
+          compute_md5_for_io
+        when Pathname
+          compute_md5_for_file_path
+        else
+          raise ArgumentError, "Unsupported input type: #{@input.class}"
+        end
+    end
+
+    def inspect
+      "#<#{self.class.name}>"
+    end
+
+    private
+      def compute_md5_for_io
+        Digest::MD5.new.tap do |md5|
+          io_buffer_size = S3Light.configuration.io_buffer_size
+          while chunk = @input.read(io_buffer_size)
+            md5 << chunk
+          end
+          @input.rewind
+        end.hexdigest
+      end
+
+      def compute_md5_for_file_path
+        Digest::MD5.new.tap do |md5|
+          io_buffer_size = S3Light.configuration.io_buffer_size
+          File.open(@input, 'rb') do |file|
+            while chunk = file.read(io_buffer_size)
+              md5 << chunk
+            end
+          end
+        end.hexdigest
+      end
+  end
+end

data/lib/s3-light/object.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module S3Light
+  Object = Struct.new(:client, :bucket, :key, :input, :persisted) do
+    def initialize(client, bucket, id, input, persisted = false)
+      super(client, bucket, id, input, persisted)
+    end
+
+    def persisted?
+      persisted
+    end
+
+    def inspect
+      "#<#{self.class.name} @id=#{key} @bucket=#{bucket.name}>"
+    end
+
+    def save!
+      raise S3Light::Error, 'Input is empty' if input.nil?
+
+      self.client.with_connection do |connection|
+        __save!(connection)
+      end
+      self.persisted = true
+
+      self
+    end
+
+    def destroy!
+      unless persisted
+        raise S3Light::Error, 'Object does not exist'
+      end
+
+      self.client.with_connection do |connection|
+        __destroy!(connection)
+      rescue S3Light::Connection::HttpError => e
+        raise e unless e.code == 404
+      end
+
+      self.persisted = false
+      true
+    end
+
+    def acl
+      return @acl if defined?(@acl)
+
+      response = client.with_connection do |connection|
+        connection.make_request(:get, "/#{bucket.name}/#{key}?acl=1")
+      end
+
+      @acl = response.xml.remove_namespaces!.xpath('//AccessControlList/Grant').each_with_object({}) do |grant, result|
+        permission = grant.at_xpath('Permission').text
+        grantee = grant.at_xpath('Grantee')
+
+        grantee_type = grantee['type']
+
+        identifier =
+          case grantee_type
+          when 'CanonicalUser'
+            grantee.at_xpath('ID')&.text || 'CanonicalUser'
+          when 'Group'
+            grantee.at_xpath('URI')&.text
+          else
+            grantee.at_xpath('EmailAddress')&.text
+          end
+
+        result[identifier] = permission if identifier
+      end
+    end
+
+
+    def acl=(new_acl)
+      xml_body = build_acl_xml(new_acl)
+
+      client.with_connection do |connection|
+        connection.make_request(:put, "/#{bucket.name}/#{key}?acl=1", body: xml_body)
+      end
+
+      @acl = new_acl
+    end
+
+    def download(to: nil)
+      raise S3Light::Error, 'Object does not exist' unless persisted
+
+      to ||= "/tmp/#{SecureRandom.hex}-#{key}"
+
+      self.client.with_connection do |connection|
+        __download(to, connection)
+      end
+    end
+
+    def open(mode = 'r', &block)
+      raise S3Light::Error, 'Object does not exist' unless persisted
+      download_path = download
+      file = File.open(download_path, mode)
+      yield file
+      self.input = file
+      file
+    end
+
+    def __save!(connection)
+      connection.make_request(:put, "/#{bucket.name}/#{key}", body: input)
+    end
+
+    def __destroy!(connection)
+      connection.make_request(:delete, "/#{bucket.name}/#{key}")
+    end
+
+    def __download(to, connection)
+      connection.download_file("/#{bucket.name}/#{key}", to)
+    end
+
+    private
+      def build_acl_xml(acl)
+        builder = Nokogiri::XML::Builder.new do |xml|
+          xml.AccessControlPolicy {
+            xml.AccessControlList {
+              acl.each do |grantee, permission|
+                xml.Grant {
+                  xml.Grantee('xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance', 'xsi:type' => 'CanonicalUser') {
+                    xml.ID grantee
+                    xml.DisplayName grantee
+                  }
+                  xml.Permission permission
+                }
+              end
+            }
+          }
+        end
+
+        builder.to_xml
+      end
+  end
+end

data/lib/s3-light/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module S3Light
+  VERSION = '1.0.0'
+end

data/lib/s3-light.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'http'
+require 'pathname'
+require 'stringio'
+require 'digest'
+require 'base64'
+require 'time'
+require 'openssl/hmac'
+require 'nokogiri'
+require 'concurrent-ruby'
+
+Dir.glob(File.join(File.dirname(__FILE__), 's3-light', '**', '*.rb')).each do |file|
+  require file
+end
+
+module S3Light
+  class Error < StandardError; end
+  class << self
+    def self.configure
+      yield(configuration)
+    end
+
+    def configuration
+      @configuration ||= Configuration.new
+    end
+  end
+end

data/sig/s3/light.rbs

@@ -0,0 +1,6 @@
+module S3
+  module Light
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end