s3-rb 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8da7974f1a862a8b59312aa1b2a14e62acb50e0c395a59cb0e9163c81ca6c1d9
+  data.tar.gz: 271348a115ab248f843912064ce39ec87466165706304cbbfda8919f47e3d00b
+SHA512:
+  metadata.gz: 89d5da5bdae4ba11bf4fd3de8145d2f992a012b4d28f2cba73c9752af4d220f788fcefb8e45fe73aea16eb31ee3d6286fd989f83ba1768cb60e4950a9063ee8c
+  data.tar.gz: a57ec119c7b94cc518a10869e95f58937df0ef553cf2d7e0aafab14341980c96fcb54e5bcf0885566b0d3cb268ecd24b9cfc9cbb793147311177ca0e4a1047b5

data/Gemfile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec
+
+group :development, :test do
+  gem 'debug', '~> 1.0'
+end

data/Gemfile.lock

@@ -0,0 +1,132 @@
+PATH
+  remote: .
+  specs:
+    s3-rb (0.1.0)
+      dynamicschema (~> 2.0)
+      faraday (~> 2.0)
+      faraday-net_http_persistent (~> 2.0)
+      nokogiri (~> 1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.8)
+      public_suffix (>= 2.0.2, < 8.0)
+    aws-eventstream (1.4.0)
+    aws-partitions (1.1211.0)
+    aws-sdk-core (3.241.4)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.992.0)
+      aws-sigv4 (~> 1.9)
+      base64
+      bigdecimal
+      jmespath (~> 1, >= 1.6.1)
+      logger
+    aws-sdk-kms (1.121.0)
+      aws-sdk-core (~> 3, >= 3.241.4)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-s3 (1.213.0)
+      aws-sdk-core (~> 3, >= 3.241.4)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.5)
+    aws-sigv4 (1.12.1)
+      aws-eventstream (~> 1, >= 1.0.2)
+    base64 (0.3.0)
+    benchmark-ips (2.14.0)
+    bigdecimal (4.0.1)
+    connection_pool (3.0.2)
+    crack (1.0.1)
+      bigdecimal
+      rexml
+    date (3.5.1)
+    debug (1.11.1)
+      irb (~> 1.10)
+      reline (>= 0.3.8)
+    dynamicschema (2.2.0)
+    erb (6.0.1)
+    faraday (2.14.0)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-net_http (3.4.2)
+      net-http (~> 0.5)
+    faraday-net_http_persistent (2.3.1)
+      faraday (~> 2.5)
+      net-http-persistent (>= 4.0.4, < 5)
+    hashdiff (1.2.1)
+    io-console (0.8.2)
+    irb (1.16.0)
+      pp (>= 0.6.0)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
+    jmespath (1.6.2)
+    json (2.18.0)
+    logger (1.7.0)
+    minitest (5.27.0)
+    net-http (0.9.1)
+      uri (>= 0.11.1)
+    net-http-persistent (4.0.8)
+      connection_pool (>= 2.2.4, < 4)
+    nokogiri (1.19.0-aarch64-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.19.0-aarch64-linux-musl)
+      racc (~> 1.4)
+    nokogiri (1.19.0-arm-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.19.0-arm-linux-musl)
+      racc (~> 1.4)
+    nokogiri (1.19.0-arm64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.19.0-x86_64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.19.0-x86_64-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.19.0-x86_64-linux-musl)
+      racc (~> 1.4)
+    pp (0.6.3)
+      prettyprint
+    prettyprint (0.2.0)
+    psych (5.3.1)
+      date
+      stringio
+    public_suffix (7.0.2)
+    racc (1.8.1)
+    rake (13.3.1)
+    rdoc (7.1.0)
+      erb
+      psych (>= 4.0.0)
+      tsort
+    reline (0.6.3)
+      io-console (~> 0.5)
+    rexml (3.4.4)
+    stringio (3.2.0)
+    tsort (0.2.0)
+    uri (1.1.1)
+    vcr (6.4.0)
+    webmock (3.26.1)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  aarch64-linux-gnu
+  aarch64-linux-musl
+  arm-linux-gnu
+  arm-linux-musl
+  arm64-darwin
+  x86_64-darwin
+  x86_64-linux-gnu
+  x86_64-linux-musl
+
+DEPENDENCIES
+  aws-sdk-s3 (~> 1.0)
+  benchmark-ips (~> 2.0)
+  debug (~> 1.0)
+  minitest (~> 5.0)
+  rake (~> 13.0)
+  s3-rb!
+  vcr (~> 6.0)
+  webmock (~> 3.0)
+
+BUNDLED WITH
+   2.7.1

data/README.md

@@ -0,0 +1,215 @@
+# Yet another S3 gem
+
+This is a lightweight, low dependency, high performance, high compatibility, 'bare metal' S3 API gem. I created it primarily because I wanted to avoid including Amazon's AWS API gem and all its various dependencies. The current implementation depends on Faraday, the Faraday 'net-http-persistent' adapter, Nokogiri, and the DynamicSchema gem. Although other S3 gems exist for Ruby I found that most of these have had no updates for many years and lacked compatibility with various alternate S3 providers.
+
+I chose to make this implementation 'bare metal' in that there is minimal abstraction layer atop the S3 API - not even an iterator for the object list. You are then free to build your own abstraction best suited to the semantics of your application or gem.
+
+At its most basic the implementation provides a request class per operation which returns a result structure. In some cases, for operations with many parameters, there is an options structure you can build. The operation is executed by 'submitting' an instance of the request.
+
+```ruby
+require 's3'
+
+# build the options for the put operation
+options = S3::ObjectPutOptions.build( content_type: 'text/plain',
+                                      acl: :public_read,
+                                      storage_class: :standard_ia )
+
+# create the request and submit it
+request = S3::ObjectPutRequest.new( access_key_id: 'AKIA...',
+                                    secret_access_key: '...',
+                                    region: 'us-east-1' )
+
+response = request.submit( bucket: 'my-bucket',
+                           key: 'hello.txt',
+                           body: 'Hello, World!',
+                           options: options )
+
+# check for success and read the result
+if response.success?
+  result = response.result
+  puts result.etag
+else
+  result = response.result
+  puts "error: #{ result.error_code } - #{ result.error_description }"
+end
+```
+
+The operation methods - similar to those in the AWS gem - then create a matching request object with the given arguments and submit a request, typically returning the same result structure.
+
+```ruby
+require 's3'
+
+# create the service
+s3 = S3::Service.new( access_key_id: 'AKIA...',
+                      secret_access_key: '...',
+                      region: 'us-east-1' )
+
+# call the method directly; it raises an exception on error
+result = s3.object_put( bucket: 'my-bucket',
+                        key: 'hello.txt',
+                        body: 'Hello, World!',
+                        content_type: 'text/plain',
+                        acl: :public_read,
+                        storage_class: :standard_ia )
+
+puts result.etag
+```
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem 's3-rb'
+```
+
+Or install directly:
+
+```bash
+gem install s3-rb
+```
+
+## Quick Start
+
+```ruby
+require 's3'
+
+# create the service for AWS
+s3 = S3::Service.new( access_key_id: 'AKIA...',
+                      secret_access_key: '...',
+                      region: 'us-east-1' )
+
+# create a service for S3-compatible providers by specifying the endpoint
+s3 = S3::Service.new( access_key_id: '...',
+                      secret_access_key: '...',
+                      endpoint: 'https://s3.us-east-005.dream.io',
+                      region: 'us-east-005' )
+
+# enable connection pooling for better performance
+s3 = S3::Service.new( access_key_id: '...',
+                      secret_access_key: '...',
+                      region: 'us-east-1',
+                      connection_pool: 5 )
+
+# upload an object
+s3.object_put( bucket: 'my-bucket',
+               key: 'hello.txt',
+               body: 'Hello, World!',
+               content_type: 'text/plain' )
+
+# download an object
+content = s3.object_get( bucket: 'my-bucket', key: 'hello.txt' )
+
+# list objects
+result = s3.object_list( bucket: 'my-bucket', prefix: 'documents/' )
+result.each { | obj | puts "#{ obj.key } - #{ obj.size } bytes" }
+
+# handle pagination explicitly
+result = s3.object_list( bucket: 'my-bucket', max_keys: 100 )
+while result.truncated?
+  result = s3.object_list( bucket: 'my-bucket',
+                           max_keys: 100,
+                           continuation_token: result.next_continuation_token )
+  result.each { | obj | process( obj ) }
+end
+```
+
+## Error Handling
+
+All S3 errors are raised as exceptions:
+
+```ruby
+begin
+  s3.object_get( bucket: 'my-bucket', key: 'missing.txt' )
+rescue S3::NoSuchKeyError => e
+  puts "object not found: #{ e.message }"
+rescue S3::AccessDeniedError => e
+  puts "access denied: #{ e.message }"
+rescue S3::Error => e
+  puts "s3 error: #{ e.code } - #{ e.message }"
+end
+```
+
+## Streaming
+
+For large files, use streaming to avoid loading everything into memory:
+
+```ruby
+# streaming upload from a file
+File.open( 'large-file.zip', 'rb' ) do | file |
+  s3.object_put( bucket: 'my-bucket',
+                 key: 'large-file.zip',
+                 body: file,
+                 content_type: 'application/zip' )
+end
+
+# streaming download to a file
+File.open( 'download.zip', 'wb' ) do | file |
+  s3.object_get( bucket: 'my-bucket', key: 'large-file.zip' ) do | chunk |
+    file.write( chunk )
+  end
+end
+```
+
+## Storage Classes and ACLs
+
+Pass symbols or lowercase strings - they're automatically normalized:
+
+```ruby
+# these are equivalent
+s3.object_put( bucket: 'b', key: 'k', body: 'x', storage_class: :standard_ia )
+s3.object_put( bucket: 'b', key: 'k', body: 'x', storage_class: 'standard_ia' )
+s3.object_put( bucket: 'b', key: 'k', body: 'x', storage_class: 'STANDARD_IA' )
+
+# same for ACLs
+s3.object_put( bucket: 'b', key: 'k', body: 'x', acl: :public_read )
+s3.object_put( bucket: 'b', key: 'k', body: 'x', acl: 'public-read' )
+```
+
+## API Reference
+
+### Bucket Operations
+
+| Method | Description |
+|--------|-------------|
+| `bucket_list` | List all buckets |
+| `bucket_create( bucket:, region:, acl: )` | Create a bucket |
+| `bucket_delete( bucket: )` | Delete a bucket |
+| `bucket_head( bucket: )` | Check bucket existence, get region |
+| `bucket_exists?( bucket: )` | Returns true/false |
+
+### Object Operations
+
+| Method | Description |
+|--------|-------------|
+| `object_list( bucket:, prefix:, ... )` | List objects |
+| `object_get( bucket:, key:, &block )` | Download object |
+| `object_put( bucket:, key:, body:, ... )` | Upload object |
+| `object_delete( bucket:, key: )` | Delete object |
+| `object_delete_batch( bucket:, keys: )` | Delete multiple objects |
+| `object_head( bucket:, key: )` | Get object metadata |
+| `object_exists?( bucket:, key: )` | Returns true/false |
+| `object_copy( source_bucket:, source_key:, bucket:, key:, ... )` | Copy object |
+| `object_metadata_set( bucket:, key:, metadata: )` | Update metadata |
+
+### Multipart Operations
+
+| Method | Description |
+|--------|-------------|
+| `multipart_create( bucket:, key:, ... )` | Initiate multipart upload |
+| `multipart_upload( bucket:, key:, upload_id:, part_number:, body: )` | Upload a part |
+| `multipart_complete( bucket:, key:, upload_id:, parts: )` | Complete upload |
+| `multipart_abort( bucket:, key:, upload_id: )` | Abort upload |
+| `multipart_list( bucket:, prefix: )` | List in-progress uploads |
+| `multipart_parts( bucket:, key:, upload_id: )` | List uploaded parts |
+
+### Presigned URLs
+
+| Method | Description |
+|--------|-------------|
+| `presign_get( bucket:, key:, expires_in: )` | Generate download URL |
+| `presign_put( bucket:, key:, expires_in:, content_type: )` | Generate upload URL |
+
+## License
+
+MIT License

data/Rakefile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'rake/testtask'
+
+Rake::TestTask.new( :test ) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList[ 'test/**/*_test.rb' ]
+end
+
+task default: :test

data/lib/s3/bucket_create_request.rb

@@ -0,0 +1,26 @@
+module S3
+  class BucketCreateRequest < Request
+    def submit( bucket:, region: nil, acl: nil )
+      headers = {}
+      headers[ 'x-amz-acl' ] = acl if acl
+
+      body = nil
+      location = region || @region
+
+      if location && location != 'us-east-1'
+        body = <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+            <LocationConstraint>#{ location }</LocationConstraint>
+          </CreateBucketConfiguration>
+        XML
+      end
+
+      response = put( "/#{ bucket }", body: body, headers: headers )
+
+      build_result( response, TrueClass ) do
+        true
+      end
+    end
+  end
+end

data/lib/s3/bucket_delete_request.rb

@@ -0,0 +1,11 @@
+module S3
+  class BucketDeleteRequest < Request
+    def submit( bucket: )
+      response = delete( "/#{ bucket }" )
+
+      build_result( response, TrueClass ) do
+        true
+      end
+    end
+  end
+end

data/lib/s3/bucket_head_request.rb

@@ -0,0 +1,19 @@
+module S3
+  class BucketHeadRequest < Request
+    def submit( bucket: )
+      response = head( "/#{ bucket }" )
+
+      if response.status == 404
+        ResponseMethods.install( response, nil )
+      else
+        build_result( response, BucketHeadResult ) do
+          BucketHeadResult.new( parse_response( response ) )
+        end
+      end
+    end
+
+    def parse_response( response )
+      { region: response.headers[ 'x-amz-bucket-region' ] }
+    end
+  end
+end

data/lib/s3/bucket_head_result.rb

@@ -0,0 +1,8 @@
+module S3
+  BucketHeadResultSchema = DynamicSchema::Struct.define do
+    region              String
+  end
+
+  class BucketHeadResult < BucketHeadResultSchema
+  end
+end

data/lib/s3/bucket_list_request.rb

@@ -0,0 +1,31 @@
+module S3
+  class BucketListRequest < Request
+    def submit
+      response = get( '/' )
+
+      build_result( response, BucketListResult ) do
+        BucketListResult.new( parse_response( response.body ) )
+      end
+    end
+
+    def parse_response( body )
+      document = parse_xml( body )
+      document.remove_namespaces!
+
+      buckets = document.xpath( '//Bucket' ).map do | node |
+        { name: node.at( 'Name' )&.text,
+          creation_date: Helpers.parse_iso8601( node.at( 'CreationDate' )&.text )
+        }
+      end
+
+      owner_node = document.at( '//Owner' )
+      owner_info = if owner_node
+                     { id: owner_node.at( 'ID' )&.text,
+                       display_name: owner_node.at( 'DisplayName' )&.text
+                     }
+                   end
+
+      { buckets: buckets, owner: owner_info }
+    end
+  end
+end

data/lib/s3/bucket_list_result.rb

@@ -0,0 +1,31 @@
+require 'forwardable'
+
+module S3
+  OwnerSchema = DynamicSchema::Struct.define do
+    id                  String
+    display_name        String
+  end
+
+  class Owner < OwnerSchema
+  end
+
+  BucketEntrySchema = DynamicSchema::Struct.define do
+    name                String
+    creation_date       Time
+  end
+
+  class BucketEntry < BucketEntrySchema
+  end
+
+  BucketListResultSchema = DynamicSchema::Struct.define do
+    buckets             BucketEntry,        array: true
+    owner               Owner
+  end
+
+  class BucketListResult < BucketListResultSchema
+    extend Forwardable
+    include Enumerable
+
+    def_delegators :buckets, :each, :[], :count, :size, :length, :first, :last, :empty?
+  end
+end

data/lib/s3/bucket_methods.rb

@@ -0,0 +1,45 @@
+module S3
+  module BucketMethods
+    def bucket_list
+      request = BucketListRequest.new( **request_options )
+      response = request.submit
+
+      raise_if_error( response )
+
+      response.result
+    end
+
+    def bucket_create( bucket:, region: nil, acl: nil )
+      request = BucketCreateRequest.new( **request_options )
+      response = request.submit( bucket: bucket, region: region, acl: acl )
+
+      raise_if_error( response )
+
+      response.result
+    end
+
+    def bucket_delete( bucket: )
+      request = BucketDeleteRequest.new( **request_options )
+      response = request.submit( bucket: bucket )
+
+      raise_if_error( response )
+
+      response.result
+    end
+
+    def bucket_head( bucket: )
+      request = BucketHeadRequest.new( **request_options )
+      response = request.submit( bucket: bucket )
+
+      return nil if response.status == 404
+
+      raise_if_error( response )
+
+      response.result
+    end
+
+    def bucket_exists?( bucket: )
+      !bucket_head( bucket: bucket ).nil?
+    end
+  end
+end

data/lib/s3/error_result.rb

@@ -0,0 +1,51 @@
+module S3
+  class ErrorResult
+    attr_reader :error_type, :error_description, :error_code, :request_id, :resource
+
+    def initialize( status_code, attributes = nil )
+      @error_type, @error_description = status_code_to_error( status_code )
+
+      if attributes.is_a?( Hash )
+        @error_code = attributes[ :code ]
+        @error_description = attributes[ :message ] if attributes[ :message ]
+        @request_id = attributes[ :request_id ]
+        @resource = attributes[ :resource ]
+      end
+    end
+
+    def success?
+      false
+    end
+
+  private
+
+    def status_code_to_error( status_code )
+      case status_code
+      when 200
+        [ :unexpected_error,
+          "The response was successful but it did not include a valid payload." ]
+      when 400
+        [ :invalid_request_error,
+          "There was an issue with the format or content of your request." ]
+      when 401, 403
+        [ :authentication_error,
+          "There's an issue with your credentials or permissions." ]
+      when 404
+        [ :not_found_error,
+          "The requested resource was not found." ]
+      when 409
+        [ :conflict_error,
+          "There was a conflict with the current state of the resource." ]
+      when 429
+        [ :rate_limit_error,
+          "Your account has hit a rate limit." ]
+      when 500..599
+        [ :server_error,
+          "The S3 service encountered an unexpected server error." ]
+      else
+        [ :unknown_error,
+          "The S3 service returned an unexpected status code: '#{ status_code }'." ]
+      end
+    end
+  end
+end

data/lib/s3/errors.rb

@@ -0,0 +1,71 @@
+module S3
+  class Error < StandardError
+    attr_reader :code, :request_id, :resource
+
+    def initialize( message = nil, code: nil, request_id: nil, resource: nil )
+      @code = code
+      @request_id = request_id
+      @resource = resource
+      super( message || code )
+    end
+  end
+
+  # Authentication & Authorization
+  class AuthenticationError < Error; end
+  class AccessDeniedError < Error; end
+
+  # Bucket
+  class BucketNotFoundError < Error; end
+  class BucketAlreadyExistsError < Error; end
+  class BucketNotEmptyError < Error; end
+  class InvalidBucketNameError < Error; end
+
+  # Object
+  class NoSuchKeyError < Error; end
+  class EntityTooLargeError < Error; end
+  class EntityTooSmallError < Error; end
+
+  # Multipart
+  class NoSuchUploadError < Error; end
+  class InvalidPartError < Error; end
+  class InvalidPartOrderError < Error; end
+
+  # Request
+  class InvalidRequestError < Error; end
+
+  # Service
+  class ServiceUnavailableError < Error; end
+  class InternalError < Error; end
+
+  # Network
+  class NetworkError < Error; end
+  class TimeoutError < Error; end
+
+  ERROR_CODE_MAP = {
+    'InvalidAccessKeyId'              => AuthenticationError,
+    'SignatureDoesNotMatch'           => AuthenticationError,
+    'AccessDenied'                    => AccessDeniedError,
+    'NoSuchBucket'                    => BucketNotFoundError,
+    'BucketAlreadyExists'             => BucketAlreadyExistsError,
+    'BucketAlreadyOwnedByYou'         => BucketAlreadyExistsError,
+    'BucketNotEmpty'                  => BucketNotEmptyError,
+    'InvalidBucketName'               => InvalidBucketNameError,
+    'NoSuchKey'                       => NoSuchKeyError,
+    'EntityTooLarge'                  => EntityTooLargeError,
+    'EntityTooSmall'                  => EntityTooSmallError,
+    'NoSuchUpload'                    => NoSuchUploadError,
+    'InvalidPart'                     => InvalidPartError,
+    'InvalidPartOrder'                => InvalidPartOrderError,
+    'MalformedXML'                    => InvalidRequestError,
+    'InvalidArgument'                 => InvalidRequestError,
+    'ServiceUnavailable'              => ServiceUnavailableError,
+    'SlowDown'                        => ServiceUnavailableError,
+    'InternalError'                   => InternalError,
+    'RequestTimeout'                  => TimeoutError
+  }.freeze
+
+  def self.build_error( code:, message:, request_id: nil, resource: nil )
+    error_class = ERROR_CODE_MAP[ code ] || Error
+    error_class.new( message, code: code, request_id: request_id, resource: resource )
+  end
+end