faster_s3_url 1.0.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f3c703ea5ea25203b0d37da4ba0272c9619d7d46bbe0eed9bd7a3fd79768473
-  data.tar.gz: a2060cdee17b8731274a4ea997859df1390cf09f4dae2530d2145e3345ffabaf
+  metadata.gz: 21d5f75ad6e25913b8540b317745ccff1dacf631685963083a5fe498bb04195f
+  data.tar.gz: 888224050710da2bf891268a0654c0bd15e89fde0d6f7d58aae2e2f3d0a3d2ca
 SHA512:
-  metadata.gz: e8867a60091b046eb1956589e14666b10bee32711bbadfed2fcfab9e25dde2459e60baf0d61fa2564e73d838a55803ffef11a5e21452d9352b994d0af9749efe
-  data.tar.gz: bb29010a02d2e89773ecf43a8b1c041f81d855f0bd80cbf3251fa00bd5eeb557b73ae51fb53a12a421296276786beed38246a6920b96639628e3dcb8b433c42c
+  metadata.gz: 65bd748773211c8cd720502fd008de4dba4f1a08d81495987f702ae7e7a2a128288cb15d40df149b0166e4d7a0ec6fd94bf8aa198398b834cd8fb423829be38e
+  data.tar.gz: 31bb93d34192d73b787d36ff87c0223b0de430fa1fe5c712d3aa6a65d67c6bfbcccf6a23bbc45139a8e871889900ae4515d38c24028f288ff4dbdc68556aadcf

data/.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: ['**']
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.2.4
+
+    - name: Bundle install
+      run: bundle install --jobs 4 --retry 3
+
+    - name: Run tests
+      run: bundle exec rake

data/CHANGELOG.md

@@ -0,0 +1,31 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## 1.2.0
+
+### Changed
+
+- Now requires at least ruby 3.1
+- uses CGI.escapeURIComponent for somewhat improved performance https://github.com/jrochkind/faster_s3_url/pull/8
+
+### Added
+
+- Add `session_token` option to `Builder` from [@BenKanouse](https://github.com/BenKanouse) https://github.com/jrochkind/faster_s3_url/pull/12
+
+- Add 'endpoint' option to Builder, thanks @BenKanouse, https://github.com/jrochkind/faster_s3_url/pull/14
+
+## 1.1.0
+
+### Fixed
+
+- response-expires header format match recent AWS ruby SDK by using #httpdate https://github.com/jrochkind/faster_s3_url/pull/5
+
+- Only define local Storage#object_key if Shrine isn't already providing https://github.com/jrochkind/faster_s3_url/pull/7
+
+### Changed
+
+- Some decrease in memory allocations made by gem https://github.com/jrochkind/faster_s3_url/pull/6

data/Gemfile

@@ -7,5 +7,6 @@ gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
 
 gem 'pry-byebug', "~> 3.9"
-# need straight from github to get latest version without deprecations, eg https://github.com/softdevteam/libkalibera/issues/5
-gem 'kalibera', github: "softdevteam/libkalibera"
+# used by benchmark-ips in our perf/ profiling scripts.
+# https://github.com/evanphx/benchmark-ips#advanced-statistics
+gem 'kalibera'

data/README.md

@@ -2,7 +2,8 @@
 
 Generate public and presigned AWS S3 `GET` URLs faster in ruby
 
-[![Gem Version](https://badge.fury.io/rb/faster_s3_url.svg)](https://badge.fury.io/rb/faster_s3_url) [![Build Status](https://travis-ci.com/jrochkind/faster_s3_url.svg?branch=master)](https://travis-ci.com/jrochkind/faster_s3_url)
+[![Gem Version](https://badge.fury.io/rb/faster_s3_url.svg)](https://badge.fury.io/rb/faster_s3_url) [![CI Status](https://github.com/jrochkind/faster_s3_url/workflows/CI/badge.svg?branch=master)](https://github.com/jrochkind/faster_s3_url/actions?query=workflow%3ACI+branch%3Amaster)
+
 
 The official [ruby AWS SDK](https://github.com/aws/aws-sdk-ruby) is actually quite slow and unoptimized when generating URLs to access S3 objects. If you are only creating a couple S3 URLs at a time this may not matter. But it can matter on the order of even two or three hundred at a time, especially when creating presigned URLs, for which the AWS SDK is especially un-optimized.
 
@@ -74,6 +75,20 @@ builder = FasterS3Url::Builder.new(
 builder.presign_url(key) # performance enhanced
 ```
 
+### Using AWS Security Token Service (AWS STS)?
+
+When using AWS Security Token Service (AWS STS), AWS requires that the X-Amz-Security-Token parameter is set on presigned URLs. To accomplish this, you will need to pass the optional `session_token` parameter into the `FasterS3Url::Builder`.
+
+```ruby
+builder = FasterS3Url::Builder.new(
+  bucket_name: "my-bucket.example.com",
+  region: "us-east-1",
+  access_key_id: ENV['AWS_ACCESS_KEY'],
+  secret_access_key: ENV['AWS_SECRET_KEY'],
+  session_token: "session_token"
+)
+builder.presign_url(key) # includes required X-Amz-Security-Token
+```
 
 ### Automatic AWS credentials lookup?
 
@@ -89,6 +104,7 @@ credentials = credentials.credentials if credentials.respond_to?(:credentials)
 
 access_key_id     = credentials.access_key_id
 secret_access_key = credentials.secret_access_key
+session_token     = credentials.session_token # only needed when using AWS Security Token Service
 region            = client.config.region
 ```
 

data/faster_s3_url.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.summary       = %q{Generate public and presigned AWS S3 GET URLs faster}
   spec.homepage      = "https://github.com/jrochkind/faster_s3_url"
   spec.license       = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.2.0")
 
   #spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
 
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "aws-sdk-s3", "~> 1.81"
+  spec.add_development_dependency "nokogiri" # aws gem now has implicit dependency on an XML processor such as
   spec.add_development_dependency "timecop", "< 2"
   spec.add_development_dependency "benchmark-ips", "~> 2.8"
   #spec.add_development_dependency "kalibera" # for benchmark-ips :bootstrap stats option

data/lib/faster_s3_url/builder.rb

@@ -20,7 +20,8 @@ module FasterS3Url
 
     MAX_CACHED_SIGNING_KEYS = 5
 
-    attr_reader :bucket_name, :region, :host, :access_key_id
+    attr_reader :bucket_name, :region, :host, :access_key_id, :session_token
+    private attr_reader :base_url, :base_path
 
     # @option params [String] :bucket_name required
     #
@@ -28,6 +29,8 @@ module FasterS3Url
     #
     # @option params[String] :host optional, host to use in generated URLs. If empty, will construct default AWS S3 host for bucket name and region.
     #
+    # @option params[String] :endpoint optional. `endpoint` as in AWS SDK S3, to point at non-standard AWS locations. Mutually exclusive with `host`, can be used to point to alternate systems including local S3 clones like minio.
+    #
     # @option params [String] :access_key_id required at present, change to allow look up from environment using standard aws sdk routines?
     #
     # @option params [String] :secret_access_key required at present, change to allow look up from environment using standard aws sdk routines?
@@ -38,24 +41,32 @@ module FasterS3Url
     #   be cached and re-used, improving performance when generating mulitple presigned urls with a single Builder by around 50%.
     #   NOTE WELL: This will make the Builder no longer technically concurrency-safe for sharing between multiple threads, is one
     #   reason it is not on by default.
-    def initialize(bucket_name:, region:, access_key_id:, secret_access_key:, host:nil, default_public: true, cache_signing_keys: false)
+    def initialize(bucket_name:, region:, access_key_id:, secret_access_key:, session_token:nil, host:nil, endpoint: nil, default_public: true, cache_signing_keys: false)
+      if endpoint && host
+        raise ArgumentError.new("`endpoint` and `host` are mutually exclusive, you can only provide one. You provided endpoint: #{endpoint.inspect} and host: #{host.inspect}")
+      end
+
       @bucket_name = bucket_name
       @region = region
-      @host = host || default_host(bucket_name)
+
+      parsed_uri = parsed_base_uri(bucket_name: bucket_name, host: host, endpoint: endpoint)
+      @base_url = URI.join(parsed_uri, "/").to_s.chomp("/") # without path
+      @base_path = parsed_uri.path # path component of base url, usually empty
+      @host = parsed_uri.host
+      @canonical_headers = "host:#{parsed_uri.port == parsed_uri.default_port ? @host : "#{parsed_uri.host}:#{parsed_uri.port}"}\n"
+
       @default_public = default_public
       @access_key_id = access_key_id
       @secret_access_key = secret_access_key
       @cache_signing_keys = cache_signing_keys
+      @session_token = session_token
       if @cache_signing_keys
         @signing_key_cache = {}
       end
-
-
-      @canonical_headers = "host:#{@host}\n"
     end
 
     def public_url(key)
-      "https://#{self.host}/#{uri_escape_key(key)}"
+      "#{self.base_url}#{self.base_path}/#{uri_escape_key(key)}"
     end
 
     # Generates a presigned GET URL for a specified S3 object key.
@@ -95,7 +106,7 @@ module FasterS3Url
                         version_id: nil)
       validate_expires_in(expires_in)
 
-      canonical_uri = "/" + uri_escape_key(key)
+      canonical_uri = self.base_path + "/" + uri_escape_key(key)
 
       now = time ? time.dup.utc : Time.now.utc # Uh Time#utc is mutating, not nice to do to an argument!
       amz_date  = now.strftime("%Y%m%dT%H%M%SZ")
@@ -103,35 +114,25 @@ module FasterS3Url
 
       credential_scope = datestamp + '/' + region + '/' + SERVICE + '/' + 'aws4_request'
 
-      canonical_query_string_parts = [
-          "X-Amz-Algorithm=#{ALGORITHM}",
-          "X-Amz-Credential=" + uri_escape(@access_key_id + "/" + credential_scope),
-          "X-Amz-Date=" + amz_date,
-          "X-Amz-Expires=" + expires_in.to_s,
-          "X-Amz-SignedHeaders=" + SIGNED_HEADERS,
-        ]
-
-      extra_params = {
-        :"response-cache-control" => response_cache_control,
-        :"response-content-disposition" => response_content_disposition,
-        :"response-content-encoding" => response_content_encoding,
-        :"response-content-language" => response_content_language,
-        :"response-content-type" => response_content_type,
-        :"response-expires" => convert_for_timestamp_shape(response_expires),
-        :"versionId" => version_id
+      # These have to be sorted, but sort is case-sensitive, and we have a fixed
+      # list of headers we know might be here... turns out they are already sorted?
+      canonical_query_params = {
+        "X-Amz-Algorithm": ALGORITHM,
+        "X-Amz-Credential": uri_escape(@access_key_id + "/" + credential_scope),
+        "X-Amz-Date": amz_date,
+        "X-Amz-Expires": expires_in.to_s,
+        "X-Amz-Security-Token": uri_escape(session_token),
+        "X-Amz-SignedHeaders": SIGNED_HEADERS,
+        "response-cache-control": uri_escape(response_cache_control),
+        "response-content-disposition": uri_escape(response_content_disposition),
+        "response-content-encoding": uri_escape(response_content_encoding),
+        "response-content-language": uri_escape(response_content_language),
+        "response-content-type": uri_escape(response_content_type),
+        "response-expires": uri_escape(convert_for_timestamp_shape(response_expires)),
+        "versionId": uri_escape(version_id)
       }.compact
 
-
-      if extra_params.size > 0
-        # These have to be sorted, but sort is case-sensitive, and we have a fixed
-        # list of headers we know might be here... turns out they are already sorted?
-        extra_param_parts = extra_params.collect {|k, v| "#{k}=#{uri_escape v}" }.join("&")
-        canonical_query_string_parts << extra_param_parts
-      end
-
-      canonical_query_string = canonical_query_string_parts.join("&")
-
-
+      canonical_query_string = canonical_query_params.collect {|k, v| "#{k}=#{v}" }.join("&")
 
       canonical_request = ["GET",
         canonical_uri,
@@ -151,7 +152,7 @@ module FasterS3Url
       signing_key = retrieve_signing_key(datestamp)
       signature = OpenSSL::HMAC.hexdigest("SHA256", signing_key, string_to_sign)
 
-      return "https://" + self.host + canonical_uri + "?" + canonical_query_string + "&X-Amz-Signature=" + signature
+      return "#{base_url}#{canonical_uri}?#{canonical_query_string}&X-Amz-Signature=#{signature}"
     end
 
     # just a convenience method that can call public_url or presigned_url based on flag
@@ -207,29 +208,48 @@ module FasterS3Url
     end
 
 
-    # Becaues CGI.escape in MRI is written in C, this really does seem
-    # to be the fastest way to get the semantics we want, starting with
-    # CGI.escape and doing extra gsubs. Alternative would be using something
-    # else in pure C that has the semantics we want, but does not seem available.
+    # CGI.escapeURIComponent has correct semantics for what AWS wants, and is
+    # implemented in C, so pretty fast.
     def uri_escape(string)
       if string.nil?
         nil
       else
-        CGI.escape(string.encode('UTF-8')).gsub('+', '%20').gsub('%7E', '~')
+        CGI.escapeURIComponent(string.encode('UTF-8'))
       end
     end
 
     # like uri_escape but does NOT escape `/`, leaves it alone. The appropriate
     # escaping algorithm for an S3 key turning into a URL.
     #
-    # Faster to un-DRY the code with uri_escape. Yes, faster to actually just gsub
-    # %2F back to /
+    # Using CGI.escapeURIComponent with a gsub is faster than anything else
+    # we found to get this semantics.
     def uri_escape_key(string)
       if string.nil?
         nil
       else
-        CGI.escape(string.encode('UTF-8')).gsub('+', '%20').gsub('%7E', '~').gsub("%2F", "/")
+        CGI.escapeURIComponent(string.encode('UTF-8')).tap do |s|
+          s.gsub!('%2F'.freeze, '/'.freeze)
+        end
+      end
+    end
+
+    # Handle endpoint, modifying host or path with bucketname, and setting
+    # host.
+    #
+    # if none set, set default host.
+    #
+    # Set base_url correct for host or endpoint.
+    def parsed_base_uri(bucket_name:, host:, endpoint:)
+      return URI.parse("https://#{host}") if host
+      return URI.parse("https://#{default_host(bucket_name)}") if endpoint.nil?
+
+      parsed = URI.parse(endpoint)
+      if parsed.host =~ /\A\d+\.\d+\.\d+\.\d+\Z/
+        parsed.path = "/#{bucket_name}"
+      else
+        parsed.host = "#{bucket_name}.#{parsed.host}"
       end
+      parsed
     end
 
     def default_host(bucket_name)
@@ -256,7 +276,7 @@ module FasterS3Url
 
     def validate_expires_in(expires_in)
       if expires_in > ONE_WEEK
-        raise ArgumentError.new("expires_in value of #{expires_in} exceeds one-week maximum.")
+        raise ArgumentError.new("expires_in value of #{expires_in} exceeds one-week (#{ONE_WEEK}) maximum.")
       elsif expires_in <= 0
         raise ArgumentError.new("expires_in value of #{expires_in} cannot be 0 or less.")
       end
@@ -265,14 +285,13 @@ module FasterS3Url
     # Crazy kind of reverse engineered from aws-sdk-ruby,
     # for compatible handling of Expires header.
     #
-    # This honestly seems to violate the HTTP spec, the result will be that for
-    # an `response-expires` param, subsequent S3 response will include an Expires
-    # header in ISO8601 instead of HTTP-date format.... but for now we'll make
-    # our tests pass by behaving equivalently to aws-sdk-s3 anyway? filed
-    # with aws-sdk-s3: https://github.com/aws/aws-sdk-ruby/issues/2415
+    # Recent versions of ruby AWS SDK use "httpdate" format here, as a result of
+    # an issue we filed: https://github.com/aws/aws-sdk-ruby/issues/2415
+    #
+    # We match what recent AWS SDK does.
     #
-    # Switch last line from `.utc.iso8601` to `.httpdate` if you want to be
-    # more correct than aws-sdk-s3?
+    # Note while the AWS SDK source says "rfc 822", it's ruby #httpdate that matches
+    # rather than ruby #rfc822 (timezone should be `GMT` to match AWS SDK, not `-0000`)
     def convert_for_timestamp_shape(arg)
       return nil if arg.nil?
 
@@ -286,7 +305,7 @@ module FasterS3Url
         else
           Time.parse(arg.to_s)
       end
-      time_value.utc.iso8601
+      time_value.utc.httpdate
     end
   end
 end

data/lib/faster_s3_url/builder_with_new_escape.rb

@@ -0,0 +1,296 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module FasterS3Url
+  # Signing algorithm based on Amazon docs at https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html ,
+  # as well as some interactive code reading of Aws::Sigv4::Signer
+  # https://github.com/aws/aws-sdk-ruby/blob/6114bc9692039ac75c8292c66472dacd14fa6f9a/gems/aws-sigv4/lib/aws-sigv4/signer.rb
+  # as used by Aws::S3::Presigner https://github.com/aws/aws-sdk-ruby/blob/6114bc9692039ac75c8292c66472dacd14fa6f9a/gems/aws-sdk-s3/lib/aws-sdk-s3/presigner.rb
+  class BuilderWithNewEscape
+    FIFTEEN_MINUTES = 60 * 15
+    ONE_WEEK = 60 * 60 * 24 * 7
+
+    SIGNED_HEADERS = "host".freeze
+    METHOD = "GET".freeze
+    ALGORITHM = "AWS4-HMAC-SHA256".freeze
+    SERVICE = "s3".freeze
+
+    DEFAULT_EXPIRES_IN = FIFTEEN_MINUTES # 15 minutes, seems to be AWS SDK default
+
+    MAX_CACHED_SIGNING_KEYS = 5
+
+    attr_reader :bucket_name, :region, :host, :access_key_id
+
+    # @option params [String] :bucket_name required
+    #
+    # @option params [String] :region eg "us-east-1", required
+    #
+    # @option params[String] :host optional, host to use in generated URLs. If empty, will construct default AWS S3 host for bucket name and region.
+    #
+    # @option params [String] :access_key_id required at present, change to allow look up from environment using standard aws sdk routines?
+    #
+    # @option params [String] :secret_access_key required at present, change to allow look up from environment using standard aws sdk routines?
+    #
+    # @option params [boolean] :default_public (true) default value of `public` when instance method #url is called.
+    #
+    # @option params [boolean] :cache_signing_keys (false). If set to true, up to five signing keys used for presigned URLs will
+    #   be cached and re-used, improving performance when generating mulitple presigned urls with a single Builder by around 50%.
+    #   NOTE WELL: This will make the Builder no longer technically concurrency-safe for sharing between multiple threads, is one
+    #   reason it is not on by default.
+    def initialize(bucket_name:, region:, access_key_id:, secret_access_key:, host:nil, default_public: true, cache_signing_keys: false)
+      @bucket_name = bucket_name
+      @region = region
+      @host = host || default_host(bucket_name)
+      @default_public = default_public
+      @access_key_id = access_key_id
+      @secret_access_key = secret_access_key
+      @cache_signing_keys = cache_signing_keys
+      if @cache_signing_keys
+        @signing_key_cache = {}
+      end
+
+
+      @canonical_headers = "host:#{@host}\n"
+    end
+
+    def public_url(key)
+      "https://#{self.host}/#{uri_escape_key(key)}"
+    end
+
+    # Generates a presigned GET URL for a specified S3 object key.
+    #
+    # @param [String] key The S3 key to create a URL pointing to.
+    #
+    # @option params [Time] :time (Time.now) The starting time for when the
+    #   presigned url becomes active.
+    #
+    # @option params [String] :response_cache_control
+    #   Adds a `response-cache-control` query param to set the `Cache-Control` header of the subsequent response from S3.
+    #
+    # @option params [String] :response_content_disposition
+    #   Adds a `response-content-disposition` query param to set the `Content-Disposition` header of the subsequent response from S3
+    #
+    # @option params [String] :response_content_encoding
+    #   Adds a `response-content-encoding` query param to set `Content-Encoding` header of the subsequent response from S3
+    #
+    # @option params [String] :response_content_language
+    #   Adds a `response-content-language` query param to sets the `Content-Language` header of the subsequent response from S3
+    #
+    # @option params [String] :response_content_type
+    #   Adds a `response-content-type` query param to sets the `Content-Type` header of the subsequent response from S3
+    #
+    # @option params [String] :response_expires
+    #   Adds a `response-expires` query param to sets the `Expires` header of of the subsequent response from S3
+    #
+    # @option params [String] :version_id
+    #   Adds a `versionId` query param to reference a specific version of the object from S3.
+    def presigned_url(key, time: nil, expires_in: DEFAULT_EXPIRES_IN,
+                        response_cache_control: nil,
+                        response_content_disposition: nil,
+                        response_content_encoding: nil,
+                        response_content_language: nil,
+                        response_content_type: nil,
+                        response_expires: nil,
+                        version_id: nil)
+      validate_expires_in(expires_in)
+
+      canonical_uri = "/" + uri_escape_key(key)
+
+      now = time ? time.dup.utc : Time.now.utc # Uh Time#utc is mutating, not nice to do to an argument!
+      amz_date  = now.strftime("%Y%m%dT%H%M%SZ")
+      datestamp = now.strftime("%Y%m%d")
+
+      credential_scope = datestamp + '/' + region + '/' + SERVICE + '/' + 'aws4_request'
+
+      canonical_query_string_parts = [
+          "X-Amz-Algorithm=#{ALGORITHM}",
+          "X-Amz-Credential=" + uri_escape(@access_key_id + "/" + credential_scope),
+          "X-Amz-Date=" + amz_date,
+          "X-Amz-Expires=" + expires_in.to_s,
+          "X-Amz-SignedHeaders=" + SIGNED_HEADERS,
+        ]
+
+      extra_params = {
+        :"response-cache-control" => response_cache_control,
+        :"response-content-disposition" => response_content_disposition,
+        :"response-content-encoding" => response_content_encoding,
+        :"response-content-language" => response_content_language,
+        :"response-content-type" => response_content_type,
+        :"response-expires" => convert_for_timestamp_shape(response_expires),
+        :"versionId" => version_id
+      }.compact
+
+
+      if extra_params.size > 0
+        # These have to be sorted, but sort is case-sensitive, and we have a fixed
+        # list of headers we know might be here... turns out they are already sorted?
+        extra_param_parts = extra_params.collect {|k, v| "#{k}=#{uri_escape v}" }.join("&")
+        canonical_query_string_parts << extra_param_parts
+      end
+
+      canonical_query_string = canonical_query_string_parts.join("&")
+
+
+
+      canonical_request = ["GET",
+        canonical_uri,
+        canonical_query_string,
+        @canonical_headers,
+        SIGNED_HEADERS,
+        'UNSIGNED-PAYLOAD'
+      ].join("\n")
+
+      string_to_sign = [
+        ALGORITHM,
+        amz_date,
+        credential_scope,
+        Digest::SHA256.hexdigest(canonical_request)
+      ].join("\n")
+
+      signing_key = retrieve_signing_key(datestamp)
+      signature = OpenSSL::HMAC.hexdigest("SHA256", signing_key, string_to_sign)
+
+      return "https://" + self.host + canonical_uri + "?" + canonical_query_string + "&X-Amz-Signature=" + signature
+    end
+
+    # just a convenience method that can call public_url or presigned_url based on flag
+    #
+    #    signer.url(object_key, public: true)
+    #      #=> forwards to signer.public_url(object_key)
+    #
+    #    signer.url(object_key, public: false, response_content_type: "image/jpeg")
+    #       #=> forwards to signer.presigned_url(object_key, response_content_type: "image/jpeg")
+    #
+    #  Options (sucn as response_content_type) that are not applicable to #public_url
+    #  are ignored in public mode.
+    #
+    #  The default value of `public` can be set by initializer arg `default_public`, which
+    #  is itself default true.
+    #
+    #      builder = FasterS3Url::Builder.new(..., default_public: false)
+    #      builder.url(object_key) # will call #presigned_url
+    def url(key, public: @default_public, **options)
+      if public
+        public_url(key)
+      else
+        presigned_url(key, **options)
+      end
+    end
+
+
+    private
+
+    def make_signing_key(datestamp)
+      aws_get_signature_key(@secret_access_key, datestamp, @region, SERVICE)
+    end
+
+    # If caching of signing keys is turned on, use and cache signing key, while
+    # making sure not to cache more than MAX_CACHED_SIGNING_KEYS
+    #
+    # Otherwise if caching of signing keys is not turned on, just generate and return
+    # a signing key.
+    def retrieve_signing_key(datestamp)
+      if @cache_signing_keys
+        if value = @signing_key_cache[datestamp]
+          value
+        else
+          value = @signing_key_cache[datestamp] =  make_signing_key(datestamp)
+          while @signing_key_cache.size > MAX_CACHED_SIGNING_KEYS
+            @signing_key_cache.delete(@signing_key_cache.keys.first)
+          end
+          value
+        end
+      else
+        make_signing_key(datestamp)
+      end
+    end
+
+    # Becaues CGI.escape in MRI is written in C, this really does seem
+    # to be the fastest way to get the semantics we want, starting with
+    # CGI.escape and doing extra gsubs. Alternative would be using something
+    # else in pure C that has the semantics we want, but does not seem available.
+    def uri_escape(string)
+      if string.nil?
+        nil
+      else
+        CGI.escapeURIComponent(string.encode('UTF-8'))
+      end
+    end
+
+    # like uri_escape but does NOT escape `/`, leaves it alone. The appropriate
+    # escaping algorithm for an S3 key turning into a URL.
+    #
+    # Faster to un-DRY the code with uri_escape. Yes, faster to actually just gsub
+    # %2F back to /
+    def uri_escape_key(string)
+      if string.nil?
+        nil
+      else
+        CGI.escapeURIComponent(string.encode('UTF-8')).tap do |s|
+          # there is a clever way to do this in one gsub, but doesn't necessarily help
+          # memory allocations or performance
+          #s.gsub!('+'.freeze, '%20'.freeze)
+          #s.gsub!('%7E'.freeze, '~'.freeze)
+          s.gsub!('%2F'.freeze, '/'.freeze)
+        end
+      end
+    end
+
+    def default_host(bucket_name)
+      if region == "us-east-1"
+        # use legacy one without region, as S3 seems to
+        "#{bucket_name}.s3.amazonaws.com".freeze
+      else
+        "#{bucket_name}.s3.#{region}.amazonaws.com".freeze
+      end
+    end
+
+    # `def get_signature_key` `from python example at https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+    def aws_get_signature_key(key, date_stamp, region_name, service_name)
+      k_date = aws_sign("AWS4" + key, date_stamp)
+      k_region = aws_sign(k_date, region_name)
+      k_service = aws_sign(k_region, service_name)
+      aws_sign(k_service, "aws4_request")
+    end
+
+    # `def sign` from python example at https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+    def aws_sign(key, data)
+      OpenSSL::HMAC.digest("SHA256", key, data)
+    end
+
+    def validate_expires_in(expires_in)
+      if expires_in > ONE_WEEK
+        raise ArgumentError.new("expires_in value of #{expires_in} exceeds one-week (#{ONE_WEEK}) maximum.")
+      elsif expires_in <= 0
+        raise ArgumentError.new("expires_in value of #{expires_in} cannot be 0 or less.")
+      end
+    end
+
+    # Crazy kind of reverse engineered from aws-sdk-ruby,
+    # for compatible handling of Expires header.
+    #
+    # Recent versions of ruby AWS SDK use "httpdate" format here, as a result of
+    # an issue we filed: https://github.com/aws/aws-sdk-ruby/issues/2415
+    #
+    # We match what recent AWS SDK does.
+    #
+    # Note while the AWS SDK source says "rfc 822", it's ruby #httpdate that matches
+    # rather than ruby #rfc822 (timezone should be `GMT` to match AWS SDK, not `-0000`)
+    def convert_for_timestamp_shape(arg)
+      return nil if arg.nil?
+
+      time_value = case arg
+        when Time
+          arg
+        when Date, DateTime
+          arg.to_time
+        when Integer, Float
+          Time.at(arg)
+        else
+          Time.parse(arg.to_s)
+      end
+      time_value.utc.httpdate
+    end
+  end
+end

data/lib/faster_s3_url/shrine/storage.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 gem "shrine", "~> 3.0"
 require 'shrine/storage/s3'
 
@@ -59,7 +61,7 @@ module FasterS3Url
       end
 
       # For older shrine versions without it, we need this...
-      unless self.method_defined?(:object_key)
+      unless self.method_defined?(:object_key) || self.private_method_defined?(:object_key)
         def object_key(id)
           [*prefix, id].join("/")
         end

data/lib/faster_s3_url/version.rb

@@ -1,3 +1,3 @@
 module FasterS3Url
-  VERSION = "1.0.0"
+  VERSION = "1.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: faster_s3_url
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Jonathan Rochkind
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-17 00:00:00.000000000 Z
+date: 2024-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.81'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -87,9 +101,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -100,6 +115,7 @@ files:
 - faster_s3_url.gemspec
 - lib/faster_s3_url.rb
 - lib/faster_s3_url/builder.rb
+- lib/faster_s3_url/builder_with_new_escape.rb
 - lib/faster_s3_url/shrine/storage.rb
 - lib/faster_s3_url/version.rb
 - perf/presigned_bench.rb
@@ -118,14 +134,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.5.9
 signing_key:
 specification_version: 4
 summary: Generate public and presigned AWS S3 GET URLs faster

data/.travis.yml

@@ -1,6 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.6.6
-before_install: gem install bundler -v 2.1.4