faster_s3_url 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8def7b598503ea770dec6923c611bb3b5fa506ba90f89852ce299457ccbefb1e
+  data.tar.gz: 79161290e666ca39b0a9f349dc69fcb326edbeb3b975c8c0ede9c0d97c892737
+SHA512:
+  metadata.gz: 62f80a646516296746d851bcdf870c4c0e06a48af6d9e677a53c391b93ec6e4fd14dc86be73f2f540d1ce71587fc3fe2fc4abf19c6b8917e63fa75001e62a261
+  data.tar.gz: c3d738c8c3811954540725227a042da8820b9dd0a670e2aa32098c3be5cf015745d5217c904f9f6b0f6f762469f1147db0a76c8f9644d4c29ed8a0ff9d5f4559

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+Gemfile.lock
+.byebug_history

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.6
+before_install: gem install bundler -v 2.1.4

data/Gemfile

@@ -0,0 +1,11 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in faster_s3_url.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "rspec", "~> 3.0"
+
+gem 'pry-byebug', "~> 3.9"
+# need straight from github to get latest version without deprecations, eg https://github.com/softdevteam/libkalibera/issues/5
+gem 'kalibera', github: "softdevteam/libkalibera"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Science History Institute, Jonathan Rochkind
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,218 @@
+# FasterS3Url
+
+Generate public and presigned AWS S3 `GET` URLs faster in ruby
+
+[![Build Status](https://travis-ci.com/jrochkind/faster_s3_url.svg?branch=master)](https://travis-ci.com/jrochkind/faster_s3_url)
+
+The official [ruby AWS SDK](https://github.com/aws/aws-sdk-ruby) is actually quite slow and unoptimized when generating URLs to access S3 objects. If you are only creating a couple S3 URLs at a time this may not matter. But it can matter on the order of even two or three hundred at a time, especially when creating presigned URLs, for which the AWS SDK is especially un-optimized.
+
+This gem provides a much faster implementation, by around an order of magnitude, for both public  and presigned S3 `GET` URLs. Additional S3 params such as `response-content-disposition` are supported for presigned URLs.
+
+## Usage
+
+```ruby
+signer = FasterS3Url::Builder.new(
+  bucket_name: "my-bucket",
+  region: "us-east-1",
+  access_key_id: ENV['AWS_ACCESS_KEY'],
+  secret_access_key: ENV['AWS_SECRET_KEY']
+)
+
+signer.public_url("my/object/key.jpg")
+  #=> "https://my-bucket.aws"
+signer.presigned_url("my/object/key.jpg")
+```
+
+You can re-use a signer object for convenience or slighlty improved performance. It should be concurrency-safe to share globally between threads.
+
+If you are using S3 keys that need to be escaped in the URLs, this gem will escpae them properly.
+
+When presigning URLs, you can pass the query parameters supported by S3 to control subsequent response headers. You can also supply a version_id for a URL to access a specific version.
+
+```ruby
+signer.presigned_url("my/object/key.jpg"
+  response_cache_control: "public, max-age=604800, immutable",
+  response_content_disposition: "attachment",
+  response_content_language: "de-DE, en-CA",
+  response_content_type: "text/html; charset=UTF-8",
+  response_content_encoding: "deflate, gzip",
+  response_expires: "Wed, 21 Oct 2030 07:28:00 GMT",
+  version_id: "BspIL8pXg_52rGXELmqZ7cgmn7u4XJgS"
+)
+```
+
+Use a CNAME or CDN or any other hostname variant other than the default this gem will come up with? Just pass in a `host` argument to initializer. Will work with both public and presigned URLs.
+
+```ruby
+builder = FasterS3Url::Builder.new(
+  bucket_name: "my-bucket.example.com",
+  host: "my-bucket.example.com",
+  region: "us-east-1",
+  access_key_id: ENV['AWS_ACCESS_KEY'],
+  secret_access_key: ENV['AWS_SECRET_KEY']
+)
+```
+
+### Cache signing keys for further performance
+
+Under most usage patterns, the presigend URLs you generate will all use a `time` with the same UTC date. In this case, a performance advantage can be had by asking the Builder to cache and re-use AWS signing keys, which only vary with calendar date of `time` arg, not time, or S3 key, or other args. It will actually cache the 5 most recently used signing keys.  This can result in around a 50% performance improvement with a re-used Builder used for generating presigned keys.
+
+**NOTE WELL: This will technically make the Builder object no longer concurrency-safe under multiple threads.** Although you might get away with it under MRI. This is one reason it is not on by default.
+
+```ruby
+builder = FasterS3Url::Builder.new(
+  bucket_name: "my-bucket.example.com",
+  region: "us-east-1",
+  access_key_id: ENV['AWS_ACCESS_KEY'],
+  secret_access_key: ENV['AWS_SECRET_KEY'],
+  cache_signing_keys: true
+)
+builder.presign_url(key) # performance enhanced
+```
+
+
+### Automatic AWS credentials lookup?
+
+Right now, you need to explicitly supply `access_key_id` and `secret_access_key`, in part to avoid a dependency on the AWS SDK (This gem doesn't have such a dependency!). Let us know if this makes you feel a certain kind of way.
+
+If you want to look up key/secret/region using the standard SDK methods of checking various places, in order to supply them to the `FasterS3Url::Builder`, you can try this (is there a better way? Cause this is kind of a mess!)
+
+```ruby
+require 'aws-sdk-s3'
+client = Aws::S3::Client.new
+credentials = client.config.credentials
+credentails = credentials.credentials if credentials.respond_to?(:credentials)
+
+access_key_id     = credentials.access_key_id
+secret_access_key = credentials.secret_access_key
+region            = client.config.region
+```
+
+### Shrine Storage
+
+Use [shrine](https://shrinerb.com/)?  We do and love it. This gem provides a storage that can be a drop-in replacement to [Shrine::Storage::S3](https://shrinerb.com/docs/storage/s3) (shrine 3.x required), but with faster URL generation.
+
+```ruby
+# Where you might have done:
+
+require "shrine/storage/s3"
+
+s3 = Shrine::Storage::S3.new(
+  bucket: "my-app", # required
+  region: "eu-west-1", # required
+  access_key_id: "abc",
+  secret_access_key: "xyz",
+)
+
+# instead do:
+
+require "faster_s3_url/shrine/storage"
+
+s3 = FasterS3Url::Shrine::Storage.new(
+  bucket: "my-app", # required
+  region: "eu-west-1", # required
+  access_key_id: "abc", # required
+  secret_access_key: "xyz", # required
+)
+```
+
+A couple minor differences, let me know if they disrupt you:
+* We don't support the `signer` initializer argument, not clear to me why you'd want to use this gem if you are using it.
+* We support a `host` arg in initializer, but not in #url method.
+
+## Performance Benchmarking
+
+Benchmarks were done using scripts checked into repo at `./perf` (which use benchmark-ips with mode `:stats => :bootstrap, :confidence => 95`), on my 2015 Macbook Pro, using ruby MRI 2.6.6. Benchmarking is never an exact science, hopefully this is reasonable.
+
+In my narrative, I normalize to how many iterations can happen in **10ms** to have numbers closer to what might be typical use cases.
+
+### Public URLs
+
+`aws-sdk-s3` can create about 180 public URLs in 10ms, not horrible, but for how simple it seems the operation should be? FasterS3Url can do 2,200 public URLs in 10ms, that's a lot better.
+
+```
+$ bundle exec ruby perf/public_bench.rb
+Warming up --------------------------------------
+          aws-sdk-s3     1.265k i/100ms
+         FasterS3Url    24.414k i/100ms
+Calculating -------------------------------------
+          aws-sdk-s3     18.701k (± 3.2%) i/s -     92.345k in   5.048062s
+         FasterS3Url    222.938k (± 3.2%) i/s -      1.123M in   5.106971s
+                   with 95.0% confidence
+```
+
+### Presigned URLs
+
+Here's where it really starts to matter.
+
+`aws-sdk-s3` can only generate about 10 presigned URLs in 10ms, painful. FasterS3URL, with a re-used Builder object, can generate about 220 presigned URLs in 10ms, much better, and actually faster than `aws-sdk-s3` can generate public urls!  Even if we re-instantiate a Builder each time, we can generate 180 presigned URLs in 10ms, don't lose too much performance that way.
+
+If we re-use the Builder *and* turn on the (not thread-safe) `cached_signing_keys` option, we can get up to 300 presigned URLs generated in 10ms.
+
+FasterS3URL supports supplying custom query params to instruct s3 HTTP response headers. This does slow things down since they need to be URI-escaped and constructed. Using this feature with `aws-sdk-s3`, it doesn't lose much speed, down to 9 instead of 10 URLs in 10ms. FasterS3URL goes down from 210 to 180 URLs generated in 10ms (without using `cached_signing_keys` option).
+
+We can compare to the ultra-fast [wt_s3_signer](https://github.com/WeTransfer/wt_s3_signer) gem, which, with a re-used signer object (that assumes the same `time` for all URLs, unlike us; and does not support per-url custom query params) can get all the way up to 680 URLs generated in 10ms, over twice as fast as we can do even with `cached_signing_keys`. If the restrictions and API of wt_s3_signer are amenable to your use case, it's definitely the fastest. But FasterS3URL is in the ballpark, and still more than an order of magnitude faster than `aws-sdk-s3`.
+
+```
+$ bundle exec ruby perf/presigned_bench.rb
+Warming up --------------------------------------
+          aws-sdk-s3   113.000  i/100ms
+aws-sdk-s3 with custom headers
+                        95.000  i/100ms
+ re-used FasterS3Url     1.820k i/100ms
+re-used FasterS3Url with cached signing keys
+                         2.920k i/100ms
+re-used FasterS3URL with custom headers
+                         1.494k i/100ms
+new FasterS3URL Builder each time
+                         1.977k i/100ms
+re-used WT::S3Signer     7.985k i/100ms
+new WT::S3Signer each time
+                         1.611k i/100ms
+Calculating -------------------------------------
+          aws-sdk-s3      1.084k (± 4.2%) i/s -      5.311k in   5.003981s
+aws-sdk-s3 with custom headers
+                        918.315  (± 4.8%) i/s -      4.560k in   5.118770s
+ re-used FasterS3Url     21.906k (± 4.0%) i/s -    107.380k in   5.046561s
+re-used FasterS3Url with cached signing keys
+                         29.756k (± 3.6%) i/s -    146.000k in   4.999910s
+re-used FasterS3URL with custom headers
+                         18.062k (± 4.3%) i/s -     85.158k in   5.025685s
+new FasterS3URL Builder each time
+                         18.312k (± 3.9%) i/s -     90.942k in   5.098636s
+re-used WT::S3Signer     68.275k (± 3.5%) i/s -    343.355k in   5.109088s
+new WT::S3Signer each time
+                         22.425k (± 2.8%) i/s -    111.159k in   5.036814s
+                   with 95.0% confidence
+```
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install local unreleased source of this gem onto your local machine for development, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Sources/Acknowledgements
+
+[wt_s3_signer](https://github.com/WeTransfer/wt_s3_signer) served as a proof of concept, but was optimized for their particular use case, assuming batch processing where all signed S3 URLs share the same "now" time. I needed to support cases that didn't assume this, and also support custom headers like `response_content_disposition`. This code is also released with a different license. But if the API and use cases of `wt_s3_signer` meet your needs, it is even faster than this code.
+
+I tried to figure out how to do the S3 presigned request from some AWS docs:
+
+* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+* https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+
+But these don't really have all info you need for generating an S3 signature. So also ended up debugger reverse engineering the ruby aws-sdk code when generating an S3 presigned url, for instance:
+
+* https://github.com/aws/aws-sdk-ruby/blob/47c11bef18a4754ec8a05dfb637dcab120138c27/gems/aws-sdk-s3/lib/aws-sdk-s3/presigner.rb
+* but especially: https://github.com/aws/aws-sdk-ruby/blob/47c11bef18a4754ec8a05dfb637dcab120138c27/gems/aws-sigv4/lib/aws-sigv4/signer.rb
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/jrochkind/faster_s3_url.
+
+Is there a feature missing that you need? I may not be able to provide it, but I would love to hear from you!
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "faster_s3_url"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/faster_s3_url.gemspec

@@ -0,0 +1,35 @@
+require_relative 'lib/faster_s3_url/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "faster_s3_url"
+  spec.version       = FasterS3Url::VERSION
+  spec.authors       = ["Jonathan Rochkind"]
+  spec.email         = ["jrochkind@sciencehistory.org"]
+
+  spec.summary       = %q{Generate public and presigned AWS S3 GET URLs faster}
+  spec.homepage      = "https://github.com/jrochkind/faster_s3_url"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  #spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/jrochkind/faster_s3_url"
+  #spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "aws-sdk-s3", "~> 1.81"
+  spec.add_development_dependency "timecop", "< 2"
+  spec.add_development_dependency "benchmark-ips", "~> 2.8"
+  #spec.add_development_dependency "kalibera" # for benchmark-ips :bootstrap stats option
+  spec.add_development_dependency "wt_s3_signer" # just for benchmarking
+  spec.add_development_dependency "shrine", "~> 3.0" # for testing shrine storage
+end

data/lib/faster_s3_url.rb

@@ -0,0 +1,5 @@
+require "faster_s3_url/version"
+require 'faster_s3_url/builder'
+
+module FasterS3Url
+end

data/lib/faster_s3_url/builder.rb

@@ -0,0 +1,290 @@
+# frozen_string_literal: true
+
+module FasterS3Url
+  # Signing algorithm based on Amazon docs at https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html ,
+  # as well as some interactive code reading of Aws::Sigv4::Signer
+  # https://github.com/aws/aws-sdk-ruby/blob/6114bc9692039ac75c8292c66472dacd14fa6f9a/gems/aws-sigv4/lib/aws-sigv4/signer.rb
+  # as used by Aws::S3::Presigner https://github.com/aws/aws-sdk-ruby/blob/6114bc9692039ac75c8292c66472dacd14fa6f9a/gems/aws-sdk-s3/lib/aws-sdk-s3/presigner.rb
+  class Builder
+    FIFTEEN_MINUTES = 60 * 15
+    ONE_WEEK = 60 * 60 * 24 * 7
+
+    SIGNED_HEADERS = "host".freeze
+    METHOD = "GET".freeze
+    ALGORITHM = "AWS4-HMAC-SHA256".freeze
+    SERVICE = "s3".freeze
+
+    DEFAULT_EXPIRES_IN = FIFTEEN_MINUTES # 15 minutes, seems to be AWS SDK default
+
+    MAX_CACHED_SIGNING_KEYS = 5
+
+    attr_reader :bucket_name, :region, :host, :access_key_id
+
+    # @option params [String] :bucket_name required
+    #
+    # @option params [String] :region eg "us-east-1", required
+    #
+    # @option params[String] :host optional, host to use in generated URLs. If empty, will construct default AWS S3 host for bucket name and region.
+    #
+    # @option params [String] :access_key_id required at present, change to allow look up from environment using standard aws sdk routines?
+    #
+    # @option params [String] :secret_access_key required at present, change to allow look up from environment using standard aws sdk routines?
+    #
+    # @option params [boolean] :default_public (true) default value of `public` when instance method #url is called.
+    #
+    # @option params [boolean] :cache_signing_keys (false). If set to true, up to five signing keys used for presigned URLs will
+    #   be cached and re-used, improving performance when generating mulitple presigned urls with a single Builder by around 50%.
+    #   NOTE WELL: This will make the Builder no longer technically concurrency-safe for sharing between multiple threads, is one
+    #   reason it is not on by default.
+    def initialize(bucket_name:, region:, access_key_id:, secret_access_key:, host:nil, default_public: true, cache_signing_keys: false)
+      @bucket_name = bucket_name
+      @region = region
+      @host = host || default_host(bucket_name)
+      @default_public = default_public
+      @access_key_id = access_key_id
+      @secret_access_key = secret_access_key
+      @cache_signing_keys = cache_signing_keys
+      if @cache_signing_keys
+        @signing_key_cache = {}
+      end
+
+
+      @canonical_headers = "host:#{@host}\n"
+    end
+
+    def public_url(key)
+      "https://#{self.host}/#{uri_escape_key(key)}"
+    end
+
+    # Generates a presigned GET URL for a specified S3 object key.
+    #
+    # @param [String] key The S3 key to create a URL pointing to.
+    #
+    # @option params [Time] :time (Time.now) The starting time for when the
+    #   presigned url becomes active.
+    #
+    # @option params [String] :response_cache_control
+    #   Adds a `response-cache-control` query param to set the `Cache-Control` header of the subsequent response from S3.
+    #
+    # @option params [String] :response_content_disposition
+    #   Adds a `response-content-disposition` query param to set the `Content-Disposition` header of the subsequent response from S3
+    #
+    # @option params [String] :response_content_encoding
+    #   Adds a `response-content-encoding` query param to set `Content-Encoding` header of the subsequent response from S3
+    #
+    # @option params [String] :response_content_language
+    #   Adds a `response-content-language` query param to sets the `Content-Language` header of the subsequent response from S3
+    #
+    # @option params [String] :response_content_type
+    #   Adds a `response-content-type` query param to sets the `Content-Type` header of the subsequent response from S3
+    #
+    # @option params [String] :response_expires
+    #   Adds a `response-expires` query param to sets the `Expires` header of of the subsequent response from S3
+    #
+    # @option params [String] :version_id
+    #   Adds a `versionId` query param to reference a specific version of the object from S3.
+    def presigned_url(key, time: nil, expires_in: DEFAULT_EXPIRES_IN,
+                        response_cache_control: nil,
+                        response_content_disposition: nil,
+                        response_content_encoding: nil,
+                        response_content_language: nil,
+                        response_content_type: nil,
+                        response_expires: nil,
+                        version_id: nil)
+      validate_expires_in(expires_in)
+
+      canonical_uri = "/" + uri_escape_key(key)
+
+      now = time ? time.dup.utc : Time.now.utc # Uh Time#utc is mutating, not nice to do to an argument!
+      amz_date  = now.strftime("%Y%m%dT%H%M%SZ")
+      datestamp = now.strftime("%Y%m%d")
+
+      credential_scope = datestamp + '/' + region + '/' + SERVICE + '/' + 'aws4_request'
+
+      canonical_query_string_parts = [
+          "X-Amz-Algorithm=#{ALGORITHM}",
+          "X-Amz-Credential=" + uri_escape(@access_key_id + "/" + credential_scope),
+          "X-Amz-Date=" + amz_date,
+          "X-Amz-Expires=" + expires_in.to_s,
+          "X-Amz-SignedHeaders=" + SIGNED_HEADERS,
+        ]
+
+      extra_params = {
+        :"response-cache-control" => response_cache_control,
+        :"response-content-disposition" => response_content_disposition,
+        :"response-content-encoding" => response_content_encoding,
+        :"response-content-language" => response_content_language,
+        :"response-content-type" => response_content_type,
+        :"response-expires" => convert_for_timestamp_shape(response_expires),
+        :"versionId" => version_id
+      }.compact
+
+
+      if extra_params.size > 0
+        # These have to be sorted, but sort is case-sensitive, and we have a fixed
+        # list of headers we know might be here... turns out they are already sorted?
+        extra_param_parts = extra_params.collect {|k, v| "#{k}=#{uri_escape v}" }.join("&")
+        canonical_query_string_parts << extra_param_parts
+      end
+
+      canonical_query_string = canonical_query_string_parts.join("&")
+
+
+
+      canonical_request = ["GET",
+        canonical_uri,
+        canonical_query_string,
+        @canonical_headers,
+        SIGNED_HEADERS,
+        'UNSIGNED-PAYLOAD'
+      ].join("\n")
+
+      string_to_sign = [
+        ALGORITHM,
+        amz_date,
+        credential_scope,
+        Digest::SHA256.hexdigest(canonical_request)
+      ].join("\n")
+
+      signing_key = retrieve_signing_key(datestamp)
+      signature = OpenSSL::HMAC.hexdigest("SHA256", signing_key, string_to_sign)
+
+      return "https://" + self.host + canonical_uri + "?" + canonical_query_string + "&X-Amz-Signature=" + signature
+    end
+
+    # just a convenience method that can call public_url or presigned_url based on flag
+    #
+    #    signer.url(object_key, public: true)
+    #      #=> forwards to signer.public_url(object_key)
+    #
+    #    signer.url(object_key, public: false, response_content_type: "image/jpeg")
+    #       #=> forwards to signer.presigned_url(object_key, response_content_type: "image/jpeg")
+    #
+    #  Options (sucn as response_content_type) that are not applicable to #public_url
+    #  are ignored in public mode.
+    #
+    #  The default value of `public` can be set by initializer arg `default_public`, which
+    #  is itself default true.
+    #
+    #      builder = FasterS3Url::Builder.new(..., default_public: false)
+    #      builder.url(object_key) # will call #presigned_url
+    def url(key, public: @default_public, **options)
+      if public
+        public_url(key)
+      else
+        presigned_url(key, **options)
+      end
+    end
+
+
+    private
+
+    def make_signing_key(datestamp)
+      aws_get_signature_key(@secret_access_key, datestamp, @region, SERVICE)
+    end
+
+    # If caching of signing keys is turned on, use and cache signing key, while
+    # making sure not to cache more than MAX_CACHED_SIGNING_KEYS
+    #
+    # Otherwise if caching of signing keys is not turned on, just generate and return
+    # a signing key.
+    def retrieve_signing_key(datestamp)
+      if @cache_signing_keys
+        if value = @signing_key_cache[datestamp]
+          value
+        else
+          value = @signing_key_cache[datestamp] =  make_signing_key(datestamp)
+          while @signing_key_cache.size > MAX_CACHED_SIGNING_KEYS
+            @signing_key_cache.delete(@signing_key_cache.keys.first)
+          end
+          value
+        end
+      else
+        make_signing_key(datestamp)
+      end
+    end
+
+
+    # Becaues CGI.escape in MRI is written in C, this really does seem
+    # to be the fastest way to get the semantics we want, starting with
+    # CGI.escape and doing extra gsubs. Alternative would be using something
+    # else in pure C that has the semantics we want, but does not seem available.
+    def uri_escape(string)
+      if string.nil?
+        nil
+      else
+        CGI.escape(string.encode('UTF-8')).gsub('+', '%20').gsub('%7E', '~')
+      end
+    end
+
+    # like uri_escape but does NOT escape `/`, leaves it alone. The appropriate
+    # escaping algorithm for an S3 key turning into a URL.
+    #
+    # Faster to un-DRY the code with uri_escape. Yes, faster to actually just gsub
+    # %2F back to /
+    def uri_escape_key(string)
+      if string.nil?
+        nil
+      else
+        CGI.escape(string.encode('UTF-8')).gsub('+', '%20').gsub('%7E', '~').gsub("%2F", "/")
+      end
+    end
+
+    def default_host(bucket_name)
+      if region == "us-east-1"
+        # use legacy one without region, as S3 seems to
+        "#{bucket_name}.s3.amazonaws.com".freeze
+      else
+        "#{bucket_name}.s3.#{region}.amazonaws.com".freeze
+      end
+    end
+
+    # `def get_signature_key` `from python example at https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+    def aws_get_signature_key(key, date_stamp, region_name, service_name)
+      k_date = aws_sign("AWS4" + key, date_stamp)
+      k_region = aws_sign(k_date, region_name)
+      k_service = aws_sign(k_region, service_name)
+      aws_sign(k_service, "aws4_request")
+    end
+
+    # `def sign` from python example at https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+    def aws_sign(key, data)
+      OpenSSL::HMAC.digest("SHA256", key, data)
+    end
+
+    def validate_expires_in(expires_in)
+      if expires_in > ONE_WEEK
+        raise ArgumentError.new("expires_in value of #{expires_in} exceeds one-week maximum.")
+      elsif expires_in <= 0
+        raise ArgumentError.new("expires_in value of #{expires_in} cannot be 0 or less.")
+      end
+    end
+
+    # Crazy kind of reverse engineered from aws-sdk-ruby,
+    # for compatible handling of Expires header.
+    #
+    # This honestly seems to violate the HTTP spec, the result will be that for
+    # an `response-expires` param, subsequent S3 response will include an Expires
+    # header in ISO8601 instead of HTTP-date format.... but for now we'll make
+    # our tests pass by behaving equivalently to aws-sdk-s3 anyway? filed
+    # with aws-sdk-s3: https://github.com/aws/aws-sdk-ruby/issues/2415
+    #
+    # Switch last line from `.utc.iso8601` to `.httpdate` if you want to be
+    # more correct than aws-sdk-s3?
+    def convert_for_timestamp_shape(arg)
+      return nil if arg.nil?
+
+      time_value = case arg
+        when Time
+          arg
+        when Date, DateTime
+          arg.to_time
+        when Integer, Float
+          Time.at(arg)
+        else
+          Time.parse(arg.to_s)
+      end
+      time_value.utc.iso8601
+    end
+  end
+end

data/lib/faster_s3_url/shrine/storage.rb

@@ -0,0 +1,66 @@
+gem "shrine", "~> 3.0"
+require 'shrine/storage/s3'
+
+module FasterS3Url
+  module Shrine
+    # More or less a drop-in replacement for Shrine::Storage::S3 , that uses FasterS3Url faster S3 URL generation.
+    # https://shrinerb.com/docs/storage/s3
+    #
+    #     require 'faster_s3_url/storage/shrine'
+    #
+    #     s3 = FasterS3Url::Shrine::Storage.new(
+    #       bucket: "my-app", # required
+    #       region: "eu-west-1", # required
+    #       access_key_id: "abc",
+    #       secret_access_key: "xyz"
+    #     )
+    #
+    # A couple incompatibilities with Shrine::Storage::S3, which I don't expect to cause problems
+    # for anyone but if they do please let me know.
+    #
+    #  * we do not support the :signer option in initialier (why would you want to use that with this? Let me know)
+    #
+    #  * We support a `host` option on initializer, but do NOT support the `host` option on #url (I don't underestand
+    #  why it's per-call in the first place, do you need it to be?)
+    #
+    class Storage < ::Shrine::Storage::S3
+      # Same options as Shrine::Storage::S3, plus `host`
+      def initialize(**options)
+        if options[:signer]
+          raise ArgumentError.new("#{self.class.name} does not support :signer option of Shrine::Storage::S3. Should it? Let us know.")
+        end
+
+        host = options.delete(:host)
+        @faster_s3_url_builder = FasterS3Url::Builder.new(
+          bucket_name: options[:bucket],
+          access_key_id: options[:access_key_id],
+          secret_access_key: options[:secret_access_key],
+          region: options[:region],
+          host: host)
+
+        super(**options)
+      end
+
+      # unlike base Shrine::Storage::S3, does not support `host` here, do it in
+      # initializer instead. Is there a really use case for doing it here?
+      # If so let us know.
+      #
+      # options are ignored when public mode, so you can send options the same
+      # for public or not, and not get an error on public for options only appropriate
+      # to presigned.
+      #
+      # Otherwise, same options as Shrine::S3::Storage should be supported, please
+      # see docs there. https://shrinerb.com/docs/storage/s3
+      def url(id, public: self.public, **options)
+        @faster_s3_url_builder.url(object_key(id), public: public, **options)
+      end
+
+      # For older shrine versions without it, we need this...
+      unless self.method_defined?(:object_key)
+        def object_key(id)
+          [*prefix, id].join("/")
+        end
+      end
+    end
+  end
+end

data/lib/faster_s3_url/version.rb

@@ -0,0 +1,3 @@
+module FasterS3Url
+  VERSION = "0.1.0"
+end

data/perf/presigned_bench.rb

@@ -0,0 +1,86 @@
+# Run as eg:
+#
+# $ bundle exec ruby perf/presigned_bench.rb
+
+require 'benchmark/ips'
+require 'faster_s3_url'
+require 'aws-sdk-s3'
+require 'wt_s3_signer'
+
+access_key_id =  "fakeExampleAccessKeyId"
+secret_access_key = "fakeExampleSecretAccessKey"
+
+bucket_name = "my-bucket"
+object_key =  "some/directory/file.jpg"
+region = "us-east-1"
+
+aws_client = Aws::S3::Client.new(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key)
+aws_bucket = Aws::S3::Bucket.new(name: bucket_name, client: aws_client)
+
+faster_s3_builder = FasterS3Url::Builder.new(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key, bucket_name: bucket_name)
+
+faster_s3_builder_with_caching = FasterS3Url::Builder.new(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key, bucket_name: bucket_name, cache_signing_keys: true)
+
+wt_signer = WT::S3Signer.new(
+                    expires_in: 15 * 60,
+                    aws_region: "us-east-1",
+                    bucket_endpoint_url: "https://#{bucket_name}.s3.amazonaws.com",
+                    bucket_host: "#{bucket_name}.s3.amazonaws.com",
+                    bucket_name: bucket_name,
+                    access_key_id: access_key_id,
+                    secret_access_key: secret_access_key,
+                    session_token: nil
+)
+
+Benchmark.ips do |x|
+  begin
+    require 'kalibera'
+    x.config(:stats => :bootstrap, :confidence => 95)
+  rescue LoadError
+
+  end
+
+  x.report("aws-sdk-s3") do
+    aws_bucket.object(object_key).presigned_url(:get)
+  end
+
+  x.report("aws-sdk-s3 with custom headers") do
+    aws_bucket.object(object_key).presigned_url(:get, response_content_type: "image/jpeg", response_content_disposition: "attachment; filename=\"foo bar.baz\"; filename*=UTF-8''foo%20bar.baz")
+  end
+
+  x.report("re-used FasterS3Url") do
+    faster_s3_builder.presigned_url(object_key)
+  end
+
+  x.report("re-used FasterS3Url with cached signing keys") do
+    faster_s3_builder_with_caching.presigned_url(object_key)
+  end
+
+  x.report("re-used FasterS3URL with custom headers") do
+    faster_s3_builder.presigned_url(object_key, response_content_type: "image/jpeg", response_content_disposition: "attachment; filename=\"foo bar.baz\"; filename*=UTF-8''foo%20bar.baz")
+  end
+
+  x.report("new FasterS3URL Builder each time") do
+    builder = FasterS3Url::Builder.new(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key, bucket_name: bucket_name)
+    builder.presigned_url(object_key)
+  end
+
+  x.report("re-used WT::S3Signer") do
+    wt_signer.presigned_get_url(object_key: object_key)
+  end
+
+  x.report("new WT::S3Signer each time") do
+    signer = WT::S3Signer.new(
+                    expires_in: 15 * 60,
+                    aws_region: "us-east-1",
+                    bucket_endpoint_url: "https://#{bucket_name}.s3.amazonaws.com",
+                    bucket_host: "#{bucket_name}.s3.amazonaws.com",
+                    bucket_name: bucket_name,
+                    access_key_id: access_key_id,
+                    secret_access_key: secret_access_key,
+                    session_token: nil)
+    signer.presigned_get_url(object_key: object_key)
+  end
+
+end
+

data/perf/public_bench.rb

@@ -0,0 +1,38 @@
+# Run as eg:
+#
+# $ bundle exec ruby perf/public_bench.rb
+
+require 'benchmark/ips'
+require 'faster_s3_url'
+require 'aws-sdk-s3'
+
+access_key_id =  "fakeExampleAccessKeyId"
+secret_access_key = "fakeExampleSecretAccessKey"
+
+bucket_name = "my-bucket"
+object_key =  "some/directory/file.jpg"
+region = "us-east-1"
+
+aws_client = Aws::S3::Client.new(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key)
+aws_bucket = Aws::S3::Bucket.new(name: bucket_name, client: aws_client)
+
+faster_s3_builder = FasterS3Url::Builder.new(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key, bucket_name: bucket_name)
+
+Benchmark.ips do |x|
+  begin
+    require 'kalibera'
+    x.config(:stats => :bootstrap, :confidence => 95)
+  rescue LoadError
+
+  end
+
+
+  x.report("aws-sdk-s3") do
+    aws_bucket.object(object_key).public_url
+  end
+
+  x.report("FasterS3Url") do
+    faster_s3_builder.public_url(object_key)
+  end
+end
+

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: faster_s3_url
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jonathan Rochkind
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2020-10-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.81'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.81'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: wt_s3_signer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shrine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description:
+email:
+- jrochkind@sciencehistory.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/rspec
+- bin/setup
+- faster_s3_url.gemspec
+- lib/faster_s3_url.rb
+- lib/faster_s3_url/builder.rb
+- lib/faster_s3_url/shrine/storage.rb
+- lib/faster_s3_url/version.rb
+- perf/presigned_bench.rb
+- perf/public_bench.rb
+homepage: https://github.com/jrochkind/faster_s3_url
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/jrochkind/faster_s3_url
+  source_code_uri: https://github.com/jrochkind/faster_s3_url
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key:
+specification_version: 4
+summary: Generate public and presigned AWS S3 GET URLs faster
+test_files: []