s3_asset_deploy 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 72c4c1220b3051d9189389b3b956d509506380ae4bef112212480e1da9ac75e4
+  data.tar.gz: b9fbd6e07c1cced6e5a20d1cb72338093eec260a540a434ee3cc60012e680549
+SHA512:
+  metadata.gz: cb3a963f7a47375414e9b51b1b3f525a890924995b380d193151e9a77193a7ddec8f1f50864fc60b37c5c768586fefa76b1335d3e849918572ec4b7be3af866a
+  data.tar.gz: d240fe42ae5578cf136f1dac4b637165603836f8cdceb7bf8d820cd6808048d78b50b4dbbe235f3e1678aedbe8f36a6b76f1310a5d64f0605622650cca2d01b1

data/.circleci/config.yml

@@ -0,0 +1,14 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@1.1.2
+
+jobs:
+  build:
+    parallelism: 1
+    docker:
+      - image: circleci/ruby:2.6.3-stretch
+    executor: ruby/default
+    steps:
+      - checkout
+      - ruby/install-deps
+      - ruby/rspec-test

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/CHANGELOG.md

@@ -0,0 +1 @@
+# Changelog

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,84 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at contact@loomly.com. All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior,  harassment of an individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0,
+available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations.

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in s3_asset_deploy.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,72 @@
+PATH
+  remote: .
+  specs:
+    s3_asset_deploy (0.1.0)
+      aws-sdk-s3 (~> 1.0)
+      mime-types (~> 3.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-eventstream (1.1.0)
+    aws-partitions (1.422.0)
+    aws-sdk-core (3.111.2)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-kms (1.41.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-s3 (1.87.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.2.2)
+      aws-eventstream (~> 1, >= 1.0.2)
+    byebug (11.1.3)
+    coderay (1.1.3)
+    diff-lcs (1.4.4)
+    jmespath (1.4.0)
+    method_source (1.0.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.1104)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    rake (13.0.3)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rspec_junit_formatter (0.4.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    timecop (0.9.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pry (~> 0.13)
+  pry-byebug (~> 3.9)
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rspec_junit_formatter (~> 0.4)
+  s3_asset_deploy!
+  timecop (~> 0.9)
+
+BUNDLED WITH
+   2.2.7

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Loomly
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,180 @@
+# S3AssetDeploy
+
+[![CircleCI](https://circleci.com/gh/Loomly/s3_asset_deploy.svg?style=shield)](https://circleci.com/gh/Loomly/s3_asset_deploy)
+
+During rolling deploys to our web instances, this is what we use at
+[Loomly](https://www.loomly.com) to safely deploy our web assets to S3 to be served via Cloudfront.
+This gem is designed to upload and clean unneeded assets from S3 in a safe manner such that older
+versions or recently removed assets are kept on S3 during the rolling deploy process.
+It also maintains a version limit and TTL (time-to-live) on assets to avoid deleting
+recent and outdated versions (up to a limit) or those that have been recently removed.
+
+## Background
+
+At the very beginning, we were serving our assets from our webservers. This isn't ideal for many reasons, but one big one is that it's problematic during rolling deploys where you temporarily have some web servers with new assets and some with old assets during the deploy. When round-robbining requests to instances behind a load balancer, this can result in requests for assets hitting web servers that don't have the asset being requested (either the new or the old depending on what web server and what's being requested). One way to fix this problem is to serve your assets from a CDN and keep both old and new versions of assets available on the CDN during the deploy process. So we decided to serve our assets from Cloudfront, backed by S3. In order to upload our assets to S3 during our deploy process, we started using [`asset_sync`](https://github.com/AssetSync/asset_sync). `asset_sync` served us well for quite some time, but our needs started to diverge a bit. Namely, `asset_sync`:
+
+- Depends on the [`fog`](https://github.com/fog/fog) gem which was an extra dependency we didn't need since we already had the [`aws-sdk-s3`](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3.html) gem as a dependency.
+- Uses a global configuration, which made it difficult to deploy to different S3 buckets depending on the environment (development, staging, production, etc.).
+- Didn't have a way to remove or retire outdated or old assets from storage (in this case S3).
+
+We took inspiration from `asset_sync` and ended up writing our own library inside our Rails app. We figured this could be useful to others, so we then moved it to an open source gem. While Rails is a "first-class citizen", this gem can be used with other frameworks by writing your own `S3AssetDeploy::LocalAssetCollector`. See the `Usage` section below for more details.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "s3_asset_deploy"
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install s3_asset_deploy
+
+## Usage
+
+Before using `S3AssetDeploy` you want to make sure to compile your assets. Assets must also be compiled using [fingerprinting](https://guides.rubyonrails.org/asset_pipeline.html#what-is-fingerprinting-and-why-should-i-care-questionmark) for things to work correctly. By default, `S3AssetDeploy` works with Rails and will find your locally compiled assets after running `rake assets:precompile`. Once you've compiled your assets, you can deploy them with:
+
+
+```ruby
+manager = S3AssetDeploy::Manager("my-s3-bucket")
+manager.deploy do
+  # Perform deploy to web instances in this block
+end
+```
+
+`S3AssetDeploy::Manager#deploy` will perform the following steps:
+- Upload your assets to the S3 bucket you specify
+- Yield to the block
+- Clean old versions assets or removed assets
+
+Since it's yielding to the block after uploading, but before cleaning, the block is an ideal place to perform a deploy, especially if it's a rolling deploy across multiple servers. If you want to perform an upload or a clean without using `#deploy`, you can call `#upload` or `#clean` directly. For more configuration options, see below.
+
+### Initializing [`S3AssetDeploy::Manager`](https://github.com/Loomly/s3_asset_deploy/blob/main/lib/s3_asset_deploy/manager.rb)
+You'll need to initialize `S3AssetDeploy::Manager` with an S3 bucket name and **optionally**:
+
+- **s3_client_options** (Hash) -> A hash that is passed directly to [`Aws::S3::Client#initialize`](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#initialize-instance_method) to configure the S3 client. By default the region is set to `us-east-1`.
+- **logger** (Logger) -> A custom logger. By default things are logged to `STDOUT`.
+- **local_asset_collector** (S3AssetDeploy::LocalAssetCollector) -> A custom instance of `S3AssetDeploy::LocalAssetCollector`. This allows you to customize how locally compiled assets are collected.
+- **upload_options** (Hash) -> A hash consisting of options that are passed directly to [`Aws::S3::Client#put_object`](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#put_object-instance_method) when each asset is uploaded. By default `acl` is set to `public-read` and `cache_control` is set to `public, max-age=31536000`.
+- **remove_fingerprint** (Lambda) -> Lambda for overriding how fingerprints are removed from asset paths. Fingerprints need to be removed during the cleaning process in order to group versions of the same file. If no Lambda is provided, [`S3AssetDeploy::AssetHelper.remove_fingerprint`](https://github.com/Loomly/s3_asset_deploy/blob/main/lib/s3_asset_deploy/asset_helper.rb#L8) is used by default.
+
+Here's an example:
+
+```ruby
+manager = S3AssetDeploy::Manager.new(
+  "mybucket",
+  s3_client_options: { region: "us-west-1", profile: "my-aws-profile" },
+  logger: Logger.new(STDOUT),
+  remove_fingerprint: ->(path) { path.gsub("-myfingerprint", "") }
+)
+```
+
+### Deploying Assets
+Once you have an instance of `S3AssetDeploy::Manager`, you can deploy your precompiled assets with `S3AssetDeploy::Manager#deploy`:
+
+```ruby
+manager.deploy(version_limit: 2, version_ttl: 3600, removed_ttl: 172800) do
+  # Perform deploy to web instances in this block
+end
+```
+
+This will upload new assets and perform a clean, which deletes removed assets and old versions from your bucket after the block is executed.
+With the arguments used above, the clean process will keep the latest version on S3, two of the most recent older versions (`version_limit`), and any versions created within the last hour (`version_ttl`).
+If you there are assets that are in your S3 bucket but no longer included in your locally compiled bundle, they will be deleted from S3 using the `removed_ttl` (after two days in the case above). This process uses [S3 object tagging](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#put_object_tagging-instance_method) to track `removed_at` timestamps. Here are a list of all the options you can pass to `#deploy`:
+
+- **version_limit** (Integer) -> Max number of older versions of an asset to keep around. Note that this limit does **not** include the current version. Therefore, setting this to 0 will keep the current version and delete any older versions. Default is `2`.
+- **version_ttl** (Integer) -> Number of seconds to keep newly uploaded versions before deleting according to `version_limit`. If an older version is still within the `version_ttl`, it will be kept on S3 even if the total number of older versions is beyond `version_limit`. Default is `3600`.
+- **removed_ttl** (Integer) -> Number of seconds to keep assets on S3 that have been removed from your compiled set of assets. If the age of a removed asset expires according to `removed_ttl`, it will be deleted on the next deploy. Default is `172800`.
+- **clean** (Boolean) -> Skip the clean process during a deploy. Default is `true`.
+- **dry_run** (Boolean) -> Run deploy in read-only mode. This is helpful for debugging purposes and seeing plan of what would happen without performing any writes or deletes. Default is `false`.
+
+`S3AssetDeploy::Manager#deploy` performs its work by delegating to `S3AssetDeploy#upload` and `S3AssetDeploy#clean`, which you can call yourself if you need some more control.
+
+
+```ruby
+# Upload new assets
+manager.upload
+
+# Delete old versions and removed assets from S3
+manager.clean
+```
+
+`S3AssetDeploy::Manager#deploy` and `S3AssetDeploy::Manager#clean` both accept `dry_run` as a keyword argument.
+`S3AssetDeploy::Manager#clean` also accepts `version_limit`, `version_ttl`, and `removed_ttl` just like `S3AssetDeploy::Manager#deploy`.
+
+
+## Customizing local asset collection
+By default, `S3AssetDeploy::Manager` will use [`S3AssetDeploy::RailsLocalAssetCollector`](https://github.com/Loomly/s3_asset_deploy/blob/main/lib/s3_asset_deploy/rails_local_asset_collector.rb) to collect locally compiled assets. This will use the `Sprockets::Manifest` and `Webpacker` config (if `Webpacker` is installed) to locate the compiled assets. `S3AssetDeploy::RailsLocalAssetCollector` inherits from the [`S3AssetDeploy::LocalAssetCollector`](https://github.com/Loomly/s3_asset_deploy/blob/main/lib/s3_asset_deploy/local_asset_collector.rb) base class. You can completely customize how your local assets are collected for deploys by creating your own class that inherits from `S3AssetDeploy::LocalAssetCollector` and passing it into the manager. You'll want override `S3AssetDeploy::LocalAssetCollector#assets` in your custom collector such that it returns an array of `S3AssetDeploy::LocalAsset` instances. Here's a basic example:
+
+
+```ruby
+class MyCustomLocalAssetCollector < S3AssetDeploy::LocalAssetCollector
+  def assets
+    # Override this method to return an array of your locally compiled assets
+    # as instances of S3AssetDeploy::LocalAsset
+    [S3AssetDeploy::LocalAsset.new("path-to-my-asset.jpg")]
+  end
+end
+
+manager = S3AssetDeploy::Manager.new(
+  "mybucket",
+  local_asset_collector: MyCustomLocalAssetCollector.new
+)
+```
+
+## Dry run
+As mentioned above, you can run operations in "dry" mode by passing `dry_run: true`.
+This will skip any write or delete operations and only perform read opeartions with log output.
+This is helpful for debugging or planning purposes.
+
+```ruby
+> manager = S3AssetDeploy::Manager("my-s3-bucket")
+> manager.deploy(dry_run: true)
+
+I, [2021-02-17T16:12:23.703677 #65335]  INFO -- : S3AssetDeploy::Manager: Cleaning assets from test-bucket S3 bucket. Dry run: true
+I, [2021-02-17T16:12:23.703677 #65335]  INFO -- : S3AssetDeploy::Manager: Determining how long ago assets/file-2-34567.jpg was removed - removed on 2021-02-15 23:12:22 UTC (172801.703677 seconds ago)
+I, [2021-02-17T16:12:23.703677 #65335]  INFO -- : S3AssetDeploy::Manager: Determining how long ago assets/file-3-9876666.jpg was removed - removed on 2021-02-15 23:12:24 UTC (172799.703677 seconds ago)
+```
+
+## AWS IAM Permissions
+`S3AsetDeploy` requires the following AWS IAM permissions:
+
+```json
+"Statement": [
+  {
+    "Action": [
+      "s3:ListBucket",
+      "s3:PutObject*",
+      "s3:DeleteObject"
+    ],
+    "Effect": "Allow",
+    "Resource": [
+      "arn:aws:s3:::#{YOUR_BUCKET}",
+      "arn:aws:s3:::#{YOUR_BUCKET}/*"
+    ]
+  }
+]
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/Loomly/s3_asset_deploy. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/Loomly/s3_asset_deploy/blob/main/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the S3AssetDeploy project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/Loomly/s3_asset_deploy/blob/main/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "s3_asset_deploy"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/s3_asset_deploy.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require "s3_asset_deploy/version"
+require "s3_asset_deploy/manager"
+
+module S3AssetDeploy
+end

data/lib/s3_asset_deploy/asset_helper.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "mime/types"
+
+class S3AssetDeploy::AssetHelper
+  FINGERPRINTED_ASSET_REGEX = /\A(.*)-([[:alnum:]]+)((?:(?:\.[[:alnum:]]+))+)\z/.freeze
+
+  def self.remove_fingerprint(path)
+    match_data = path.match(FINGERPRINTED_ASSET_REGEX)
+    return asset_path unless match_data
+    "#{match_data[1]}#{match_data[3]}"
+  end
+
+  def self.mime_type_for_path(path)
+    extension = File.extname(path)[1..-1]
+    return "application/json" if extension == "map"
+    MIME::Types.type_for(extension).first
+  end
+end

data/lib/s3_asset_deploy/errors.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class S3AssetDeploy::Error < StandardError
+end
+
+class S3AssetDeploy::DuplicateAssetsError < S3AssetDeploy::Error
+  def initialize(msg = "Duplicate precompiled assets detected. Please make sure there are no duplicate precompiled assets in the public dir.")
+    super
+  end
+end

data/lib/s3_asset_deploy/local_asset.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "s3_asset_deploy/asset_helper"
+
+class S3AssetDeploy::LocalAsset
+  attr_reader :path
+
+  def initialize(path, remove_fingerprint: nil)
+    @path = path
+    @remove_fingerprint = remove_fingerprint
+  end
+
+  def original_path
+    @original_path ||=
+      if @remove_fingerprint
+        @remove_fingerprint.call(path)
+      else
+        S3AssetDeploy::AssetHelper.remove_fingerprint(path)
+      end
+  end
+
+  def full_path
+    File.join(ENV["PWD"], "public", path)
+  end
+
+  def mime_type
+    S3AssetDeploy::AssetHelper.mime_type_for_path(path).to_s
+  end
+
+  def ==(other_asset)
+    path == other_asset.path
+  end
+
+  def to_s
+    path
+  end
+end

data/lib/s3_asset_deploy/local_asset_collector.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class S3AssetDeploy::LocalAssetCollector
+  def initialize(remove_fingerprint: nil)
+    @remove_fingerprint = remove_fingerprint
+  end
+
+  def assets
+    []
+  end
+
+  def asset_paths
+    assets.map(&:path)
+  end
+
+  def asset_map
+    assets.map do |asset|
+      [asset.original_path, asset]
+    end.to_h
+  end
+
+  def original_asset_paths
+    assets.map(&:original_path)
+  end
+
+  def full_file_path(asset_path)
+    asset_path
+  end
+end

data/lib/s3_asset_deploy/manager.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require "logger"
+require "time"
+require "aws-sdk-s3"
+require "s3_asset_deploy/errors"
+require "s3_asset_deploy/removal_manifest"
+require "s3_asset_deploy/rails_local_asset_collector"
+require "s3_asset_deploy/remote_asset_collector"
+
+class S3AssetDeploy::Manager
+  attr_reader :bucket_name, :logger, :local_asset_collector, :remote_asset_collector
+
+  def initialize(bucket_name, s3_client_options: {}, logger: nil, local_asset_collector: nil, upload_options: {}, remove_fingerprint: nil)
+    @bucket_name = bucket_name.to_s
+    @logger = logger || Logger.new(STDOUT)
+
+    @s3_client_options = {
+      region: "us-east-1",
+      logger: @logger
+    }.merge(s3_client_options)
+    @upload_options = upload_options
+
+    @local_asset_collector = local_asset_collector || S3AssetDeploy::RailsLocalAssetCollector.new(remove_fingerprint: remove_fingerprint)
+    @remote_asset_collector = S3AssetDeploy::RemoteAssetCollector.new(
+      bucket_name,
+      s3_client_options: @s3_client_options,
+      remove_fingerprint: remove_fingerprint
+    )
+  end
+
+  def removal_manifest
+    @removal_manifest ||= S3AssetDeploy::RemovalManifest.new(
+      bucket_name,
+      s3_client_options: @s3_client_options
+    )
+  end
+
+  def local_assets_to_upload
+    remote_asset_paths = remote_asset_collector.asset_paths
+    local_asset_collector.assets.reject { |asset| remote_asset_paths.include?(asset.path) }
+  end
+
+  def upload(dry_run: false)
+    verify_no_duplicate_assets!
+
+    removal_manifest.load
+    assets_to_upload = local_assets_to_upload
+
+    (removal_manifest.keys & local_asset_collector.asset_paths).each do |path|
+      log "#{path} has been re-added. Deleting from removal manifest."
+      removal_manifest.delete(path) unless dry_run
+    end
+
+    uploaded_assets = []
+    assets_to_upload.each do |asset|
+      next unless File.file?(asset.full_path)
+      log "Uploading #{asset.path}..."
+      upload_asset(asset) unless dry_run
+      uploaded_assets << asset.path
+    end
+
+    removal_manifest.save unless dry_run
+    remote_asset_collector.clear_cache
+    uploaded_assets
+  end
+
+  # Cleanup old assets on S3. By default it will
+  # keep the latest version, 2 older versions (version_limit) and any created within the past hour (version_ttl).
+  # When assets are removed completely, they are tagged with a removed_at timestamp
+  # and eventually deleted based on the removed_ttl.
+  def clean(version_limit: 2, version_ttl: 3600, removed_ttl: 172800, dry_run: false)
+    verify_no_duplicate_assets!
+
+    version_ttl = version_ttl.to_i
+    removed_ttl = removed_ttl.to_i
+
+    log "Cleaning assets from #{bucket_name} S3 bucket. Dry run: #{dry_run}."
+    s3_keys_to_delete = []
+
+    unless local_assets_to_upload.empty?
+      log "WARNING: Please upload latest asset versions to remote host before cleaning."
+      return s3_keys_to_delete
+    end
+
+    removal_manifest.load
+    local_asset_map = local_asset_collector.asset_map
+
+    remote_asset_collector.grouped_assets.each do |original_path, versions|
+      current_asset = local_asset_map[original_path]
+
+      # Remove current asset version from the list
+      versions_to_delete = versions.reject do |version|
+        version.path == current_asset.path if current_asset
+      end
+
+      # Sort remaining versions from newest to oldest
+      versions_to_delete = versions_to_delete.sort_by(&:last_modified).reverse
+
+      # If the asset has been completely removed from our set of locally compiled assets
+      # then use removed_at timestamp from manifest and removed_ttl to determine if it
+      # should be deleted from remote host.
+      # Otherwise, use version_ttl and version_limit to dermine whether version should be kept.
+      versions_to_delete = versions_to_delete.each_with_index.drop_while do |version, index|
+        if !current_asset
+          if (removed_at = removal_manifest[version.path])
+            removed_at = Time.parse(removed_at)
+            removed_age = Time.now.utc - removed_at
+            log "Determining how long ago #{version.path} was removed - removed on #{removed_at} (#{removed_age} seconds ago)."
+            drop = removed_age < removed_ttl
+            log "Marking removed asset #{version.path} for deletion." unless drop
+            removal_manifest.delete(version.path) unless drop || dry_run
+            drop
+          else
+            log "Adding #{version.path} to removal manifest."
+            removed_at = Time.now.utc
+            removal_manifest[version.path] = removed_at.iso8601 unless dry_run
+            true
+          end
+        else
+          # Keep if under age or within the version_limit
+          version_age = [0, Time.now - version.last_modified].max
+          drop = version_age < version_ttl || index < version_limit
+          log "Marking #{version.path} for deletion. Version age: #{version_age}. Version: #{index + 1}." unless drop
+          drop
+        end
+      end.map(&:first)
+
+      s3_keys_to_delete += versions_to_delete.map(&:path)
+    end
+
+    unless dry_run
+      delete_objects(s3_keys_to_delete)
+      removal_manifest.save
+    end
+
+    remote_asset_collector.clear_cache
+    s3_keys_to_delete
+  end
+
+  def deploy(version_limit: 2, version_ttl: 3600, removed_ttl: 172800, clean: true, dry_run: false)
+    upload(dry_run: dry_run)
+    yield if block_given?
+    if clean
+      clean(
+        dry_run: dry_run,
+        version_limit: version_limit,
+        version_ttl: version_ttl,
+        removed_ttl: removed_ttl
+      )
+    end
+  end
+
+  def to_s
+    "#<#{self.class.name}:#{"0x0000%x" % (object_id << 1)} @bucket_name='#{bucket_name}'>"
+  end
+
+  def inspect
+    to_s
+  end
+
+  protected
+
+  def upload_asset(asset)
+    file_handle = File.open(asset.full_path)
+
+    params = {
+      bucket: bucket_name,
+      key: asset.path,
+      body: file_handle,
+      acl: "public-read",
+      content_type: asset.mime_type,
+      cache_control: "public, max-age=31536000"
+    }.merge(@upload_options)
+
+    put_object(params)
+  ensure
+    file_handle.close
+  end
+
+  def s3
+    @s3 ||= Aws::S3::Client.new(@s3_client_options)
+  end
+
+  def verify_no_duplicate_assets!
+    if local_asset_collector.original_asset_paths.uniq.length != local_asset_collector.asset_paths.length
+      raise S3AssetDeploy::DuplicateAssetsError
+    end
+  end
+
+  def put_object(object)
+    s3.put_object(object)
+  end
+
+  def delete_objects(keys = [])
+    return if keys.empty?
+    s3.delete_objects(
+      bucket: bucket_name,
+      delete: { objects: keys.map { |key| { key: key }} }
+    )
+  end
+
+  def log(msg)
+    logger.info("#{self.class.name}: #{msg}")
+  end
+end

data/lib/s3_asset_deploy/rails_local_asset.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "s3_asset_deploy/local_asset"
+
+class S3AssetDeploy::RailsLocalAsset < S3AssetDeploy::LocalAsset
+  attr_reader :path
+
+  def full_path
+    File.join(public_path, path)
+  end
+
+  protected
+
+  def public_path
+    ::Rails.public_path
+  end
+end

data/lib/s3_asset_deploy/rails_local_asset_collector.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "s3_asset_deploy/rails_local_asset"
+require "s3_asset_deploy/local_asset_collector"
+
+class S3AssetDeploy::RailsLocalAssetCollector < S3AssetDeploy::LocalAssetCollector
+  def assets
+    assets_from_manifest + pack_assets
+  end
+
+  def assets_from_manifest
+    manifest = ::Sprockets::Manifest.new(
+      ::ActionView::Base.assets_manifest.environment,
+      ::ActionView::Base.assets_manifest.dir
+    )
+    manifest.assets.values.map do |f|
+      S3AssetDeploy::RailsLocalAsset.new(
+        File.join(assets_prefix, f),
+        remove_fingerprint: @remove_fingerprint
+      )
+    end
+  end
+
+  def pack_assets
+    return [] unless defined?(::Webpacker)
+
+    Dir.chdir(public_path) do
+      packs_dir = ::Webpacker.config.public_output_path.relative_path_from(public_path)
+
+      Dir[File.join(packs_dir, "/**/**")]
+        .select { |path| File.file?(path) }
+        .reject { |path| path.ends_with?(".gz") || path.ends_with?("manifest.json") }
+        .map { |path| S3AssetDeploy::RailsLocalAsset.new(path, remove_fingerprint: @remove_fingerprint) }
+    end
+  end
+
+  private
+
+  def public_path
+    ::Rails.public_path
+  end
+
+  def assets_prefix
+    ::Rails.application.config.assets.prefix.sub(/^\//, "")
+  end
+end

data/lib/s3_asset_deploy/remote_asset.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "s3_asset_deploy/asset_helper"
+
+class S3AssetDeploy::RemoteAsset
+  attr_reader :s3_object
+
+  def initialize(s3_object, remove_fingerprint: nil)
+    @s3_object = s3_object
+    @remove_fingerprint = remove_fingerprint
+  end
+
+  def original_path
+    @original_path ||=
+      if @remove_fingerprint
+        @remove_fingerprint.call(path)
+      else
+        S3AssetDeploy::AssetHelper.remove_fingerprint(path)
+      end
+  end
+
+  def last_modified
+    s3_object.last_modified
+  end
+
+  def path
+    s3_object.key
+  end
+
+  def ==(other_asset)
+    path == other_asset.path
+  end
+
+  def to_s
+    path
+  end
+end

data/lib/s3_asset_deploy/remote_asset_collector.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "aws-sdk-s3"
+require "s3_asset_deploy/removal_manifest"
+require "s3_asset_deploy/remote_asset"
+
+class S3AssetDeploy::RemoteAssetCollector
+  attr_reader :bucket_name
+
+  def initialize(bucket_name, s3_client_options: {}, remove_fingerprint: nil)
+    @bucket_name = bucket_name
+    @remove_fingerprint = remove_fingerprint
+    @s3_client_options = {
+      region: "us-east-1",
+      logger: @logger
+    }.merge(s3_client_options)
+  end
+
+  def s3
+    @s3 ||= Aws::S3::Client.new(@s3_client_options)
+  end
+
+  def assets
+    @assets ||= s3.list_objects_v2(bucket: bucket_name).each_with_object([]) do |response, array|
+      remote_assets = response
+        .contents
+        .reject { |obj| obj.key == S3AssetDeploy::RemovalManifest::PATH }
+        .map do |obj|
+          S3AssetDeploy::RemoteAsset.new(obj, remove_fingerprint: @remove_fingerprint)
+        end
+
+      array.concat(remote_assets)
+    end
+  end
+
+  def clear_cache
+    @assets = nil
+  end
+
+  def asset_paths
+    assets.map(&:path)
+  end
+
+  def grouped_assets
+    assets.group_by(&:original_path)
+  end
+
+  def to_s
+    "#<#{self.class.name}:#{"0x0000%x" % (object_id << 1)} @bucket_name='#{bucket_name}'>"
+  end
+
+  def inspect
+    to_s
+  end
+end

data/lib/s3_asset_deploy/removal_manifest.rb

@@ -0,0 +1,99 @@
+require "json"
+
+class S3AssetDeploy::RemovalManifest
+  attr_reader :bucket_name
+
+  PATH = "s3-asset-deploy-removal-manifest.json".freeze
+
+  def initialize(bucket_name, s3_client_options: {})
+    @bucket_name = bucket_name
+    @loaded = false
+    @changed = false
+    @manifest = {}
+    @s3_client_options = {
+      region: "us-east-1",
+      logger: @logger
+    }.merge(s3_client_options)
+  end
+
+  def s3
+    @s3 ||= Aws::S3::Client.new(@s3_client_options)
+  end
+
+  def load
+    return true if loaded?
+    @manifest = fetch_manifest
+    @loaded = true
+  rescue Aws::S3::Errors::NoSuchKey
+    @manifest = {}
+    @loaded = true
+  end
+
+  def loaded?
+    @loaded
+  end
+
+  def changed?
+    @changed
+  end
+
+  def save
+    return false unless loaded?
+    return true unless changed?
+
+    s3.put_object({
+      bucket: bucket_name,
+      key: PATH,
+      body: @manifest.to_json,
+      acl: "private",
+      content_type: "application/json"
+    })
+
+    @changed = false
+
+    true
+  end
+
+  def keys
+    @manifest.keys
+  end
+
+  def delete(key)
+    return unless loaded?
+    @changed = true
+    @manifest.delete(key)
+  end
+
+  def [](key)
+    @manifest[key]
+  end
+
+  def []=(key, value)
+    return unless loaded?
+    @changed = true
+    @manifest[key] = value
+  end
+
+  def to_h
+    @manifest
+  end
+
+  def to_s
+    @manifest.to_s
+  end
+
+  def inspect
+    "#<#{self.class.name}:#{"0x0000%x" % (object_id << 1)} @bucket_name='#{bucket_name}'>"
+  end
+
+  private
+
+  def fetch_manifest
+    resp = s3.get_object({
+      bucket: bucket_name,
+      key: PATH
+    })
+
+    JSON.parse(resp.body.read)
+  end
+end

data/lib/s3_asset_deploy/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module S3AssetDeploy
+  VERSION = "0.1.0"
+end

data/s3_asset_deploy.gemspec

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "lib/s3_asset_deploy/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "s3_asset_deploy"
+  spec.version       = S3AssetDeploy::VERSION
+  spec.authors       = ["Loomly"]
+  spec.email         = ["contact@loomly.com"]
+
+  spec.summary       = "Safely deploy web app assets to S3 during rolling or multi-step deploys."
+  spec.homepage      = "https://www.loomly.com"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/Loomly/s3_asset_deploy"
+  spec.metadata["changelog_uri"] = "https://github.com/Loomly/s3_asset_deploy/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "aws-sdk-s3", "~> 1.0"
+  spec.add_dependency "mime-types", "~> 3.0"
+
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "timecop", "~> 0.9"
+  spec.add_development_dependency "pry", "~> 0.13"
+  spec.add_development_dependency "pry-byebug", "~> 3.9"
+  spec.add_development_dependency "rspec_junit_formatter", "~> 0.4"
+
+  # For more information and examples about making a new gem, checkout our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

metadata

@@ -0,0 +1,183 @@
+--- !ruby/object:Gem::Specification
+name: s3_asset_deploy
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Loomly
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2021-02-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+description:
+email:
+- contact@loomly.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rspec"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/s3_asset_deploy.rb
+- lib/s3_asset_deploy/asset_helper.rb
+- lib/s3_asset_deploy/errors.rb
+- lib/s3_asset_deploy/local_asset.rb
+- lib/s3_asset_deploy/local_asset_collector.rb
+- lib/s3_asset_deploy/manager.rb
+- lib/s3_asset_deploy/rails_local_asset.rb
+- lib/s3_asset_deploy/rails_local_asset_collector.rb
+- lib/s3_asset_deploy/remote_asset.rb
+- lib/s3_asset_deploy/remote_asset_collector.rb
+- lib/s3_asset_deploy/removal_manifest.rb
+- lib/s3_asset_deploy/version.rb
+- s3_asset_deploy.gemspec
+homepage: https://www.loomly.com
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://www.loomly.com
+  source_code_uri: https://github.com/Loomly/s3_asset_deploy
+  changelog_uri: https://github.com/Loomly/s3_asset_deploy/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key:
+specification_version: 4
+summary: Safely deploy web app assets to S3 during rolling or multi-step deploys.
+test_files: []