s3_asset_deploy 0.1.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 72c4c1220b3051d9189389b3b956d509506380ae4bef112212480e1da9ac75e4
-  data.tar.gz: b9fbd6e07c1cced6e5a20d1cb72338093eec260a540a434ee3cc60012e680549
+  metadata.gz: 3590e77179d747a7737ad4032877bc1e2f2a219d58005711f70ff737cf107fb5
+  data.tar.gz: cfc9de4221ce547ddc818d1569b70b3c5ffb6184d79108eec3145534a2361304
 SHA512:
-  metadata.gz: cb3a963f7a47375414e9b51b1b3f525a890924995b380d193151e9a77193a7ddec8f1f50864fc60b37c5c768586fefa76b1335d3e849918572ec4b7be3af866a
-  data.tar.gz: d240fe42ae5578cf136f1dac4b637165603836f8cdceb7bf8d820cd6808048d78b50b4dbbe235f3e1678aedbe8f36a6b76f1310a5d64f0605622650cca2d01b1
+  metadata.gz: 4a91acae364734b965b2c1176eee978b23fe4244c0ad4c89993f2a52f528a8d2efe6d7652daf4ca50f2e473eab5d9bb153c5af04f1000e3731e3294d0b5737c0
+  data.tar.gz: d1a6dfd9164072da6aa1acbd5a36e280508e291d525a907b9500e5af76e0b07816da5dc6a60ff6d4dba5d95ddc3e891da44627a526189ebb56d723a251063c9c

data/CHANGELOG.md

@@ -1 +1,11 @@
 # Changelog
+
+## [v1.0.1](https://github.com/Loomly/s3_asset_deploy/compare/v1.0.0...v1.0.1) - 2022-02-23
+- Batch delete API calls with max 1000 keys - [PR #32](https://github.com/Loomly/s3_asset_deploy/pull/32)
+
+## [v1.0.0](https://github.com/Loomly/s3_asset_deploy/compare/v0.1.1...v1.0.0) - 2021-05-13
+### Breaking Changes
+- Remove default `acl` setting when uploading assets to bucket - [PR #25](https://github.com/Loomly/s3_asset_deploy/pull/25)
+
+## [v0.1.1](https://github.com/Loomly/s3_asset_deploy/compare/v0.1.0...v0.1.1) - 2021-03-22
+- Fix bug in AssetHelper.remove_fingerprint referencing asset_path - [4f370ad](https://github.com/Loomly/s3_asset_deploy/commit/4f370ad9c0c1c274acb9b1d8585b878f47020277)

data/Gemfile.lock

@@ -1,43 +1,48 @@
 PATH
   remote: .
   specs:
-    s3_asset_deploy (0.1.0)
+    s3_asset_deploy (1.0.1)
       aws-sdk-s3 (~> 1.0)
       mime-types (~> 3.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-eventstream (1.1.0)
-    aws-partitions (1.422.0)
-    aws-sdk-core (3.111.2)
+    aws-eventstream (1.2.0)
+    aws-partitions (1.558.0)
+    aws-sdk-core (3.126.2)
       aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.239.0)
+      aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.41.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-kms (1.54.0)
+      aws-sdk-core (~> 3, >= 3.126.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.87.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-s3 (1.112.0)
+      aws-sdk-core (~> 3, >= 3.126.0)
       aws-sdk-kms (~> 1)
-      aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.2.2)
+      aws-sigv4 (~> 1.4)
+    aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
     byebug (11.1.3)
     coderay (1.1.3)
     diff-lcs (1.4.4)
-    jmespath (1.4.0)
+    jmespath (1.6.0)
     method_source (1.0.0)
-    mime-types (3.3.1)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.1104)
+    mime-types-data (3.2022.0105)
+    mini_portile2 (2.8.0)
+    nokogiri (1.13.3)
+      mini_portile2 (~> 2.8.0)
+      racc (~> 1.4)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     pry-byebug (3.9.0)
       byebug (~> 11.0)
       pry (~> 0.13.0)
+    racc (1.6.0)
     rake (13.0.3)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -60,6 +65,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  nokogiri (~> 1.13)
   pry (~> 0.13)
   pry-byebug (~> 3.9)
   rake (~> 13.0)
@@ -69,4 +75,4 @@ DEPENDENCIES
   timecop (~> 0.9)
 
 BUNDLED WITH
-   2.2.7
+   2.2.32

data/README.md

@@ -1,6 +1,7 @@
 # S3AssetDeploy
 
 [![CircleCI](https://circleci.com/gh/Loomly/s3_asset_deploy.svg?style=shield)](https://circleci.com/gh/Loomly/s3_asset_deploy)
+[![Gem Version](https://badge.fury.io/rb/s3_asset_deploy.svg)](https://badge.fury.io/rb/s3_asset_deploy)
 
 During rolling deploys to our web instances, this is what we use at
 [Loomly](https://www.loomly.com) to safely deploy our web assets to S3 to be served via Cloudfront.
@@ -41,7 +42,7 @@ Before using `S3AssetDeploy` you want to make sure to compile your assets. Asset
 
 
 ```ruby
-manager = S3AssetDeploy::Manager("my-s3-bucket")
+manager = S3AssetDeploy::Manager.new("my-s3-bucket")
 manager.deploy do
   # Perform deploy to web instances in this block
 end
@@ -60,7 +61,7 @@ You'll need to initialize `S3AssetDeploy::Manager` with an S3 bucket name and **
 - **s3_client_options** (Hash) -> A hash that is passed directly to [`Aws::S3::Client#initialize`](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#initialize-instance_method) to configure the S3 client. By default the region is set to `us-east-1`.
 - **logger** (Logger) -> A custom logger. By default things are logged to `STDOUT`.
 - **local_asset_collector** (S3AssetDeploy::LocalAssetCollector) -> A custom instance of `S3AssetDeploy::LocalAssetCollector`. This allows you to customize how locally compiled assets are collected.
-- **upload_options** (Hash) -> A hash consisting of options that are passed directly to [`Aws::S3::Client#put_object`](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#put_object-instance_method) when each asset is uploaded. By default `acl` is set to `public-read` and `cache_control` is set to `public, max-age=31536000`.
+- **upload_options** (Hash) -> A hash consisting of options that are passed directly to [`Aws::S3::Client#put_object`](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#put_object-instance_method) when each asset is uploaded. By default `cache_control` is set to `public, max-age=31536000`.
 - **remove_fingerprint** (Lambda) -> Lambda for overriding how fingerprints are removed from asset paths. Fingerprints need to be removed during the cleaning process in order to group versions of the same file. If no Lambda is provided, [`S3AssetDeploy::AssetHelper.remove_fingerprint`](https://github.com/Loomly/s3_asset_deploy/blob/main/lib/s3_asset_deploy/asset_helper.rb#L8) is used by default.
 
 Here's an example:
@@ -107,6 +108,46 @@ manager.clean
 `S3AssetDeploy::Manager#deploy` and `S3AssetDeploy::Manager#clean` both accept `dry_run` as a keyword argument.
 `S3AssetDeploy::Manager#clean` also accepts `version_limit`, `version_ttl`, and `removed_ttl` just like `S3AssetDeploy::Manager#deploy`.
 
+### Practical Example of Usage
+There are many ways to use and invoke `S3AssetDeploy`. How you use it will depend on your deploy process and pipeline. At Loomly, we have some rake tasks that are invoked from our CI/CD pipeline to perform deploys.
+Here's a basic example of how we use `S3AssetDeploy`:
+
+
+```ruby
+# lib/tasks/deploy.rake
+
+require "s3_asset_deploy"
+
+namespace :deploy do
+  task precompile: :environment do
+    puts "Precompiling assets..."
+    sh("RAILS_ENV=production SECRET_KEY_BASE='secret key' bundle exec rake assets:precompile")
+  end
+
+  task clobber_assets: :environment do
+    puts "Clobbering assets..."
+    sh("RAILS_ENV=production SECRET_KEY_BASE='secret key' bundle exec rake assets:clobber")
+  end
+
+  task :production do
+    Rake::Task["deploy:precompile"].invoke
+
+    manager = S3AssetDeploy::Manager.new("my-s3-bucket")
+    manager.deploy do
+      # Perform deploy to web instances in this block.
+      # How you do this will depend on where you are hosting your application and what tools you use to deploy.
+    end
+
+    Rake::Task["deploy:clobber_assets"].invoke # <-- If you are running on CI where the precompiled assets directory is ephemeral, this may be unnecessary
+  end
+end
+```
+
+Given the example above, we can perform a deploy by running `bundle exec rake deploy:production`. This task will:
+1. Precompile assets
+2. Upload any new assets to S3 using `S3AssetDeploy`
+3. Deploy a new version of our application
+4. Clean any outdated or unused assets from S3 using `S3AssetDeploy`
 
 ## Customizing local asset collection
 By default, `S3AssetDeploy::Manager` will use [`S3AssetDeploy::RailsLocalAssetCollector`](https://github.com/Loomly/s3_asset_deploy/blob/main/lib/s3_asset_deploy/rails_local_asset_collector.rb) to collect locally compiled assets. This will use the `Sprockets::Manifest` and `Webpacker` config (if `Webpacker` is installed) to locate the compiled assets. `S3AssetDeploy::RailsLocalAssetCollector` inherits from the [`S3AssetDeploy::LocalAssetCollector`](https://github.com/Loomly/s3_asset_deploy/blob/main/lib/s3_asset_deploy/local_asset_collector.rb) base class. You can completely customize how your local assets are collected for deploys by creating your own class that inherits from `S3AssetDeploy::LocalAssetCollector` and passing it into the manager. You'll want override `S3AssetDeploy::LocalAssetCollector#assets` in your custom collector such that it returns an array of `S3AssetDeploy::LocalAsset` instances. Here's a basic example:
@@ -133,7 +174,7 @@ This will skip any write or delete operations and only perform read opeartions w
 This is helpful for debugging or planning purposes.
 
 ```ruby
-> manager = S3AssetDeploy::Manager("my-s3-bucket")
+> manager = S3AssetDeploy::Manager.new("my-s3-bucket")
 > manager.deploy(dry_run: true)
 
 I, [2021-02-17T16:12:23.703677 #65335]  INFO -- : S3AssetDeploy::Manager: Cleaning assets from test-bucket S3 bucket. Dry run: true
@@ -142,7 +183,7 @@ I, [2021-02-17T16:12:23.703677 #65335]  INFO -- : S3AssetDeploy::Manager: Determ
 ```
 
 ## AWS IAM Permissions
-`S3AsetDeploy` requires the following AWS IAM permissions:
+`S3AsetDeploy` requires the following AWS IAM permissions to list, put, and delete objects in your S3 Bucket:
 
 ```json
 "Statement": [
@@ -161,6 +202,65 @@ I, [2021-02-17T16:12:23.703677 #65335]  INFO -- : S3AssetDeploy::Manager: Determ
 ]
 ```
 
+## Configuration with Cloudfront
+
+### Restricting Access with Origin Access Identity
+If you want to setup Cloudfront to serve your assets, you can [restrict access to the bucket by using an Origin Access Identity](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-restricting-access-to-s3.html#private-content-granting-permissions-to-oai) so that only Cloudfront can access the objects in your bucket.
+
+If you do this, your bucket policy will look something like this:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "AllowGetObject",
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                  "arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity #{YOUR_OAI_ID}"
+                ]
+            },
+            "Action": "s3:GetObject",
+            "Resource": "arn:aws:s3:::#{YOUR_BUCKET}/*"
+        },
+        {
+            "Sid": "DenyGetObject",
+            "Effect": "Deny",
+            "Principal": {
+                "AWS": "arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity #{YOUR_OAI_ID}"
+            },
+            "Action": "s3:GetObject",
+            "Resource": "arn:aws:s3:::#{YOUR_BUCKET}/s3-asset-deploy-removal-manifest.json"
+        }
+    ]
+}
+```
+
+This policy allows Cloudfront to access everything **except** the removal manifest uploaded and maintained by this gem since this manifest does not need to be served to clients.
+
+### CORS
+Your CORS configuration on the bucket might look something like this:
+
+```json
+[
+    {
+        "AllowedHeaders": [
+            "Authorization"
+        ],
+        "AllowedMethods": [
+            "GET",
+            "HEAD"
+        ],
+        "AllowedOrigins": [
+            "https://*.#{YOUR_SITE}.com"
+        ],
+        "ExposeHeaders": [],
+        "MaxAgeSeconds": 3000
+    }
+]
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/s3_asset_deploy/asset_helper.rb

@@ -7,7 +7,7 @@ class S3AssetDeploy::AssetHelper
 
   def self.remove_fingerprint(path)
     match_data = path.match(FINGERPRINTED_ASSET_REGEX)
-    return asset_path unless match_data
+    return path unless match_data
     "#{match_data[1]}#{match_data[3]}"
   end
 

data/lib/s3_asset_deploy/manager.rb

@@ -168,7 +168,6 @@ class S3AssetDeploy::Manager
       bucket: bucket_name,
       key: asset.path,
       body: file_handle,
-      acl: "public-read",
       content_type: asset.mime_type,
       cache_control: "public, max-age=31536000"
     }.merge(@upload_options)
@@ -194,10 +193,12 @@ class S3AssetDeploy::Manager
 
   def delete_objects(keys = [])
     return if keys.empty?
-    s3.delete_objects(
-      bucket: bucket_name,
-      delete: { objects: keys.map { |key| { key: key }} }
-    )
+    keys.each_slice(1000) do |key_slice|
+      s3.delete_objects(
+        bucket: bucket_name,
+        delete: { objects: key_slice.map { |key| { key: key }} }
+      )
+    end
   end
 
   def log(msg)

data/lib/s3_asset_deploy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module S3AssetDeploy
-  VERSION = "0.1.0"
+  VERSION = "1.0.1"
 end

data/s3_asset_deploy.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Loomly"]
   spec.email         = ["contact@loomly.com"]
 
-  spec.summary       = "Safely deploy web app assets to S3 during rolling or multi-step deploys."
-  spec.homepage      = "https://www.loomly.com"
+  spec.summary       = "Deploy & manage static assets on S3 with rolling deploys & rollbacks in mind."
+  spec.homepage      = "https://github.com/Loomly/s3_asset_deploy"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
 
@@ -36,6 +36,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry-byebug", "~> 3.9"
   spec.add_development_dependency "rspec_junit_formatter", "~> 0.4"
 
+  # Required for aws-sdk-ruby.
+  # See https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/xml/parser.rb#L74
+  spec.add_development_dependency "nokogiri", "~> 1.13"
+
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_asset_deploy
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Loomly
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-02-27 00:00:00.000000000 Z
+date: 2022-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
 description:
 email:
 - contact@loomly.com
@@ -154,11 +168,11 @@ files:
 - lib/s3_asset_deploy/removal_manifest.rb
 - lib/s3_asset_deploy/version.rb
 - s3_asset_deploy.gemspec
-homepage: https://www.loomly.com
+homepage: https://github.com/Loomly/s3_asset_deploy
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://www.loomly.com
+  homepage_uri: https://github.com/Loomly/s3_asset_deploy
   source_code_uri: https://github.com/Loomly/s3_asset_deploy
   changelog_uri: https://github.com/Loomly/s3_asset_deploy/CHANGELOG.md
 post_install_message:
@@ -176,8 +190,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
-summary: Safely deploy web app assets to S3 during rolling or multi-step deploys.
+summary: Deploy & manage static assets on S3 with rolling deploys & rollbacks in mind.
 test_files: []