s3_website 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46287f622fae5f7d6dd707ce8fbdd617060b46ae
-  data.tar.gz: 5f3d2f1c19f9c16695ce896ac52e5c9e6ae35d34
+  metadata.gz: 9dd48d8a609cc024fe3ecd567bcd390077b07f1d
+  data.tar.gz: 1fd28bdff75aaed3f3a6d3590ae8258a05d9af97
 SHA512:
-  metadata.gz: 06835789338deb7cb5b93f173d9c840fb8a0d52683a059a5e56c9d13c596668155956cc21d11995010939e15ecaf1cdeb52b41ecbc8b85b8cf02d5519a455d37
-  data.tar.gz: 7212c008b7d0b6e90d89c04b0d736ef427f09e97312f5a7e1514bcfd17effd7f2fe5911b5fa53935f144d752de01232c641678e21278ddad4d2daf8708b567a5
+  metadata.gz: 5cf5a79c29bab7b6c2820e80c6068f4aa7cbf2fde9936310eb3a16dabdbcfc0da294d5e38125d1929dd17b485a3dd51cfc18dad802fec520a4f710e090198c07
+  data.tar.gz: 62ac09a5a98f025a894a9fbfe83b66d7ad6a2519e78ab5c2df77b832ec449c98db52b777bfc832f31f7d22699d9638601941abdfc8acb92a91ecd6f3538f2a38

data/README.md

@@ -52,6 +52,10 @@ If you want to store the `s3_website.yml` file in a directory other than
 the project's root you can specify the directory like so:
 `s3_website push --config-dir config`.
 
+### Using standard AWS credentials
+
+If you omit `s3_id` from your `s3_website.yml`, S3_website will fall back to reading from the [default AWS SDK locations](http://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html). For instance, if you've used `aws configure` to set up credentials in `~/.aws/credentials`, S3_website can use these.
+
 ### Using environment variables
 
 You can use ERB in your `s3_website.yml` file which incorporates environment variables:
@@ -77,6 +81,8 @@ Your `.env` file should containing the following variables:
     S3_ID=FOO
     S3_SECRET=BAR
 
+S3_website will also honor environment variables named `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` (if using STS) automatically if `s3_id` is ommitted from `s3_website.yml`.
+
 ## Project goals
 
 * Provide a command-line interface tool for deploying and managing S3 websites
@@ -280,7 +286,7 @@ thus force the CDN system to reload the changes from your website S3 bucket.
 
 `s3_website` lets you define custom settings for your CloudFront distribution.
 
-For example, like this you can define a your own TTL and CNAME:
+For example, like this you can define your own TTL and CNAME:
 
 ```yaml
 cloudfront_distribution_config:
@@ -289,7 +295,7 @@ cloudfront_distribution_config:
   aliases:
     quantity: 1
     items:
-      - your.website.com
+      CNAME: your.website.com
 ```
 
 Once you've saved the configuration into `s3_website.yml`, you can apply them by
@@ -333,27 +339,30 @@ index file, your source bucket in Cloudfront likely is pointing to the S3 Origin
 
 ### Configuring redirects on your S3 website
 
-You can set HTTP redirects on your S3 website in two ways. If you only need
-simple "301 Moved Premanently" redirects for certain keys, use the Simple
-Redirects method. Otherwise, use the Routing Rules method.
+You can set HTTP redirects on your S3 website in three ways. 
 
-#### Simple Redirects
+#### Exact page match for moving a single page
+If a request is received matching a string e.g. /heated-towel-rail/ redirect to a page e.g. /  
 
-For simple redirects `s3_website` uses Amazon S3's
-[`x-amz-website-redirect-location`](http://docs.aws.amazon.com/AmazonS3/latest/dev/how-to-page-redirect.html)
-metadata. It will create zero-byte objects for each path you want
-redirected with the appropriate `x-amz-website-redirect-location` value.
-
-For setting up simple redirect rules, simply list each path and target
-as key-value pairs under the `redirects` configuration option:
+This kind of redirect is created in the s3_website.yml file under the ```redirects:``` section as follows: 
 
 ```yaml
 redirects:
   index.php: /
   about.php: /about.html
   music-files/promo.mp4: http://www.youtube.com/watch?v=dQw4w9WgXcQ
+  heated-towel-rail/index.html: /
+  
 ```
 
+Note that the root forward slash is omitted in the requested page path and included in the redirect-to path.  Note also that in the heated-towel-rail example this also matches heated-towel-rail/ since S3 appends index.html to request URLs terminated with a slash. 
+
+This redirect will be created as a 301 redirect from the first URL to the destination URL on the same server with the same http protocol.
+
+Under the hood `s3_website` creates a zero-byte index.html page for each path you want redirected with the appropriate `x-amz-website-redirect-location` value in the metadata for the object. See Amazon S3's
+[`x-amz-website-redirect-location`](http://docs.aws.amazon.com/AmazonS3/latest/dev/how-to-page-redirect.html)
+documentation for more details.
+
 On terminology: the left value is the redirect source and the right value is the redirect
 target. For example above, *about.php* is the redirect source and */about.html* the target.
 
@@ -362,21 +371,31 @@ target if and only if the redirect target points to a site-local resource and
 does not start with a slash. E.g., `about.php: about.html` will be translated
 into `about.php: VALUE-OF-S3_KEY_PREFIX/about.html`.
 
-#### Routing Rules
+#### Prefix replacement for moving a folder of pages
+Common to content migrations, content pages often move from one subdirectory to another. For example if you're moving all the case studies on your site under /portfolio/work/ to /work/. In this case we use a prefix replacement such that /portfolio/work/walkjogrun/ gets 301 redirected to /work/walkjogrun/.
 
-You can configure more complex redirect rules by adding the following
-configuration into the `s3_website.yml` file:
+To do this we add a new rule to the routing_rules: section as follows:
 
-```yaml
-routing_rules:
+```
   - condition:
-      key_prefix_equals: blog/some_path
+        key_prefix_equals: portfolio/work/
     redirect:
-      host_name: blog.example.com
-      replace_key_prefix_with: some_new_path/
-      http_redirect_code: 301
+        protocol: https
+        host_name: <%= ENV['REDIRECT_DOMAIN_NAME'] %>
+        replace_key_prefix_with: work/
+        http_redirect_code: 301
 ```
 
+Here:
+
+* ```-condition:``` indicates the start of a new rule. 
+* ```key_prefix_equals:``` introduces the path prefix (also without the leading / per the exact page match). Note that this prefix matches anything underneath it so every case study under that path will be handled by the subsequent redirect
+* ```redirect:``` indicates the start of the redirect definition 
+* ```protocol:``` is optional and defaults to http.
+* ```host_name:``` is optional but the default is the amazonaws.com bucket name not the actual domain name so this also effectively required for our site. In this example we use an environment variable to store the server hostname to support building to different environments. ```REDIRECT_DOMAIN_NAME``` can be configured on a CI server as well any CodePipelines responsible for building the site to different environments.  If you're running locally you'll need to set ```REDIRECT_DOMAIN_NAME=local.myhostname.com```
+* ```replace_key_prefix_with:``` indicates the substitution to use in place of the matched prefix. This is the only field required by `s3_website`, so effectively this rule works like a replace rule e.g. replace portfolio/work with /work in the string portfolio/work/walkjogrun
+* ```http_redirect_code:``` is optional and defaults to 302 Temporary redirect **which is terrible for SEO** since your content temporarily vanishes from the Google index until the response changes for the URL. This is almost never what you want. You *can* use this to temporarily redirect any content you haven't migrated to the new site yet as long as you remove or replace the 302 with a link to a permanent home. This tells Google to forget the old location of the page and use the new content at the new URL. For pages that move you'll see little if any discrepancy in Google traffic. 
+
 After adding the configuration, run the command `s3_website cfg apply` on your
 command-line interface. This will apply the routing rules on your S3 bucket.
 
@@ -385,6 +404,25 @@ For more information on configuring redirects, see the documentation of the
 gem, which comes as a transitive dependency of the `s3_website` gem. (The
 command `s3_website cfg apply` internally calls the `configure-s3-website` gem.)
 
+#### Prefix coallescing for deleting pages (or consolidating)
+If you 301 redirect lots of content into one new path you're telling Google that the old pages are gone so only the destination page is important moving forward. E.g. if you had 10 services pages and consolidate them into 1 services listing page you'll lose the 10 pages uniquely optimized for different sets of keywords and retain just 1 page with no real keyword focus and hence less SEO value.
+
+For example, we're not porting the entire set of pages under the folder /experience to the new website. Some of these pages still get traffic from either Google or inbound links so we don't want to just show a 404 content not found error. We will 301 redirect them to the most useful replacement page. In the case of /experience we don't have anything better to show than just the home page so that is how the redirect is configured.
+
+Here's how to redirect to indicate a deleted page:
+
+```
+  - condition:
+        key_prefix_equals: experience/
+    redirect:
+        protocol: https
+        host_name: <%= ENV['REDIRECT_DOMAIN_NAME'] %>
+        **replace_key_with**: /
+        http_redirect_code: 301
+```
+
+Note the only difference is that instead of using ```replace_key_prefix_with``` we use ```replace_key_with``` to effectively say "replace the entire path matching the prefix specfied in the condition with the new path".
+
 #### On skipping application of redirects
 
 If your website has a lot of redirects, you may find the following setting

data/additional-docs/example-configurations.md

@@ -5,19 +5,22 @@ This document shows examples of complete `s3_website.yml` configurations.
 ## Minimal
 
 ````yaml
-s3_id: abcd
-s3_secret: 2s+x92
 s3_bucket: your.domain.net
 ````
 
-## Minimal with EC2 IAM roles
+This configuration will use AWS access credentials from the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. If those are not set, it will fall back to the credentials saved by `aws configure`.
+
+If you run `s3_website` on an EC2 instance with IAM roles, this configuration will use the instance's role instead.
+
+## Minimal with explicit credentials
 
 ````yaml
+s3_id: abcd
+s3_secret: 2s+x92
 s3_bucket: your.domain.net
 ````
 
-If you run `s3_website` on an EC2 instance with IAM roles, it is possible to omit
-the `s3_id` and `s3_secret`.
+Use caution when embedding AWS credentials directly in `s3_website.yml`. Do not commit the file to a public Git repository or share it publicly.
 
 ## Minimal for temporary security credentials
 
@@ -33,8 +36,6 @@ s3_bucket: your.domain.net
 Use CloudFront, gzip, cache headers and greater concurrency:
 
 ````yaml
-s3_id: <%= ENV['your_domain_net_aws_key'] %>
-s3_secret: <%= ENV['your_domain_net_aws_secret'] %>
 s3_bucket: your.domain.net
 cloudfront_distribution_id: <%= ENV['your_domain_net_cloudfront_distribution_id'] %>
 cloudfront_distribution_config:
@@ -48,10 +49,9 @@ max_age: 120
 gzip: true
 ````
 
-Above, we store the AWS credentials and the id of the CloudFront distribution as
-environment variables. It's convenient, since you can keep the `s3_website.yml`
-in a public Git repo, and thus have your deployment configurations
-version-controlled.
+In this example, we keep the CloudFront distribution ID in an environment variable.
+This is convenient, since you can keep the `s3_website.yml` in a public Git repo, and
+thus have your deployment configurations version-controlled.
 
 ## Setup for HTTP2 and Custom SNI SSL Certificate
 
@@ -65,8 +65,6 @@ This isn't yet automated by s3_website, [but is a few manual steps](https://medi
 which is now free thanks to Let's Encrypt. 
 
 ````yaml
-s3_id: <%= ENV['your_domain_net_aws_key'] %>
-s3_secret: <%= ENV['your_domain_net_aws_secret'] %>
 s3_bucket: your.domain.net
 cloudfront_distribution_id: <%= ENV['your_domain_net_cloudfront_distribution_id'] %>
 cloudfront_distribution_config:
@@ -82,8 +80,6 @@ gzip: true
 Sometimes you want to use multiple CNAMEs aliases in your CloudFront distribution:
 
 ````yaml
-s3_id: <%= ENV['your_domain_net_aws_key'] %>
-s3_secret: <%= ENV['your_domain_net_aws_secret'] %>
 s3_bucket: your.domain.net
 cloudfront_distribution_id: <%= ENV['your_domain_net_cloudfront_distribution_id'] %>
 cloudfront_distribution_config:
@@ -104,8 +100,6 @@ Always remember to set the 'quantity' property to match the number of items you
 ## Using redirects
 
 ````yaml
-s3_id: hello
-s3_secret: galaxy
 redirects:
   index.php: /
   about.php: about.html

data/additional-docs/running-from-ec2-with-dropbox.md

@@ -0,0 +1,6 @@
+# Running from an EC2 instance with jekyll files on Dropbox
+
+Based on [this](http://namelesshorror.com/2015/02/26/jekyll-dropbox-aws-and-ifttt-easy-blogging/) article about automating the deployment of jekyll via an EC2 instance, I wrote the following shell script for use with such a setup. It assumes that you have a Linux Amazon EC2 instance, with the Dropbox client running as a daemon, and s3_website installed and configured. The script could be run from the default user's cron every so often, and allow anyone to effectively serve their jekyll source files to AWS from Dropbox.
+
+
+[jekyll-s3-dropbox.sh](https://gist.github.com/RNCTX/359489a5432937578bf5736850917d70)

data/changelog.md

@@ -2,6 +2,10 @@
 
 This project uses [Semantic Versioning](http://semver.org).
 
+## 3.2.0
+
+* Fall back to [the default credentials sources](http://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html) if `s3_id` is not provided in `s3_website.yml`
+
 ## 3.1.0
 
 Support for [session tokens](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)

data/lib/s3_website/version.rb

@@ -1,3 +1,3 @@
 module S3Website
-  VERSION = '3.1.0'
+  VERSION = '3.2.0'
 end

data/resources/configuration_file_template.yml

@@ -1,9 +1,12 @@
+# You can remove these two lines to have credentials read from
+# the environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY or
+# ~/.aws/credentials.
 s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
 s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY
 s3_bucket: your.blog.bucket.com
 
-# set s3_token if using temporary credentials with a session token (eg: when assuming a role)
-# s3_token: YOUR_AWS_S3_SESSION_TOKEN
+# set session_token if using temporary credentials with a session token (eg: when assuming a role)
+# session_token: YOUR_AWS_S3_SESSION_TOKEN
 
 # Below are examples of all the available configurations.
 # See README for more detailed info on each of them.

data/s3_website.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.default_executable = %q{s3_website}
 
   s.add_dependency 'thor', '~> 0.18'
-  s.add_dependency 'configure-s3-website', '= 2.0.0'
+  s.add_dependency 'configure-s3-website', '= 2.1.0'
   s.add_dependency 'colored', '1.2'
   s.add_dependency 'dotenv', '~> 1.0'
 

data/src/main/scala/s3/website/model/Config.scala

@@ -8,7 +8,7 @@ import scala.util.{Failure, Try}
 import scala.collection.JavaConversions._
 import s3.website.Ruby.rubyRuntime
 import s3.website._
-import com.amazonaws.auth.{AWSCredentialsProvider, BasicAWSCredentials, BasicSessionCredentials, DefaultAWSCredentialsProviderChain}
+import com.amazonaws.auth.{AWSCredentialsProvider, BasicAWSCredentials, BasicSessionCredentials, AWSStaticCredentialsProvider, DefaultAWSCredentialsProviderChain}
 
 case class Config(
   s3_id:                                  Option[String], // If undefined, use IAM Roles (http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-roles.html)
@@ -37,25 +37,22 @@ case class Config(
 object Config {
 
   def awsCredentials(config: Config): AWSCredentialsProvider = {
-    val credentialsFromConfigFile =
-      if (config.session_token.isEmpty) {
+    val credentialsFromConfigFile: Option[AWSStaticCredentialsProvider] =
+      if (config.s3_id.isEmpty) {
+         None
+      } else if (config.session_token.isEmpty) {
         for {
           s3_id <- config.s3_id
           s3_secret <- config.s3_secret
-        } yield new BasicAWSCredentials(s3_id, s3_secret)
+        } yield new AWSStaticCredentialsProvider(new BasicAWSCredentials(s3_id, s3_secret))
       } else {
         for {
           s3_id <- config.s3_id
           s3_secret <- config.s3_secret
           session_token <- config.session_token
-        } yield new BasicSessionCredentials(s3_id, s3_secret, session_token)
+        } yield new AWSStaticCredentialsProvider(new BasicSessionCredentials(s3_id, s3_secret, session_token))
       }
-    credentialsFromConfigFile.fold(new DefaultAWSCredentialsProviderChain: AWSCredentialsProvider)(credentials =>
-      new AWSCredentialsProvider {
-        def getCredentials = credentials
-        def refresh() = {}
-      }
-    )
+    credentialsFromConfigFile getOrElse new DefaultAWSCredentialsProviderChain
   }
 
   def loadOptionalBooleanOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Boolean, Seq[String]]]] = {

data/src/test/scala/s3/website/ConfigSpec.scala

@@ -0,0 +1,87 @@
+package s3.website
+import com.amazonaws.auth.{BasicAWSCredentials, BasicSessionCredentials, DefaultAWSCredentialsProviderChain}
+import org.specs2.mutable.Specification
+import s3.website.model.{Config, S3Endpoint}
+
+class ConfigSpec extends Specification {
+
+  "Config#awsCredentials" should {
+    s"return ${classOf[BasicAWSCredentials]} when s3_id and s3_secret are defined in the config" in {
+      Config.awsCredentials(Config(
+        s3_id = Some("test"),
+        s3_secret = Some("secret"),
+        session_token = None,
+        s3_bucket = "foo",
+        s3_endpoint = S3Endpoint.defaultEndpoint,
+        site = None,
+        max_age = None,
+        cache_control = None,
+        gzip = None,
+        gzip_zopfli = None,
+        s3_key_prefix = None,
+        ignore_on_server = None,
+        exclude_from_upload = None,
+        s3_reduced_redundancy = None,
+        cloudfront_distribution_id = None,
+        cloudfront_invalidate_root = None,
+        content_type = None,
+        redirects = None,
+        concurrency_level = 1,
+        cloudfront_wildcard_invalidation = None,
+        treat_zero_length_objects_as_redirects = None
+      )).getCredentials must beAnInstanceOf[BasicAWSCredentials]
+    }
+
+    s"return ${classOf[BasicSessionCredentials]} when s3_id, s3_secret and session_token are defined in the config" in {
+      Config.awsCredentials(Config(
+        s3_id = Some("test"),
+        s3_secret = Some("secret"),
+        session_token = Some("Token"),
+        s3_bucket = "foo",
+        s3_endpoint = S3Endpoint.defaultEndpoint,
+        site = None,
+        max_age = None,
+        cache_control = None,
+        gzip = None,
+        gzip_zopfli = None,
+        s3_key_prefix = None,
+        ignore_on_server = None,
+        exclude_from_upload = None,
+        s3_reduced_redundancy = None,
+        cloudfront_distribution_id = None,
+        cloudfront_invalidate_root = None,
+        content_type = None,
+        redirects = None,
+        concurrency_level = 1,
+        cloudfront_wildcard_invalidation = None,
+        treat_zero_length_objects_as_redirects = None
+      )).getCredentials must beAnInstanceOf[BasicSessionCredentials]
+    }
+
+    s"return ${classOf[DefaultAWSCredentialsProviderChain]} when s3_id and s3_secret are not defined in the config" in {
+      Config.awsCredentials(Config(
+        s3_id = None,
+        s3_secret = None,
+        session_token = None,
+        s3_bucket = "foo",
+        s3_endpoint = S3Endpoint.defaultEndpoint,
+        site = None,
+        max_age = None,
+        cache_control = None,
+        gzip = None,
+        gzip_zopfli = None,
+        s3_key_prefix = None,
+        ignore_on_server = None,
+        exclude_from_upload = None,
+        s3_reduced_redundancy = None,
+        cloudfront_distribution_id = None,
+        cloudfront_invalidate_root = None,
+        content_type = None,
+        redirects = None,
+        concurrency_level = 1,
+        cloudfront_wildcard_invalidation = None,
+        treat_zero_length_objects_as_redirects = None
+      )) must beAnInstanceOf[DefaultAWSCredentialsProviderChain]
+    }
+  }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_website
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-08 00:00:00.000000000 Z
+date: 2017-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: colored
   requirement: !ruby/object:Gem::Requirement
@@ -126,6 +126,7 @@ files:
 - additional-docs/debugging.md
 - additional-docs/development.md
 - additional-docs/example-configurations.md
+- additional-docs/running-from-ec2-with-dropbox.md
 - additional-docs/setting-up-aws-credentials.md
 - assembly.sbt
 - bin/s3_website
@@ -156,6 +157,7 @@ files:
 - src/main/scala/s3/website/model/ssg.scala
 - src/main/scala/s3/website/package.scala
 - src/test/scala/s3/website/AwsSdkSpec.scala
+- src/test/scala/s3/website/ConfigSpec.scala
 - src/test/scala/s3/website/S3EndpointSpec.scala
 - src/test/scala/s3/website/S3WebsiteSpec.scala
 - src/test/scala/s3/website/UnitTest.scala
@@ -186,6 +188,7 @@ specification_version: 4
 summary: Manage your S3 website
 test_files:
 - src/test/scala/s3/website/AwsSdkSpec.scala
+- src/test/scala/s3/website/ConfigSpec.scala
 - src/test/scala/s3/website/S3EndpointSpec.scala
 - src/test/scala/s3/website/S3WebsiteSpec.scala
 - src/test/scala/s3/website/UnitTest.scala