s3_website 2.6.1 → 2.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c6bd443ae587cab4726f6ed2e3f11051bb62f7c4
-  data.tar.gz: 4e7410d745fc3a59f26fdc83c16d182a0ba9396f
+  metadata.gz: 778dcd186e3206eaf5e03252a3926747dd8d7ba1
+  data.tar.gz: 74dd5fbc8caa9f73115e9fcba347301cf99eeb9d
 SHA512:
-  metadata.gz: 6672d992af55beb7954846d8b47652d6c3918ab71616a68b54b4559a59705957572a99d87b364d15f92c81202401173608f26b35acc54f03d8711d0d60358e3e
-  data.tar.gz: 4c2fed94893a54a957f895c3ca57d54f02de2517437a564cccf90320fb18ee7349b9dd37b84e9b1e308e67fe672d7b16f91e879cc2ab221ece55af3e947c040d
+  metadata.gz: 1cd34d83b87621b4cb05204c651a6af6c1fda88e2d7c0edfbfafa9d782217099650d7a3309d48c76b960328c0406f97df530861840a1e57bc10c859b17de5c23
+  data.tar.gz: 0f9260d64f41f661334de67f665b817245d15938abb3582443ab3f2b13915f17927a101c2c1581586196cd47203ef1e9147e54d5ef1fcf9e0ca9161268c4dbbd

data/README.md

@@ -313,6 +313,20 @@ For more information on configuring redirects, see the documentation of the
 gem, which comes as a transitive dependency of the `s3_website` gem. (The
 command `s3_website cfg apply` internally calls the `configure-s3-website` gem.)
 
+#### On skipping application of redirects
+
+If your website has a lot of redirects, you may find the following setting
+helpful:
+
+    treat_zero_length_objects_as_redirects: true
+
+The setting allows `s3_website push` to infer whether a redirect exists or not.
+You will experience faster `push` performance when this setting is `true`.
+However, if this setting is enabled and you modify the `redirects` setting in
+*s3_website.yml*, use `push --force` to apply the modified values.
+
+For backward-compatibility reasons, this setting is `false` by default.
+
 ### Specifying custom concurrency level
 
 By default, `s3_website` does 3 operations in parallel. An operation can be an

data/changelog.md

@@ -2,6 +2,16 @@
 
 This project uses [Semantic Versioning](http://semver.org).
 
+## 2.7.0
+
+* Add setting `treat_zero_length_objects_as_redirects`
+
+  Before, `s3_website push` always uploaded one zero-length object for each
+  configured redirect. From now on, you can instruct s3\_website to treat
+  zero-length S3 objects as existing redirects. If you have a large amount of
+  redirects on your site, you may find that this feature decreases the duration
+  of `s3_website push`.
+
 ## 2.6.1
 
 * Always invalidate the object */index.html* if the setting `cloudfront_invalidate_root` is on

data/lib/s3_website/version.rb

@@ -1,3 +1,3 @@
 module S3Website
-  VERSION = '2.6.1'
+  VERSION = '2.7.0'
 end

data/resources/s3_website.jar.md5

@@ -1 +1 @@
-7a47d1df31ffe3bb8093719149222afd
+ddc9ee41507905b63f16ab01634e9e5a

data/src/main/scala/s3/website/Push.scala

@@ -1,11 +1,12 @@
 package s3.website
 
 import s3.website.model.Config.S3_website_yml
+import s3.website.model.Redirect.{Redirects, resolveRedirects}
 import s3.website.model.Site._
 import scala.concurrent.{ExecutionContextExecutor, Future, Await}
 import scala.concurrent.duration._
 import scala.language.postfixOps
-import s3.website.UploadHelper.{resolveDeletes, resolveUploads}
+import s3.website.UploadHelper.{FutureUploads, resolveDeletes, resolveUploads}
 import s3.website.S3._
 import scala.concurrent.ExecutionContext.fromExecutor
 import java.util.concurrent.Executors.newFixedThreadPool
@@ -80,29 +81,42 @@ object Push {
                 pushOptions: PushOptions
                 ): ExitCode = {
     logger.info(s"${Deploy.renderVerb} ${site.rootDirectory}/* to ${site.config.s3_bucket}")
-    val redirects = Redirect.resolveRedirects
     val s3FilesFuture = resolveS3Files()
-    val redirectReports: PushReports = redirects.map(S3 uploadRedirect _) map (Right(_))
-
-    val pushReports: Future[PushReports] = for {
-      errorOrUploads: Either[ErrorReport, Seq[Upload]] <- resolveUploads(s3FilesFuture)
+    val redirectsFuture: Redirects = resolveRedirects(s3FilesFuture)
+    val redirectReports: Future[Either[ErrorReport, Seq[Future[PushErrorOrSuccess]]]] =
+      redirectsFuture.map { (errOrRedirects: Either[ErrorReport, Seq[Redirect]]) =>
+        errOrRedirects.right.map(_.filter(_.needsUpload).map(S3 uploadRedirect _))
+      }
+
+    val uploadFutures: FutureUploads = resolveUploads(s3FilesFuture)
+    val uploadReports: Future[Either[ErrorReport, Seq[Future[PushErrorOrSuccess]]]] = for {
+      errorOrUploads: Either[ErrorReport, Seq[Upload]] <- uploadFutures
+    } yield errorOrUploads.right.map(_.map(S3 uploadFile _))
+
+    val deleteReports: Future[Either[ErrorReport, Seq[Future[PushErrorOrSuccess]]]] = for {
+      errorOrUploads: Either[ErrorReport, Seq[Upload]] <- uploadFutures
     } yield {
-      val uploadReports: PushReports = errorOrUploads.fold(
-        error => Left(error) :: Nil,
-        uploads => {
-          uploads.map(S3 uploadFile _).map(Right(_))
-        }
-      )
-      val deleteReports =
-        Await.result(resolveDeletes(s3FilesFuture, redirects), 1 day).right.map { keysToDelete =>
-          keysToDelete map (S3 delete _)
-        }.fold(
-          error => Left(error) :: Nil,
-          (pushResults: Seq[Future[PushErrorOrSuccess]]) => pushResults map (Right(_))
+      val errorsOrDeleteReports = redirectsFuture.flatMap { (errOrRedirects: Either[ErrorReport, Seq[Redirect]]) =>
+        errOrRedirects.fold(
+          err => Future(Left(err)),
+          redirects => resolveDeletes(s3FilesFuture, redirects)
         )
-      uploadReports ++ deleteReports ++ redirectReports
+      }.map { (deletes: Either[ErrorReport, Seq[S3Key]]) =>
+        deletes.right.map(keysToDelete => keysToDelete.map(S3 delete _))
+      }
+      Await.result(errorsOrDeleteReports, 1 day)
+    }
+    val allReports = Future.sequence(redirectReports :: uploadReports :: deleteReports :: Nil) map { reports =>
+      reports.foldLeft(Nil: PushReports) { (memo, report: Either[ErrorReport, Seq[Future[PushErrorOrSuccess]]]) =>
+        report match {
+          case Left(err) =>
+            memo :+ Left(err)
+          case Right(pushResults: Seq[Future[PushErrorOrSuccess]]) =>
+            memo ++ pushResults.map(Right(_))
+        }
+      }
     }
-    val finishedPushOps = awaitForResults(Await.result(pushReports, 1 day))
+    val finishedPushOps = awaitForResults(Await.result(allReports, 1 day))
     val invalidationSucceeded = invalidateCloudFrontItems(finishedPushOps)
     
     afterPushFinished(finishedPushOps, invalidationSucceeded)

data/src/main/scala/s3/website/model/Config.scala

@@ -23,7 +23,8 @@ case class Config(
   cloudfront_distribution_id: Option[String],
   cloudfront_invalidate_root: Option[Boolean],
   redirects:                  Option[Map[String, String]],
-  concurrency_level:          Int
+  concurrency_level:          Int,
+  treat_zero_length_objects_as_redirects: Option[Boolean]
 )
 
 object Config {

data/src/main/scala/s3/website/model/Site.scala

@@ -50,6 +50,7 @@ object Site {
           cloudfront_invalidate_root <- loadOptionalBoolean("cloudfront_invalidate_root").right
           concurrency_level <- loadOptionalInt("concurrency_level").right
           redirects <- loadRedirects.right
+          treat_zero_length_objects_as_redirects <- loadOptionalBoolean("treat_zero_length_objects_as_redirects").right
         } yield {
           gzip_zopfli.foreach(_ => logger.info(
             """|Zopfli is not currently supported. Falling back to regular gzip.
@@ -73,7 +74,8 @@ object Site {
             cloudfront_distribution_id,
             cloudfront_invalidate_root,
             redirects,
-            concurrency_level.fold(20)(_ max 20) // At minimum, run 20 concurrent operations
+            concurrency_level.fold(20)(_ max 20),
+            treat_zero_length_objects_as_redirects
           )
         }
       case Failure(error) =>

data/src/main/scala/s3/website/model/push.scala

@@ -11,6 +11,7 @@ import s3.website.model.Upload.tika
 import s3.website.model.Encoding.encodingOnS3
 import java.io.File.createTempFile
 import org.apache.commons.io.IOUtils.copy
+import scala.concurrent.{ExecutionContextExecutor, Future}
 import scala.util.Try
 
 object Encoding {
@@ -148,17 +149,39 @@ object Files {
   }
 }
 
-case class Redirect(s3Key: String, redirectTarget: String) {
+case class Redirect(s3Key: String, redirectTarget: String, needsUpload: Boolean) {
   def uploadType = RedirectFile
 }
 
+private case class RedirectSetting(source: String, target: String)
+
 object Redirect {
-  def resolveRedirects(implicit config: Config): Seq[Redirect] =
-    config.redirects.fold(Nil: Seq[Redirect]) { sourcesToTargets =>
-      sourcesToTargets.foldLeft(Seq(): Seq[Redirect]) {
+  type Redirects = Future[Either[ErrorReport, Seq[Redirect]]]
+
+  def resolveRedirects(s3FileFutures: Future[Either[ErrorReport, Seq[S3File]]])
+                      (implicit config: Config, executor: ExecutionContextExecutor, pushOptions: PushOptions): Redirects = {
+    val redirectSettings = config.redirects.fold(Nil: Seq[RedirectSetting]) { sourcesToTargets =>
+      sourcesToTargets.foldLeft(Seq(): Seq[RedirectSetting]) {
         (redirects, sourceToTarget) =>
-          redirects :+ Redirect(sourceToTarget._1, applySlashIfNeeded(sourceToTarget._2))
+          redirects :+ RedirectSetting(sourceToTarget._1, applySlashIfNeeded(sourceToTarget._2))
       }
+    }
+    def redirectsWithExistsOnS3Info =
+      s3FileFutures.map(_.right.map { s3Files =>
+        val existingRedirectKeys = s3Files.filter(_.size == 0).map(_.s3Key).toSet
+        redirectSettings.map(redirectSetting => 
+          Redirect(redirectSetting, needsUpload = !existingRedirectKeys.contains(redirectSetting.source))
+        )
+      })
+    val uploadOnlyMissingRedirects =
+      config.treat_zero_length_objects_as_redirects.contains(true) && !pushOptions.force
+    val allConfiguredRedirects = Future(Right(redirectSettings.map(redirectSetting =>
+      Redirect(redirectSetting, needsUpload = true)
+    )))
+    if (uploadOnlyMissingRedirects) 
+      redirectsWithExistsOnS3Info 
+    else 
+      allConfiguredRedirects
   }
 
   private def applySlashIfNeeded(redirectTarget: String) = {
@@ -169,10 +192,13 @@ object Redirect {
     else
       "/" + redirectTarget // let the user have redirect settings like "index.php: index.html" in s3_website.ml
   }
+
+  def apply(redirectSetting: RedirectSetting, needsUpload: Boolean): Redirect =
+      Redirect(redirectSetting.source, redirectSetting.target, needsUpload)
 }
 
-case class S3File(s3Key: String, md5: MD5)
+case class S3File(s3Key: String, md5: MD5, size: Long)
 
 object S3File {
-  def apply(summary: S3ObjectSummary): S3File = S3File(summary.getKey, summary.getETag)
+  def apply(summary: S3ObjectSummary): S3File = S3File(summary.getKey, summary.getETag, summary.getSize)
 }

data/src/test/scala/s3/website/S3WebsiteSpec.scala

@@ -35,7 +35,7 @@ class S3WebsiteSpec extends Specification {
     "update a gzipped S3 object if the contents has changed" in new BasicSetup {
       config = "gzip: true"
       setLocalFileWithContent(("styles.css", "<h1>hi again</h1>"))
-      setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
+      setS3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */)
       push
       sentPutObjectRequest.getKey must equalTo("styles.css")
     }
@@ -43,7 +43,7 @@ class S3WebsiteSpec extends Specification {
     "not update a gzipped S3 object if the contents has not changed" in new BasicSetup {
       config = "gzip: true"
       setLocalFileWithContent(("styles.css", "<h1>hi</h1>"))
-      setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
+      setS3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */)
       push
       noUploadsOccurred must beTrue
     }
@@ -59,7 +59,7 @@ class S3WebsiteSpec extends Specification {
         |  - .xml
       """.stripMargin
       setLocalFileWithContent(("file.xml", "<h1>hi again</h1>"))
-      setS3Files(S3File("file.xml", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
+      setS3File("file.xml", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */)
       push
       sentPutObjectRequest.getKey must equalTo("file.xml")
     }
@@ -68,14 +68,14 @@ class S3WebsiteSpec extends Specification {
   "push" should {
     "not upload a file if it has not changed" in new BasicSetup {
       setLocalFileWithContent(("index.html", "<div>hello</div>"))
-      setS3Files(S3File("index.html", md5Hex("<div>hello</div>")))
+      setS3File("index.html", md5Hex("<div>hello</div>"))
       push
       noUploadsOccurred must beTrue
     }
 
     "update a file if it has changed" in new BasicSetup {
       setLocalFileWithContent(("index.html", "<h1>old text</h1>"))
-      setS3Files(S3File("index.html", md5Hex("<h1>new text</h1>")))
+      setS3File("index.html", md5Hex("<h1>new text</h1>"))
       push
       sentPutObjectRequest.getKey must equalTo("index.html")
     }
@@ -87,7 +87,7 @@ class S3WebsiteSpec extends Specification {
     }
 
     "delete files that are on S3 but not on local file system" in new BasicSetup {
-      setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
+      setS3File("old.html", md5Hex("<h1>old text</h1>"))
       push
       sentDelete must equalTo("old.html")
     }
@@ -131,14 +131,14 @@ class S3WebsiteSpec extends Specification {
     }
 
     "try again if the delete fails" in new BasicSetup {
-      setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
+      setS3File("old.html", md5Hex("<h1>old text</h1>"))
       deleteFailsAndThenSucceeds(howManyFailures = 5)
       push
       verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
     }
 
     "try again if the object listing fails" in new BasicSetup {
-      setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
+      setS3File("old.html", md5Hex("<h1>old text</h1>"))
       objectListingFailsAndThenSucceeds(howManyFailures = 5)
       push
       verify(amazonS3Client, times(6)).listObjects(Matchers.any(classOf[ListObjectsRequest]))
@@ -292,7 +292,7 @@ class S3WebsiteSpec extends Specification {
     }
 
     "be 1 if an object listing fails" in new BasicSetup {
-      setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
+      setS3File("old.html", md5Hex("<h1>old text</h1>"))
       objectListingFailsAndThenSucceeds(howManyFailures = 6)
       push must equalTo(1)
     }
@@ -343,13 +343,13 @@ class S3WebsiteSpec extends Specification {
   "ignore_on_server: value" should {
     "not delete the S3 objects that match the ignore value" in new BasicSetup {
       config = "ignore_on_server: logs"
-      setS3Files(S3File("logs/log.txt", ""))
+      setS3File("logs/log.txt")
       push
       noDeletesOccurred must beTrue
     }
 
     "support non-US-ASCII files"  in new BasicSetup {
-      setS3Files(S3File("tags/笔记/test.html", ""))
+      setS3File("tags/笔记/test.html", "")
       config = "ignore_on_server: tags/笔记/test.html"
       push
       noDeletesOccurred must beTrue
@@ -361,7 +361,7 @@ class S3WebsiteSpec extends Specification {
       config = s"""
         |ignore_on_server: $DELETE_NOTHING_MAGIC_WORD
       """.stripMargin
-      setS3Files(S3File("file.txt", ""))
+      setS3File("file.txt")
       push
       noDeletesOccurred
     }
@@ -377,13 +377,13 @@ class S3WebsiteSpec extends Specification {
         |ignore_on_server:
         |  - .*txt
       """.stripMargin
-      setS3Files(S3File("logs/log.txt", ""))
+      setS3File("logs/log.txt", "")
       push
       noDeletesOccurred must beTrue
     }
 
     "support non-US-ASCII files" in new BasicSetup {
-      setS3Files(S3File("tags/笔记/test.html", ""))
+      setS3File("tags/笔记/test.html", "")
       config = """
                  |ignore_on_server:
                  |  - tags/笔记/test.html
@@ -570,7 +570,7 @@ class S3WebsiteSpec extends Specification {
         |  index.php: /index.html
       """.stripMargin
       setLocalFile("index.php")
-      setS3Files(S3File("index.php", "md5"))
+      setS3File("index.php", "md5")
       push
       noDeletesOccurred must beTrue
     }
@@ -625,7 +625,7 @@ class S3WebsiteSpec extends Specification {
   "push --force" should {
     "push all the files whether they have changed or not" in new ForcePush {
       setLocalFileWithContent(("index.html", "<h1>hi</h1>"))
-      setS3Files(S3File("index.html", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
+      setS3File("index.html", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */)
       push
       sentPutObjectRequest.getKey must equalTo("index.html")
     }
@@ -634,7 +634,7 @@ class S3WebsiteSpec extends Specification {
   "dry run" should {
     "not push updates" in new DryRun {
       setLocalFileWithContent(("index.html", "<div>new</div>"))
-      setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
+      setS3File("index.html", md5Hex("<div>old</div>"))
       push
       noUploadsOccurred must beTrue
     }
@@ -650,7 +650,7 @@ class S3WebsiteSpec extends Specification {
     }
 
     "not push deletes" in new DryRun {
-      setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
+      setS3File("index.html", md5Hex("<div>old</div>"))
       push
       noUploadsOccurred must beTrue
     }
@@ -663,7 +663,7 @@ class S3WebsiteSpec extends Specification {
 
     "not invalidate files" in new DryRun {
       config = "cloudfront_invalidation_id: AABBCC"
-      setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
+      setS3File("index.html", md5Hex("<div>old</div>"))
       push
       noInvalidationsOccurred must beTrue
     }
@@ -684,6 +684,55 @@ class S3WebsiteSpec extends Specification {
       sentPutObjectRequests.length must equalTo(1)
     }
   }
+
+  "the setting treat_zero_length_objects_as_redirects" should {
+    "skip application of redirect on a zero-length S3 object" in new BasicSetup {
+      config =
+        """
+          |treat_zero_length_objects_as_redirects: true
+          |redirects:
+          |  index.php: /index.html
+        """.stripMargin
+      setRedirectObject("index.php")
+      push
+      noUploadsOccurred
+    }
+
+    "not delete the redirect objects on the bucket" in new BasicSetup {
+      config =
+        """
+          |treat_zero_length_objects_as_redirects: true
+          |redirects:
+          |  index.php: /index.html
+        """.stripMargin
+      setRedirectObject("index.php")
+      push
+      noDeletesOccurred
+    }
+
+    "apply redirects that are missing from S3" in new BasicSetup {
+      config =
+        """
+          |treat_zero_length_objects_as_redirects: true
+          |redirects:
+          |  index.php: /index.html
+        """.stripMargin
+      push
+      sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
+    }
+
+    "apply all redirects when the user invokes `push --force`" in new ForcePush {
+      config =
+        """
+          |treat_zero_length_objects_as_redirects: true
+          |redirects:
+          |  index.php: /index.html
+        """.stripMargin
+      setRedirectObject("index.php")
+      push
+      sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
+    }
+  }
   
   trait BasicSetup extends SiteLocationFromCliArg with EmptySite with MockAWS with DefaultRunMode
 
@@ -768,22 +817,28 @@ class S3WebsiteSpec extends Specification {
     val s3ObjectListing = new ObjectListing
     when(amazonS3Client.listObjects(Matchers.any(classOf[ListObjectsRequest]))).thenReturn(s3ObjectListing)
 
+    // Simulate the situation where the file on S3 is outdated (as compared to the local file)
     def setOutdatedS3Keys(s3Keys: String*) {
-      s3Keys
-        .map(key =>
-          S3File(key, md5Hex(Random.nextLong().toString)) // Simulate the situation where the file on S3 is outdated (as compared to the local file)
-        )
-        .foreach (setS3Files(_))
+      s3Keys.map(setS3File(_))
+    }
+
+    def setRedirectObject(s3Key: String) {
+      setS3File(s3Key, size = 0)
+    }
+
+    def setS3File(s3Key: String, md5: String = "", size: Int = 10) {
+      s3ObjectListing.getObjectSummaries.add({
+        val summary = new S3ObjectSummary
+        summary.setETag(md5)
+        summary.setKey(s3Key)
+        summary.setSize(size)
+        summary
+      })
     }
 
     def setS3Files(s3Files: S3File*) {
       s3Files.foreach { s3File =>
-        s3ObjectListing.getObjectSummaries.add({
-          val summary = new S3ObjectSummary
-          summary.setETag(s3File.md5)
-          summary.setKey(s3File.s3Key)
-          summary
-        })
+        setS3File(s3File.s3Key, s3File.md5)
       }
     }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_website
 version: !ruby/object:Gem::Version
-  version: 2.6.1
+  version: 2.7.0
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-05 00:00:00.000000000 Z
+date: 2014-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor