s3_website_monadic 0.0.9 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 92e9b4b0029ead9d9f441f2a5fdb2687f535a729
-  data.tar.gz: ea74c668aa1f6accc39a1541e56af5b16781f507
+  metadata.gz: 83042f3d9526d801a4c3ca1b843b8bbf3e03f02e
+  data.tar.gz: a5e5ac7ae8fcc22d519d8d3503d74734050ac415
 SHA512:
-  metadata.gz: 866b36966c21be8e2dd88748e9ca3955807668b3bd4efe9a33c70d1458c3d45ed4dc81d8a2b26b2399dfaf7c2a874ebd72006bc9324d7326a43f4457a2629415
-  data.tar.gz: 25613874bf0d7bc119f0307aabf6eecb70a958d010ab486a9ab931b2ae3b6183ebda195c888040f6a0c660b8c50be0cbf9aea5d22e90f3aa11dec2c7cf2a250a
+  metadata.gz: 8625012df6b394782179cd5795f512f8dc6c99335eaa9d382b750b564acd0c697f9c4fbb652730499dc79d55fcecbb5be8d94c4f47c6afa9e80415b6bb3b6fda
+  data.tar.gz: 98d5ac27855f7b212ed644e7df6d2122bd7fba976206643fd398fe8275e157e50f3b44127180ecdfd36d2cb11b74ad5b38357715041e4ab8c24e4f0c61043ccb

data/s3_website.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "s3_website_monadic"
-  s.version     = "0.0.9"
+  s.version     = "0.0.11"
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Lauri Lehmijoki"]
   s.email       = ["lauri.lehmijoki@iki.fi"]

data/src/main/scala/s3/website/CloudFront.scala

@@ -1,54 +1,47 @@
 package s3.website
 
-import s3.website.model.{Redirect, Config}
+import s3.website.model.{Update, Redirect, Config}
 import com.amazonaws.services.cloudfront.{AmazonCloudFrontClient, AmazonCloudFront}
-import s3.website.CloudFront.{CloudFrontSettings, CloudFrontClientProvider, SuccessfulInvalidation, FailedInvalidation}
-import scala.util.Try
+import s3.website.CloudFront.{CloudFrontSettings, SuccessfulInvalidation, FailedInvalidation}
 import com.amazonaws.services.cloudfront.model.{TooManyInvalidationsInProgressException, Paths, InvalidationBatch, CreateInvalidationRequest}
 import scala.collection.JavaConversions._
 import scala.concurrent.duration._
-import s3.website.S3.PushSuccessReport
+import s3.website.S3.{SuccessfulDelete, PushSuccessReport, SuccessfulUpload}
 import com.amazonaws.auth.BasicAWSCredentials
 import s3.website.Logger._
-import s3.website.S3.SuccessfulUpload
-import scala.util.Failure
-import scala.util.Success
 import java.net.URI
 import Utils._
+import scala.concurrent.{ExecutionContextExecutor, Future}
 
 class CloudFront(implicit cloudFrontSettings: CloudFrontSettings, config: Config) {
   val cloudFront = cloudFrontSettings.cfClient(config)
 
-  def invalidate(invalidationBatch: InvalidationBatch, distributionId: String): InvalidationResult = {
-    def tryInvalidate(implicit attempt: Int = 1): Try[SuccessfulInvalidation] =
-      Try {
-        val invalidationReq = new CreateInvalidationRequest(distributionId, invalidationBatch)
-        cloudFront.createInvalidation(invalidationReq)
-        val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
-        info(result)
-        result
-      } recoverWith {
-        case e: TooManyInvalidationsInProgressException =>
-          implicit val duration: Duration = Duration(
-            (fibs drop attempt).head min 15, /* AWS docs way that invalidations complete in 15 minutes */
-            cloudFrontSettings.cloudFrontSleepTimeUnit
-          )
-          pending(maxInvalidationsExceededInfo)
-          Thread.sleep(duration.toMillis)
-          tryInvalidate(attempt + 1)
-      }
+  def invalidate(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt = 1)
+                (implicit ec: ExecutionContextExecutor): InvalidationResult =
+    Future {
+      val invalidationReq = new CreateInvalidationRequest(distributionId, invalidationBatch)
+      cloudFront.createInvalidation(invalidationReq)
+      val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
+      info(result)
+      Right(result)
+    } recoverWith (tooManyInvalidationsRetry(invalidationBatch, distributionId, attempt) orElse retry(attempt)(
+      createFailureReport = error => FailedInvalidation(error),
+      retryAction = nextAttempt => invalidate(invalidationBatch, distributionId, nextAttempt)
+    ))
 
-    tryInvalidate() match {
-      case Success(res) =>
-        Right(res)
-      case Failure(err) =>
-        val report = FailedInvalidation(err)
-        info(report)
-        Left(report)
-    }
+  def tooManyInvalidationsRetry(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt)
+                          (implicit ec: ExecutionContextExecutor): PartialFunction[Throwable, InvalidationResult] = {
+    case e: TooManyInvalidationsInProgressException =>
+      val duration: Duration = Duration(
+        (fibs drop attempt).head min 15, /* CloudFront invalidations complete within 15 minutes */
+        cloudFrontSettings.retryTimeUnit
+      )
+      pending(maxInvalidationsExceededInfo(duration, attempt))
+      Thread.sleep(duration.toMillis)
+      invalidate(invalidationBatch, distributionId, attempt + 1)
   }
 
-  def maxInvalidationsExceededInfo(implicit sleepDuration: Duration, attempt: Int) = {
+  def maxInvalidationsExceededInfo(sleepDuration: Duration, attempt: Int) = {
     val basicInfo = s"The maximum amount of CloudFront invalidations has exceeded. Trying again in $sleepDuration, please wait."
     val extendedInfo =
       s"""|$basicInfo
@@ -60,8 +53,7 @@ class CloudFront(implicit cloudFrontSettings: CloudFrontSettings, config: Config
       basicInfo
   }
 
-  type InvalidationResult = Either[FailedInvalidation, SuccessfulInvalidation]
-
+  type InvalidationResult = Future[Either[FailedInvalidation, SuccessfulInvalidation]]
 }
 
 object CloudFront {
@@ -72,7 +64,7 @@ object CloudFront {
     def reportMessage = s"Invalidated $invalidatedItemsCount item(s) on CloudFront"
   }
 
-  case class FailedInvalidation(error: Throwable) extends FailureReport{
+  case class FailedInvalidation(error: Throwable) extends FailureReport {
     def reportMessage = s"Failed to invalidate the CloudFront distribution (${error.getMessage})"
   }
 
@@ -81,21 +73,9 @@ object CloudFront {
   
   def toInvalidationBatches(pushSuccessReports: Seq[PushSuccessReport])(implicit config: Config): Seq[InvalidationBatch] =
     pushSuccessReports
-      .filterNot(isRedirect) // Assume that redirect objects are never cached.
-      .map(report =>
-        new URI(
-          "http",
-          "cloudfront", // We want to use the encoder in the URI class. These must be passed in.
-          "/" + report.s3Key,  // CloudFront keys have the slash in front
-          null
-        ).toURL.getPath // The URL class encodes the unsafe characters
-      )
-      .map { path =>
-        if (config.cloudfront_invalidate_root.exists(_ == true))
-          path.replaceFirst("/index.html$", "/")
-        else
-          path
-      }
+      .filter(needsInvalidation) // Assume that redirect objects are never cached.
+      .map(toInvalidationPath)
+      .map (applyInvalidateRootSetting)
       .grouped(1000) // CloudFront supports max 1000 invalidations in one request (http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#InvalidationLimits)
       .map { batchKeys =>
         new InvalidationBatch() withPaths
@@ -104,16 +84,36 @@ object CloudFront {
       }
       .toSeq
 
-  def isRedirect: PartialFunction[PushSuccessReport, Boolean] = {
+  def applyInvalidateRootSetting(path: String)(implicit config: Config) =
+    if (config.cloudfront_invalidate_root.exists(_ == true))
+      path.replaceFirst("/index.html$", "/")
+    else
+      path
+
+  def toInvalidationPath(report: PushSuccessReport) = {
+    def encodeUnsafeChars(path: String) =
+      new URI(
+        "http",
+        "cloudfront", // We want to use the encoder in the URI class. These must be passed in.
+        "/" + report.s3Key,  // CloudFront keys have the slash in front
+        path
+      ).toURL.getPath // The URL class encodes the unsafe characters
+    val invalidationPath = "/" + report.s3Key  // CloudFront keys have the slash in front
+    encodeUnsafeChars(invalidationPath)
+  }
+
+
+  def needsInvalidation: PartialFunction[PushSuccessReport, Boolean] = {
     case SuccessfulUpload(upload) => upload.uploadType match {
-      case Redirect => true
+      case Update => true
       case _ => false
     }
+    case SuccessfulDelete(_) => true
     case _ => false
   }
 
   case class CloudFrontSettings(
     cfClient: CloudFrontClientProvider = CloudFront.awsCloudFrontClient,
-    cloudFrontSleepTimeUnit: TimeUnit = MINUTES
-  )
+    retryTimeUnit: TimeUnit = MINUTES
+  ) extends RetrySettings
 }

data/src/main/scala/s3/website/Diff.scala

@@ -17,26 +17,20 @@ object Diff {
     }
   }
 
-  def resolveUploads(localFiles: Seq[LocalFile], s3Files: Seq[S3File])(implicit config: Config):
+  def resolveNewFiles(localFiles: Seq[LocalFile], s3Files: Seq[S3File])(implicit config: Config):
   Stream[Either[ErrorReport, Upload with UploadTypeResolved]] = {
     val remoteS3KeysIndex = s3Files.map(_.s3Key).toSet
-    val remoteMd5Index = s3Files.map(_.md5).toSet
     localFiles
       .toStream // Load lazily, because the MD5 computation for the local file requires us to read the whole file
       .map(resolveUploadSource)
       .collect {
       case errorOrUpload if errorOrUpload.right.exists(isNewUpload(remoteS3KeysIndex)) =>
         for (upload <- errorOrUpload.right) yield upload withUploadType NewFile
-      case errorOrUpload if errorOrUpload.right.exists(isUpdate(remoteS3KeysIndex, remoteMd5Index)) =>
-        for (upload <- errorOrUpload.right) yield upload withUploadType Update
     }
   }
 
   def isNewUpload(remoteS3KeysIndex: Set[String])(u: Upload) = !remoteS3KeysIndex.exists(_ == u.s3Key)
 
-  def isUpdate(remoteS3KeysIndex: Set[String], remoteMd5Index: Set[String])(u: Upload) =
-    remoteS3KeysIndex.exists(_ == u.s3Key) && !remoteMd5Index.exists(remoteMd5 => u.essence.right.exists(_.md5 == remoteMd5))
-
   def resolveUploadSource(localFile: LocalFile)(implicit config: Config): Either[ErrorReport, Upload] =
     for (upload <- LocalFile.toUpload(localFile).right)
     yield upload

data/src/main/scala/s3/website/Push.scala

@@ -5,7 +5,7 @@ import scala.concurrent.{ExecutionContextExecutor, Future, Await}
 import scala.concurrent.duration._
 import com.lexicalscope.jewel.cli.CliFactory
 import scala.language.postfixOps
-import s3.website.Diff.{resolveUploads, resolveDeletes}
+import s3.website.Diff.{resolveNewFiles, resolveDeletes}
 import s3.website.S3._
 import scala.concurrent.ExecutionContext.fromExecutor
 import java.util.concurrent.Executors.newFixedThreadPool
@@ -39,18 +39,20 @@ object Push {
     val utils: Utils = new Utils
 
     val redirects = Redirect.resolveRedirects
-    val redirectResults = redirects.map(new S3() upload)
+    val redirectResults = redirects.map(new S3() upload(_))
 
     val errorsOrReports = for {
-      s3Files    <- Await.result(resolveS3Files(), 1 minutes).right
-      localFiles <- resolveLocalFiles.right
+      localFiles        <- resolveLocalFiles.right
+      errorOrS3FilesAndUpdateFutures <- Await.result(resolveS3FilesAndUpdates(localFiles)(), 7 days).right
+      s3Files <- errorOrS3FilesAndUpdateFutures._1.right
     } yield {
+      val updateReports: PushReports = errorOrS3FilesAndUpdateFutures._2.par
       val deleteReports: PushReports = utils toParSeq resolveDeletes(localFiles, s3Files, redirects)
         .map { s3File => new S3() delete s3File.s3Key }
         .map { Right(_) } // To make delete reports type-compatible with upload reports
-      val uploadReports: PushReports = utils toParSeq resolveUploads(localFiles, s3Files)
-        .map { _.right.map(new S3() upload) }
-      uploadReports ++ deleteReports ++ redirectResults.map(Right(_))
+      val uploadReports: PushReports = utils toParSeq resolveNewFiles(localFiles, s3Files)
+        .map { _.right.map(new S3() upload(_)) }
+      uploadReports ++ deleteReports ++ updateReports ++ redirectResults.map(Right(_))
     }
     val errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations] = errorsOrReports.right map {
       uploadReports => awaitForUploads(uploadReports)
@@ -62,7 +64,7 @@ object Push {
   
   def invalidateCloudFrontItems
     (errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations])
-    (implicit config: Config, cloudFrontSettings: CloudFrontSettings): Option[InvalidationSucceeded] = {
+    (implicit config: Config, cloudFrontSettings: CloudFrontSettings, ec: ExecutionContextExecutor): Option[InvalidationSucceeded] = {
     config.cloudfront_distribution_id.map {
       distributionId =>
         val pushSuccessReports = errorsOrFinishedPushOps.fold(
@@ -84,7 +86,12 @@ object Push {
           }
         )
         val invalidationResults: Seq[Either[FailedInvalidation, SuccessfulInvalidation]] =
-          toInvalidationBatches(pushSuccessReports) map (new CloudFront().invalidate(_, distributionId))
+          toInvalidationBatches(pushSuccessReports) map { invalidationBatch =>
+            Await.result(
+              new CloudFront().invalidate(invalidationBatch, distributionId),
+              atMost = 1 day
+            )
+          }
         if (invalidationResults.exists(_.isLeft))
           false // If one of the invalidations failed, mark the whole process as failed
         else
@@ -164,8 +171,8 @@ object Push {
                          redirects: Int = 0, 
                          deletes: Int = 0
                          )
-  type FinishedPushOperations = ParSeq[Either[ErrorReport, Either[PushFailureReport, PushSuccessReport]]]
-  type PushReports = ParSeq[Either[ErrorReport, Future[Either[PushFailureReport, PushSuccessReport]]]]
+  type FinishedPushOperations = ParSeq[Either[ErrorReport, PushErrorOrSuccess]]
+  type PushReports = ParSeq[Either[ErrorReport, Future[PushErrorOrSuccess]]]
   case class PushResult(threadPool: ExecutorService, uploadReports: PushReports)
   type ExitCode = Int
 

data/src/main/scala/s3/website/S3.scala

@@ -6,11 +6,9 @@ import com.amazonaws.auth.BasicAWSCredentials
 import com.amazonaws.services.s3.model._
 import scala.collection.JavaConversions._
 import scala.concurrent.{ExecutionContextExecutor, Future}
-import s3.website.S3._
 import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
 import s3.website.Logger._
-import s3.website.Utils._
-import scala.concurrent.duration.{Duration, TimeUnit}
+import scala.concurrent.duration.TimeUnit
 import java.util.concurrent.TimeUnit.SECONDS
 import s3.website.S3.SuccessfulUpload
 import s3.website.S3.SuccessfulDelete
@@ -18,30 +16,29 @@ import s3.website.S3.FailedUpload
 import scala.Some
 import s3.website.S3.FailedDelete
 import s3.website.S3.S3Settings
-import s3.website.model.Error.isClientError
 
 class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
 
-  def upload(upload: Upload with UploadTypeResolved)(implicit a: Attempt = 1, config: Config): Future[Either[FailedUpload, SuccessfulUpload]] =
+  def upload(upload: Upload with UploadTypeResolved, a: Attempt = 1)(implicit config: Config): Future[Either[FailedUpload, SuccessfulUpload]] =
     Future {
       s3Settings.s3Client(config) putObject toPutObjectRequest(upload)
       val report = SuccessfulUpload(upload)
       info(report)
       Right(report)
-    } recoverWith retry(
+    } recoverWith retry(a)(
       createFailureReport = error => FailedUpload(upload.s3Key, error),
-      retryAction  = newAttempt => this.upload(upload)(newAttempt, config)
+      retryAction  = newAttempt => this.upload(upload, newAttempt)
     )
 
-  def delete(s3Key: String)(implicit a: Attempt = 1, config: Config): Future[Either[FailedDelete, SuccessfulDelete]] =
+  def delete(s3Key: String,  a: Attempt = 1)(implicit config: Config): Future[Either[FailedDelete, SuccessfulDelete]] =
     Future {
       s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
       val report = SuccessfulDelete(s3Key)
       info(report)
       Right(report)
-    } recoverWith retry(
+    } recoverWith retry(a)(
       createFailureReport = error => FailedDelete(s3Key, error),
-      retryAction  = newAttempt => this.delete(s3Key)(newAttempt, config)
+      retryAction  = newAttempt => this.delete(s3Key, newAttempt)
     )
       
   def toPutObjectRequest(upload: Upload)(implicit config: Config) =
@@ -83,49 +80,62 @@ class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
 object S3 {
   def awsS3Client(config: Config) = new AmazonS3Client(new BasicAWSCredentials(config.s3_id, config.s3_secret))
 
-  def resolveS3Files(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil)
-                    (implicit attempt: Attempt = 1, config: Config, s3Settings: S3Settings, ec: ExecutionContextExecutor): ObjectListingResult = Future {
-    nextMarker.foreach(m => info(s"Fetching the next part of the object listing from S3 (starting from $m)"))
+  def resolveS3FilesAndUpdates(localFiles: Seq[LocalFile])
+                              (nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil,  attempt: Attempt = 1, onFlightUpdateFutures: UpdateFutures = Nil)
+                              (implicit config: Config, s3Settings: S3Settings, ec: ExecutionContextExecutor):
+  ErrorOrS3FilesAndUpdates = Future {
+    debug(nextMarker.fold
+      ("Querying S3 files")
+      {m => s"Querying more S3 files (starting from $m)"}
+    )
     val objects: ObjectListing = s3Settings.s3Client(config).listObjects({
       val req = new ListObjectsRequest()
       req.setBucketName(config.s3_bucket)
       nextMarker.foreach(req.setMarker)
       req
     })
-    objects
-  } flatMap { objects =>
+    val summaryIndex = objects.getObjectSummaries.map { summary => (summary.getETag, summary.getKey) }.toSet // Index to avoid O(n^2) lookups 
+    def isUpdate(lf: LocalFile) =
+      summaryIndex.exists((md5AndS3Key) => 
+        md5AndS3Key._1 != lf.md5 && md5AndS3Key._2 == lf.s3Key
+      )
+    val updateFutures: UpdateFutures = localFiles.collect {
+      case lf: LocalFile if isUpdate(lf) =>
+        val errorOrUpdate = LocalFile
+          .toUpload(lf)
+          .right
+          .map { (upload: Upload) =>
+            upload.withUploadType(Update)
+          }
+        errorOrUpdate.right.map(update => new S3 upload update)
+    }
+
+    (objects, onFlightUpdateFutures ++ updateFutures)
+  } flatMap { (objectsAndUpdateFutures) =>
+    val objects: ObjectListing = objectsAndUpdateFutures._1
+    val updateFutures: UpdateFutures = objectsAndUpdateFutures._2
     val s3Files = alreadyResolved ++ (objects.getObjectSummaries.toIndexedSeq.toSeq map (S3File(_)))
     Option(objects.getNextMarker)
-      .fold(Future(Right(s3Files)): ObjectListingResult)(nextMarker => resolveS3Files(Some(nextMarker), s3Files))
-  } recoverWith retry(
-    createFailureReport = error => UserError(s"Failed to fetch an object listing (${error.getMessage})"),
-    retryAction = nextAttempt => resolveS3Files(nextMarker, alreadyResolved)(nextAttempt, config, s3Settings, ec)
+      .fold(Future(Right((Right(s3Files), updateFutures))): ErrorOrS3FilesAndUpdates) // We've received all the S3 keys from the bucket
+      { nextMarker => // There are more S3 keys on the bucket. Fetch them.
+        resolveS3FilesAndUpdates(localFiles)(Some(nextMarker), s3Files, attempt = attempt, updateFutures)
+      }
+  } recoverWith retry(attempt)(
+    createFailureReport = error => ClientError(s"Failed to fetch an object listing (${error.getMessage})"),
+    retryAction = nextAttempt => resolveS3FilesAndUpdates(localFiles)(nextMarker, alreadyResolved, nextAttempt, onFlightUpdateFutures)
   )
 
-  type ObjectListingResult = Future[Either[ErrorReport, Seq[S3File]]]
+  type S3FilesAndUpdates = (ErrorOrS3Files, UpdateFutures)
+  type S3FilesAndUpdatesFuture = Future[S3FilesAndUpdates]
+  type ErrorOrS3FilesAndUpdates = Future[Either[ErrorReport, S3FilesAndUpdates]]
+  type UpdateFutures = Seq[Either[ErrorReport, Future[PushErrorOrSuccess]]]
+  type ErrorOrS3Files = Either[ErrorReport, Seq[S3File]]
 
   sealed trait PushFailureReport extends FailureReport
   sealed trait PushSuccessReport extends SuccessReport {
     def s3Key: String
   }
 
-  def retry[L <: Report, R](createFailureReport: (Throwable) => L, retryAction: (Attempt) => Future[Either[L, R]])
-           (implicit attempt: Attempt, s3Settings: S3Settings, ec: ExecutionContextExecutor):
-  PartialFunction[Throwable, Future[Either[L, R]]] = {
-    case error: Throwable if attempt == 6 || isClientError(error) =>
-      val failureReport = createFailureReport(error)
-      fail(failureReport.reportMessage)
-      Future(Left(failureReport))
-    case error: Throwable =>
-      val failureReport = createFailureReport(error)
-      val sleepDuration = Duration(fibs.drop(attempt + 1).head, s3Settings.retrySleepTimeUnit)
-      pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
-      Thread.sleep(sleepDuration.toMillis)
-      retryAction(attempt + 1)
-  }
-
-  type Attempt = Int
-
   case class SuccessfulUpload(upload: Upload with UploadTypeResolved) extends PushSuccessReport {
     def reportMessage =
       upload.uploadType match {
@@ -153,6 +163,6 @@ object S3 {
 
   case class S3Settings(
     s3Client: S3ClientProvider = S3.awsS3Client,
-    retrySleepTimeUnit: TimeUnit = SECONDS
-  )
+    retryTimeUnit: TimeUnit = SECONDS
+  ) extends RetrySettings
 }

data/src/main/scala/s3/website/Utils.scala

@@ -18,6 +18,7 @@ object Utils {
 
 object Logger {
   import Rainbow._
+  def debug(msg: String) = println(s"[${"debg".cyan}] $msg")
   def info(msg: String) = println(s"[${"info".blue}] $msg")
   def fail(msg: String) = println(s"[${"fail".red}] $msg")
 

data/src/main/scala/s3/website/model/Config.scala

@@ -33,7 +33,7 @@ object Config {
       })
     }
 
-    yamlValue getOrElse Left(UserError(s"The key $key has to have a boolean or [string] value"))
+    yamlValue getOrElse Left(ClientError(s"The key $key has to have a boolean or [string] value"))
   }
 
   def loadOptionalStringOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[String, Seq[String]]]] = {
@@ -46,7 +46,7 @@ object Config {
       })
     }
 
-    yamlValue getOrElse Left(UserError(s"The key $key has to have a string or [string] value"))
+    yamlValue getOrElse Left(ClientError(s"The key $key has to have a string or [string] value"))
   }
 
   def loadMaxAge(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Int, Map[String, Int]]]] = {
@@ -60,7 +60,7 @@ object Config {
       })
     }
 
-    yamlValue getOrElse Left(UserError(s"The key $key has to have an int or (string -> int) value"))
+    yamlValue getOrElse Left(ClientError(s"The key $key has to have an int or (string -> int) value"))
   }
 
   def loadEndpoint(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[S3Endpoint]] =
@@ -79,7 +79,7 @@ object Config {
       redirects <- Try(redirectsOption.map(_.asInstanceOf[java.util.Map[String,String]].toMap))
     } yield Right(redirects)
 
-    yamlValue getOrElse Left(UserError(s"The key $key has to have a (string -> string) value"))
+    yamlValue getOrElse Left(ClientError(s"The key $key has to have a (string -> string) value"))
   }
 
   def loadRequiredString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, String] = {
@@ -91,7 +91,7 @@ object Config {
     }
 
     yamlValue getOrElse {
-      Left(UserError(s"The key $key has to have a string value"))
+      Left(ClientError(s"The key $key has to have a string value"))
     }
   }
 
@@ -104,7 +104,7 @@ object Config {
     }
 
     yamlValueOption getOrElse {
-      Left(UserError(s"The key $key has to have a string value"))
+      Left(ClientError(s"The key $key has to have a string value"))
     }
   }
 
@@ -117,7 +117,7 @@ object Config {
     }
 
     yamlValueOption getOrElse {
-      Left(UserError(s"The key $key has to have a boolean value"))
+      Left(ClientError(s"The key $key has to have a boolean value"))
     }
   }
 
@@ -130,7 +130,7 @@ object Config {
     }
 
     yamlValueOption getOrElse {
-      Left(UserError(s"The key $key has to have an integer value"))
+      Left(ClientError(s"The key $key has to have an integer value"))
     }
   }
 

data/src/main/scala/s3/website/model/S3Endpoint.scala

@@ -8,7 +8,7 @@ case class S3Endpoint(
 object S3Endpoint {
   val defaultEndpoint = S3Endpoint("s3-website-us-east-1.amazonaws.com", "s3.amazonaws.com")
 
-  def forString(locationConstraint: String): Either[UserError, S3Endpoint] = locationConstraint match {
+  def forString(locationConstraint: String): Either[ClientError, S3Endpoint] = locationConstraint match {
     case "EU" | "eu-west-1" => Right(S3Endpoint("s3-website-eu-west-1.amazonaws.com",      "s3-eu-west-1.amazonaws.com"))
     case "us-east-1" =>        Right(defaultEndpoint)
     case "us-west-1" =>        Right(S3Endpoint("s3-website-us-west-1.amazonaws.com",      "s3-us-west-1.amazonaws.com"))
@@ -17,6 +17,6 @@ object S3Endpoint {
     case "ap-southeast-2" =>   Right(S3Endpoint("s3-website-ap-southeast-2.amazonaws.com", "s3-ap-southeast-2.amazonaws.com"))
     case "ap-northeast-1" =>   Right(S3Endpoint("s3-website-ap-northeast-1.amazonaws.com", "s3-ap-northeast-1.amazonaws.com"))
     case "sa-east-1" =>        Right(S3Endpoint("s3-website-sa-east-1.amazonaws.com",      "s3-sa-east-1.amazonaws.com"))
-    case _ =>                  Left(UserError(s"Unrecognised endpoint: $locationConstraint"))
+    case _ =>                  Left(ClientError(s"Unrecognised endpoint: $locationConstraint"))
   }
 }

data/src/main/scala/s3/website/model/errors.scala

@@ -9,7 +9,7 @@ object Error {
     error.isInstanceOf[AmazonServiceException] && error.asInstanceOf[AmazonServiceException].getErrorType == Client
 }
 
-case class UserError(reportMessage: String) extends ErrorReport
+case class ClientError(reportMessage: String) extends ErrorReport
 
 case class IOError(exception: Throwable) extends ErrorReport {
   def reportMessage = exception.getMessage

data/src/main/scala/s3/website/model/push.scala

@@ -55,38 +55,51 @@ case object Update extends UploadType
 
 case class LocalFile(
   s3Key: String,
-  sourceFile: File,
+  originalFile: File,
   encodingOnS3: Option[Either[Gzip, Zopfli]]
-) extends S3KeyProvider
+) extends S3KeyProvider {
 
-object LocalFile {
-  def toUpload(localFile: LocalFile)(implicit config: Config): Either[ErrorReport, Upload] = Try {
-    def fis(file: File): InputStream = new FileInputStream(file)
-    def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
-    val sourceFile: File = localFile
-      .encodingOnS3
-      .fold(localFile.sourceFile)(algorithm => {
-      val tempFile = File.createTempFile(localFile.sourceFile.getName, "gzip")
-      tempFile.deleteOnExit()
-      using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
-        IOUtils.copy(fis(localFile.sourceFile), stream)
-      }
-      tempFile
-    })
-    val md5 = using(fis(sourceFile)) { inputStream =>
-      DigestUtils.md5Hex(inputStream)
+  // May throw an exception, so remember to call this in a Try or Future monad
+  lazy val length = uploadFile.length()
+
+  /**
+   * This is the file we should upload, because it contains the potentially gzipped contents of the original file.
+   *
+   * May throw an exception, so remember to call this in a Try or Future monad
+   */
+  lazy val uploadFile: File = encodingOnS3
+    .fold(originalFile)(algorithm => {
+    val tempFile = File.createTempFile(originalFile.getName, "gzip")
+    tempFile.deleteOnExit()
+    using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
+      IOUtils.copy(fis(originalFile), stream)
     }
+    tempFile
+  })
+
+  /**
+   * May throw an exception, so remember to call this in a Try or Future monad
+   */
+  lazy val md5 = using(fis(uploadFile)) { inputStream =>
+    DigestUtils.md5Hex(inputStream)
+  }
 
+  private[this] def fis(file: File): InputStream = new FileInputStream(file)
+  private[this] def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
+}
+
+object LocalFile {
+  def toUpload(localFile: LocalFile)(implicit config: Config): Either[ErrorReport, Upload] = Try {
     Upload(
       s3Key = localFile.s3Key,
       essence = Right(
         UploadBody(
-          md5 = md5,
+          md5 = localFile.md5,
           contentEncoding = localFile.encodingOnS3.map(_ => "gzip"),
-          contentLength = sourceFile.length(),
+          contentLength = localFile.length,
           maxAge = resolveMaxAge(localFile),
-          contentType = resolveContentType(localFile.sourceFile),
-          openInputStream = () => new FileInputStream(sourceFile)
+          contentType = resolveContentType(localFile.originalFile),
+          openInputStream = () => new FileInputStream(localFile.uploadFile)
         )
       )
     )
@@ -109,7 +122,7 @@ object LocalFile {
     type GlobsMap = Map[String, Int]
     config.max_age.flatMap { (intOrGlobs: Either[Int, GlobsMap]) =>
       type GlobsSeq = Seq[(String, Int)]
-      def respectMostSpecific(globs: GlobsMap): GlobsSeq = globs.toSeq.sortBy(_._1).reverse
+      def respectMostSpecific(globs: GlobsMap): GlobsSeq = globs.toSeq.sortBy(_._1.length).reverse
       intOrGlobs
         .right.map(respectMostSpecific)
         .fold(
@@ -135,7 +148,7 @@ object LocalFile {
         (exclusionRegex: String) => rubyRegexMatches(file.s3Key, exclusionRegex),
         (exclusionRegexes: Seq[String]) => exclusionRegexes exists (rubyRegexMatches(file.s3Key, _))
       ) }
-    } filterNot { _.sourceFile.getName == "s3_website.yml" } // For security reasons, the s3_website.yml should never be pushed
+    } filterNot { _.originalFile.getName == "s3_website.yml" } // For security reasons, the s3_website.yml should never be pushed
   } match {
     case Success(localFiles) =>
       Right(

data/src/main/scala/s3/website/package.scala

@@ -1,5 +1,12 @@
 package s3
 
+import scala.concurrent.{ExecutionContextExecutor, Future}
+import s3.website.model.Error._
+import s3.website.Logger._
+import scala.concurrent.duration.{TimeUnit, Duration}
+import s3.website.Utils._
+import s3.website.S3.{PushSuccessReport, PushFailureReport}
+
 package object website {
   trait Report {
     def reportMessage: String
@@ -9,4 +16,28 @@ package object website {
   trait FailureReport extends Report
 
   trait ErrorReport extends Report
+
+  trait RetrySettings {
+    def retryTimeUnit: TimeUnit
+  }
+
+  type PushErrorOrSuccess = Either[PushFailureReport, PushSuccessReport]
+
+  type Attempt = Int
+
+  def retry[L <: Report, R](attempt: Attempt)
+                           (createFailureReport: (Throwable) => L, retryAction: (Attempt) => Future[Either[L, R]])
+                           (implicit retrySettings: RetrySettings, ec: ExecutionContextExecutor):
+  PartialFunction[Throwable, Future[Either[L, R]]] = {
+    case error: Throwable if attempt == 6 || isClientError(error) =>
+      val failureReport = createFailureReport(error)
+      fail(failureReport.reportMessage)
+      Future(Left(failureReport))
+    case error: Throwable =>
+      val failureReport = createFailureReport(error)
+      val sleepDuration = Duration(fibs.drop(attempt + 1).head, retrySettings.retryTimeUnit)
+      pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
+      Thread.sleep(sleepDuration.toMillis)
+      retryAction(attempt + 1)
+  }
 }

data/src/test/scala/s3/website/S3WebsiteSpec.scala

@@ -27,6 +27,7 @@ import org.mockito.stubbing.Answer
 import org.mockito.invocation.InvocationOnMock
 import com.amazonaws.AmazonServiceException.ErrorType
 import java.util.concurrent.atomic.AtomicInteger
+import scala.collection.immutable.IndexedSeq
 
 class S3WebsiteSpec extends Specification {
 
@@ -132,15 +133,25 @@ class S3WebsiteSpec extends Specification {
   }
 
   "push with CloudFront" should {
-    "invalidate the CloudFront items" in new SiteDirectory with MockAWS {
+    "invalidate the updated CloudFront items" in new SiteDirectory with MockAWS {
       implicit val site = siteWithFiles(
         config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
         localFiles = "test.css" :: "articles/index.html" :: Nil
       )
+      setOutdatedS3Keys("test.css", "articles/index.html")
       Push.pushSite
       sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/test.css" :: "/articles/index.html" :: Nil).sorted)
     }
 
+    "not send CloudFront invalidation requests on new objects"  in new SiteDirectory with MockAWS {
+      implicit val site = siteWithFiles(
+        config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
+        localFiles = "newfile.js" :: Nil
+      )
+      Push.pushSite
+      noInvalidationsOccurred must beTrue
+    }
+
     "not send CloudFront invalidation requests on redirect objects" in new SiteDirectory with MockAWS {
       implicit val site = buildSite(
         config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z"), redirects = Some(Map("/index.php" -> "index.html")))
@@ -155,15 +166,28 @@ class S3WebsiteSpec extends Specification {
         config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
         localFiles = "test.css" :: Nil
       )
+      setOutdatedS3Keys("test.css")
       Push.pushSite must equalTo(0) // The retries should finally result in a success
       sentInvalidationRequests.length must equalTo(4)
     }
 
+    "retry if CloudFront is temporarily unreachable" in new SiteDirectory with MockAWS {
+      invalidationsFailAndThenSucceed(5)
+      implicit val site = siteWithFiles(
+        config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
+        localFiles = "test.css" :: Nil
+      )
+      setOutdatedS3Keys("test.css")
+      Push.pushSite
+      sentInvalidationRequests.length must equalTo(6)
+    }
+
     "encode unsafe characters in the keys" in new SiteDirectory with MockAWS {
       implicit val site = siteWithFiles(
         config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
         localFiles = "articles/arnold's file.html" :: Nil
       )
+      setOutdatedS3Keys("articles/arnold's file.html")
       Push.pushSite
       sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/articles/arnold's%20file.html" :: Nil).sorted)
     }
@@ -175,6 +199,7 @@ class S3WebsiteSpec extends Specification {
         config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z"), cloudfront_invalidate_root = Some(true)),
         localFiles = "index.html" :: "articles/index.html" :: Nil
       )
+      setOutdatedS3Keys("index.html", "articles/index.html")
       Push.pushSite
       sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/" :: "/articles/" :: Nil).sorted)
     }
@@ -182,10 +207,12 @@ class S3WebsiteSpec extends Specification {
 
   "a site with over 1000 items" should {
     "split the CloudFront invalidation requests into batches of 1000 items" in new SiteDirectory with MockAWS {
+      val files = (1 to 1002).map { i => s"file-$i"}
       implicit val site = siteWithFiles(
         config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
-        localFiles = (1 to 1002).map { i => s"file-$i"}
+        localFiles = files
       )
+      setOutdatedS3Keys(files:_*)
       Push.pushSite
       sentInvalidationRequests.length must equalTo(2)
       sentInvalidationRequests(0).getInvalidationBatch.getPaths.getItems.length must equalTo(1000)
@@ -219,12 +246,13 @@ class S3WebsiteSpec extends Specification {
       Push.pushSite must equalTo(0)
     }
 
-    "be 1 if CloudFront invalidation fails"in new SiteDirectory with MockAWS {
+    "be 1 if CloudFront is unreachable or broken"in new SiteDirectory with MockAWS {
       setCloudFrontAsInternallyBroken()
       implicit val site = siteWithFiles(
         config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
-        localFiles = "test.css" :: "articles/index.html" :: Nil
+        localFiles = "test.css" :: Nil
       )
+      setOutdatedS3Keys("test.css")
       Push.pushSite must equalTo(1)
     }
 
@@ -340,13 +368,13 @@ class S3WebsiteSpec extends Specification {
     "respect the more specific glob" in new SiteDirectory with MockAWS {
       implicit val site = siteWithFiles(
         defaultConfig.copy(max_age = Some(Right(Map(
-          "**" -> 150,
-          "assets/**" -> 86400
+          "assets/*" -> 150,
+          "assets/*.gif" -> 86400
         )))),
-        localFiles = "index.html" :: "assets/picture.gif" :: Nil
+        localFiles = "assets/jquery.js" :: "assets/picture.gif" :: Nil
       )
       Push.pushSite
-      sentPutObjectRequests.find(_.getKey == "index.html").get.getMetadata.getCacheControl must equalTo("max-age=150")
+      sentPutObjectRequests.find(_.getKey == "assets/jquery.js").get.getMetadata.getCacheControl must equalTo("max-age=150")
       sentPutObjectRequests.find(_.getKey == "assets/picture.gif").get.getMetadata.getCacheControl must equalTo("max-age=86400")
     }
   }
@@ -429,11 +457,11 @@ class S3WebsiteSpec extends Specification {
   
   trait MockAWS extends MockS3 with MockCloudFront with Scope
   
-  trait MockCloudFront {
+  trait MockCloudFront extends MockAWSHelper {
     val amazonCloudFrontClient = mock(classOf[AmazonCloudFront])
     implicit val cfSettings: CloudFrontSettings = CloudFrontSettings(
       cfClient = _ => amazonCloudFrontClient,
-      cloudFrontSleepTimeUnit = MICROSECONDS
+      retryTimeUnit = MICROSECONDS
     )
 
     def sentInvalidationRequests: Seq[CreateInvalidationRequest] = {
@@ -449,6 +477,12 @@ class S3WebsiteSpec extends Specification {
       true // Mockito is based on exceptions
     }
 
+    def invalidationsFailAndThenSucceed(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
+      doAnswer(temporaryFailure(classOf[CreateInvalidationResult]))
+        .when(amazonCloudFrontClient)
+        .createInvalidation(Matchers.anyObject())
+    }
+
     def setTooManyInvalidationsInProgress(attemptWhenInvalidationSucceeds: Int) {
       var callCount = 0
       doAnswer(new Answer[CreateInvalidationResult] {
@@ -462,20 +496,30 @@ class S3WebsiteSpec extends Specification {
       }).when(amazonCloudFrontClient).createInvalidation(Matchers.anyObject())
     }
 
+
+
     def setCloudFrontAsInternallyBroken() {
       when(amazonCloudFrontClient.createInvalidation(Matchers.anyObject())).thenThrow(new AmazonServiceException("CloudFront is down"))
     }
   }
   
-  trait MockS3 {
+  trait MockS3 extends MockAWSHelper {
     val amazonS3Client = mock(classOf[AmazonS3])
     implicit val s3Settings: S3Settings = S3Settings(
       s3Client = _ => amazonS3Client,
-      retrySleepTimeUnit = MICROSECONDS
+      retryTimeUnit = MICROSECONDS
     )
     val s3ObjectListing = new ObjectListing
     when(amazonS3Client.listObjects(Matchers.any(classOf[ListObjectsRequest]))).thenReturn(s3ObjectListing)
 
+    def setOutdatedS3Keys(s3Keys: String*) {
+      s3Keys
+        .map(key =>
+          S3File(key, md5Hex(Random.nextLong().toString)) // Simulate the situation where the file on S3 is outdated (as compared to the local file)
+        )
+        .foreach (setS3Files(_))
+    }
+
     def setS3Files(s3Files: S3File*) {
       s3Files.foreach { s3File =>
         s3ObjectListing.getObjectSummaries.add({
@@ -507,16 +551,6 @@ class S3WebsiteSpec extends Specification {
         .listObjects(Matchers.any(classOf[ListObjectsRequest]))
     }
 
-    def temporaryFailure[T](clazz: Class[T])(implicit callCount: AtomicInteger, howManyFailures: Int) = new Answer[T] {
-      def answer(invocation: InvocationOnMock) = {
-        callCount.incrementAndGet()
-        if (callCount.get() <= howManyFailures)
-          throw new AmazonServiceException("AWS is temporarily down")
-        else
-          mock(clazz)
-      }
-    }
-
     def asSeenByS3Client(upload: Upload)(implicit config: Config): PutObjectRequest = {
       Await.ready(s3.upload(upload withUploadType NewFile), Duration("1 s"))
       val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])
@@ -552,7 +586,19 @@ class S3WebsiteSpec extends Specification {
 
     type S3Key = String 
   }
-  
+
+  trait MockAWSHelper {
+    def temporaryFailure[T](clazz: Class[T])(implicit callCount: AtomicInteger, howManyFailures: Int) = new Answer[T] {
+      def answer(invocation: InvocationOnMock) = {
+        callCount.incrementAndGet()
+        if (callCount.get() <= howManyFailures)
+          throw new AmazonServiceException("AWS is temporarily down")
+        else
+          mock(clazz)
+      }
+    }
+  }
+
   trait SiteDirectory extends After {
     val siteDir = new File(FileUtils.getTempDirectory, "site" + Random.nextLong())
     siteDir.mkdir()

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_website_monadic
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.11
 platform: ruby
 authors:
 - Lauri Lehmijoki