s3_website_monadic 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 64e9b6093bd40e0e6728a66dcfcf5134e5b72d52
-  data.tar.gz: 6aa1819b0581575150f767fc253f63a635c53a43
+  metadata.gz: 9b120ee4e68407bd3c41617d5f2fd2ef96142f3c
+  data.tar.gz: 3db76e44515ec798bc4bab1907fa612aefef9e64
 SHA512:
-  metadata.gz: 7dcb71a7dd601dca1d6d85f387f97170f7c09eb9e185db2755334af4137f99e42877fffca2ec46ca8995efbaa17ab78fd32dbae866cb91dfbf7b8122263f2f68
-  data.tar.gz: 3fc944d61d92e04cf7d20532a091a3492c9d22e237c41a659087cb3c8175ff2c023ca8d4fbf59fcf92636007c93fa2b08241f68f21589ab670d7669204004dca
+  metadata.gz: 7aa168c9af8a8e74261a76b829300d488e4f7765682504ac8d86cbc963059c11952a6b448e911724d4c5ab52b90eb72f0f278cdf1928485a7146fe558250aab6
+  data.tar.gz: 7b1800e6dcfa02bbed5e9b4034f59db180e4864b0dadacebaac526da23e88ecc4a1053addbea5e513e937de613795fe275aaa78634418ab4cebc8e190713a12d

data/s3_website.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "s3_website_monadic"
-  s.version     = "0.0.5"
+  s.version     = "0.0.6"
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Lauri Lehmijoki"]
   s.email       = ["lauri.lehmijoki@iki.fi"]

data/src/main/scala/s3/website/CloudFront.scala

@@ -2,29 +2,28 @@ package s3.website
 
 import s3.website.model.{Redirect, Config}
 import com.amazonaws.services.cloudfront.{AmazonCloudFrontClient, AmazonCloudFront}
-import s3.website.CloudFront.{FailedInvalidation, SuccessfulInvalidation, CloudFrontClientProvider}
-import scala.util.{Failure, Success, Try}
+import s3.website.CloudFront.{CloudFrontSettings, CloudFrontClientProvider, SuccessfulInvalidation, FailedInvalidation}
+import scala.util.Try
 import com.amazonaws.services.cloudfront.model.{TooManyInvalidationsInProgressException, Paths, InvalidationBatch, CreateInvalidationRequest}
 import scala.collection.JavaConversions._
 import scala.concurrent.duration._
-import s3.website.S3.{SuccessfulUpload, PushSuccessReport}
+import s3.website.S3.PushSuccessReport
 import com.amazonaws.auth.BasicAWSCredentials
 import s3.website.Logger._
 import s3.website.S3.SuccessfulUpload
 import scala.util.Failure
-import s3.website.CloudFront.SuccessfulInvalidation
 import scala.util.Success
-import s3.website.CloudFront.FailedInvalidation
-import java.net.{URI, URLEncoder}
-import java.net.URLEncoder.encode
+import java.net.URI
+import Utils._
 
-class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUnit) {
+class CloudFront(implicit cloudFrontSettings: CloudFrontSettings, config: Config) {
+  val cloudFront = cloudFrontSettings.cfClient(config)
 
-  def invalidate(invalidationBatch: InvalidationBatch, distributionId: String)(implicit config: Config): InvalidationResult = {
+  def invalidate(invalidationBatch: InvalidationBatch, distributionId: String): InvalidationResult = {
     def tryInvalidate(implicit attempt: Int = 1): Try[SuccessfulInvalidation] =
       Try {
         val invalidationReq = new CreateInvalidationRequest(distributionId, invalidationBatch)
-        cfClient(config).createInvalidation(invalidationReq)
+        cloudFront.createInvalidation(invalidationReq)
         val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
         info(result)
         result
@@ -32,7 +31,7 @@ class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUni
         case e: TooManyInvalidationsInProgressException =>
           implicit val duration: Duration = Duration(
             (fibs drop attempt).head min 15, /* AWS docs way that invalidations complete in 15 minutes */
-            sleepUnit
+            cloudFrontSettings.cloudFrontSleepTimeUnit
           )
           pending(maxInvalidationsExceededInfo)
           Thread.sleep(duration.toMillis)
@@ -63,7 +62,6 @@ class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUni
 
   type InvalidationResult = Either[FailedInvalidation, SuccessfulInvalidation]
 
-  lazy val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map { n => n._1 + n._2 }
 }
 
 object CloudFront {
@@ -113,4 +111,9 @@ object CloudFront {
     }
     case _ => false
   }
+
+  case class CloudFrontSettings(
+    cfClient: CloudFrontClientProvider = CloudFront.awsCloudFrontClient,
+    cloudFrontSleepTimeUnit: TimeUnit = MINUTES
+  )
 }

data/src/main/scala/s3/website/Push.scala

@@ -18,18 +18,11 @@ import s3.website.model.Update
 import s3.website.model.NewFile
 import s3.website.S3.PushSuccessReport
 import scala.collection.mutable.ArrayBuffer
-import s3.website.CloudFront.{CloudFrontClientProvider, toInvalidationBatches}
-import s3.website.S3.SuccessfulDelete
-import s3.website.CloudFront.SuccessfulInvalidation
-import s3.website.S3.SuccessfulUpload
-import s3.website.CloudFront.FailedInvalidation
+import s3.website.CloudFront._
 import s3.website.Logger._
 import s3.website.S3.SuccessfulDelete
 import s3.website.CloudFront.SuccessfulInvalidation
-import s3.website.S3.SuccessfulUpload
-import s3.website.CloudFront.FailedInvalidation
-import s3.website.S3.SuccessfulDelete
-import s3.website.CloudFront.SuccessfulInvalidation
+import s3.website.S3.S3Settings
 import s3.website.S3.SuccessfulUpload
 import s3.website.CloudFront.FailedInvalidation
 
@@ -38,9 +31,8 @@ object Push {
   def pushSite(
                 implicit site: Site,
                 executor: ExecutionContextExecutor,
-                s3ClientProvider: S3ClientProvider = S3.awsS3Client,
-                cloudFrontClientProvider: CloudFrontClientProvider = CloudFront.awsCloudFrontClient,
-                cloudFrontSleepTimeUnit: TimeUnit = MINUTES
+                s3Settings: S3Settings,
+                cloudFrontSettings: CloudFrontSettings
                 ): ExitCode = {
     info(s"Deploying ${site.rootDirectory}/* to ${site.config.s3_bucket}")
     val utils: Utils = new Utils
@@ -54,7 +46,7 @@ object Push {
         .map { s3File => new S3() delete s3File.s3Key }
         .map { Right(_) } // To make delete reports type-compatible with upload reports
       val uploadReports: PushReports = utils toParSeq (redirects.toStream.map(Right(_)) ++ resolveUploads(localFiles, s3Files))
-        .map { errorOrUpload => errorOrUpload.right.map(new S3() upload ) }
+        .map { errorOrUpload => errorOrUpload.right.map(new S3() upload) }
       uploadReports ++ deleteReports
     }
     val errorsOrFinishedPushOps: Either[Error, FinishedPushOperations] = errorsOrReports.right map {
@@ -67,7 +59,7 @@ object Push {
   
   def invalidateCloudFrontItems
     (errorsOrFinishedPushOps: Either[Error, FinishedPushOperations])
-    (implicit config: Config, cloudFrontClientProvider: CloudFrontClientProvider, cloudFrontSleepTimeUnit: TimeUnit): Option[InvalidationSucceeded] = {
+    (implicit config: Config, cloudFrontSettings: CloudFrontSettings): Option[InvalidationSucceeded] = {
     config.cloudfront_distribution_id.map {
       distributionId =>
         val pushSuccessReports = errorsOrFinishedPushOps.fold(
@@ -190,6 +182,8 @@ object Push {
       implicit site =>
         val threadPool = newFixedThreadPool(site.config.concurrency_level)
         implicit val executor = fromExecutor(threadPool)
+        implicit val s3Settings = S3Settings()
+        implicit val cloudFrontSettings = CloudFrontSettings()
         val pushStatus = pushSite
         threadPool.shutdownNow()
         pushStatus

data/src/main/scala/s3/website/S3.scala

@@ -10,14 +10,8 @@ import com.amazonaws.AmazonClientException
 import scala.concurrent.{ExecutionContextExecutor, Future}
 import s3.website.S3._
 import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
-import s3.website.S3.SuccessfulUpload
-import s3.website.S3.FailedUpload
-import scala.util.Failure
-import scala.Some
-import s3.website.model.IOError
-import scala.util.Success
-import s3.website.model.UserError
 import s3.website.Logger._
+import s3.website.Utils._
 import s3.website.S3.SuccessfulUpload
 import s3.website.S3.SuccessfulDelete
 import s3.website.S3.FailedUpload
@@ -27,34 +21,50 @@ import s3.website.S3.FailedDelete
 import s3.website.model.IOError
 import scala.util.Success
 import s3.website.model.UserError
+import scala.concurrent.duration.{TimeUnit, Duration}
+import java.util.concurrent.TimeUnit.SECONDS
 
-class S3(implicit s3Client: S3ClientProvider) {
+class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
 
-  def upload(upload: Upload with UploadTypeResolved)(implicit config: Config, executor: ExecutionContextExecutor): Future[Either[FailedUpload, SuccessfulUpload]] =
+  def upload(upload: Upload with UploadTypeResolved)(implicit a: Attempt = 1, config: Config): S3PushResult =
     Future {
-      s3Client(config) putObject toPutObjectRequest(upload)
+      s3Settings.s3Client(config) putObject toPutObjectRequest(upload)
       val report = SuccessfulUpload(upload)
       info(report)
       Right(report)
-    } recover usingErrorHandler { error =>
-      FailedUpload(upload.s3Key, error)
-    }
+    } recoverWith retry(
+      createReport = error => FailedUpload(upload.s3Key, error),
+      retryAction  = newAttempt => this.upload(upload)(newAttempt, config)
+    )
 
-  def delete(s3Key: String)(implicit config: Config, executor: ExecutionContextExecutor): Future[Either[FailedDelete, SuccessfulDelete]] =
+  def delete(s3Key: String)(implicit a: Attempt = 1, config: Config): S3PushResult =
     Future {
-      s3Client(config) deleteObject(config.s3_bucket, s3Key)
+      s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
       val report = SuccessfulDelete(s3Key)
       info(report)
       Right(report)
-    } recover usingErrorHandler { error =>
-      FailedDelete(s3Key, error)
-    }
-
-  def usingErrorHandler[T <: PushFailureReport, F <: PushFailureReport](f: (Throwable) => T): PartialFunction[Throwable, Either[T, F]] = {
-    case error =>
-      val report = f(error)
-      info(report)
-      Left(report)
+    } recoverWith retry(
+      createReport = error => FailedDelete(s3Key, error),
+      retryAction  = newAttempt => this.delete(s3Key)(newAttempt, config)
+    )
+      
+  type S3PushResult = Future[Either[PushFailureReport, PushSuccessReport]]
+  
+  type Attempt = Int
+  
+  def retry(createReport: (Throwable) => PushFailureReport, retryAction: (Attempt) => S3PushResult)(implicit attempt: Attempt):
+  PartialFunction[Throwable, S3PushResult] = {
+    case error: Throwable =>
+      val failureReport = createReport(error)
+      if (attempt == 6) {
+        info(failureReport)
+        Future(Left(failureReport))
+      } else {
+        val sleepDuration = Duration(fibs.drop(attempt + 1).head, s3Settings.retrySleepTimeUnit)
+        pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
+        Thread.sleep(sleepDuration.toMillis)
+        retryAction(attempt + 1)
+      }
   }
 
   def toPutObjectRequest(upload: Upload)(implicit config: Config) =
@@ -89,7 +99,7 @@ class S3(implicit s3Client: S3ClientProvider) {
 object S3 {
   def awsS3Client(config: Config) = new AmazonS3Client(new BasicAWSCredentials(config.s3_id, config.s3_secret))
 
-  def resolveS3Files(implicit config: Config, s3ClientProvider: S3ClientProvider): Either[Error, Stream[S3File]] = Try {
+  def resolveS3Files(implicit config: Config, s3Settings: S3Settings): Either[Error, Stream[S3File]] = Try {
     objectSummaries()
   } match {
     case Success(remoteFiles) =>
@@ -100,8 +110,8 @@ object S3 {
       Left(IOError(error))
   }
 
-  def objectSummaries(nextMarker: Option[String] = None)(implicit config: Config, s3ClientProvider: S3ClientProvider): Stream[S3File] = {
-    val objects: ObjectListing = s3ClientProvider(config).listObjects({
+  def objectSummaries(nextMarker: Option[String] = None)(implicit config: Config, s3Settings: S3Settings): Stream[S3File] = {
+    val objects: ObjectListing = s3Settings.s3Client(config).listObjects({
       val req = new ListObjectsRequest()
       req.setBucketName(config.s3_bucket)
       nextMarker.foreach(req.setMarker)
@@ -143,4 +153,9 @@ object S3 {
   }
 
   type S3ClientProvider = (Config) => AmazonS3
+
+  case class S3Settings(
+    s3Client: S3ClientProvider = S3.awsS3Client,
+    retrySleepTimeUnit: TimeUnit = SECONDS
+  )
 }

data/src/main/scala/s3/website/Utils.scala

@@ -12,6 +12,10 @@ class Utils(implicit config: Config) {
   }
 }
 
+object Utils {
+  lazy val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map { n => n._1 + n._2 }
+}
+
 object Logger {
   import Rainbow._
   def info(msg: String) = println(s"[${"info".blue}] $msg")

data/src/test/scala/s3/website/S3WebsiteSpec.scala

@@ -14,13 +14,13 @@ import com.amazonaws.services.s3.model._
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Await
 import scala.concurrent.duration._
-import s3.website.S3.S3ClientProvider
+import s3.website.S3.{S3Settings, S3ClientProvider}
 import scala.collection.JavaConversions._
 import s3.website.model.NewFile
 import scala.Some
 import com.amazonaws.AmazonServiceException
 import org.apache.commons.codec.digest.DigestUtils.md5Hex
-import s3.website.CloudFront.CloudFrontClientProvider
+import s3.website.CloudFront.{CloudFrontSettings, CloudFrontClientProvider}
 import com.amazonaws.services.cloudfront.AmazonCloudFront
 import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, CreateInvalidationRequest, TooManyInvalidationsInProgressException}
 import org.mockito.stubbing.Answer
@@ -93,6 +93,21 @@ class S3WebsiteSpec extends Specification {
       Push.pushSite
       sentDelete must equalTo("old.html")
     }
+
+    "try again if the upload fails" in new SiteDirectory with MockAWS {
+      implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
+      uploadFailsAndThenSucceeds(howManyFailures = 5)
+      Push.pushSite
+      verify(amazonS3Client, times(6)).putObject(Matchers.any(classOf[PutObjectRequest]))
+    }
+
+    "try again if the delete fails" in new SiteDirectory with MockAWS {
+      implicit val site = buildSite()
+      setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
+      deleteFailsAndThenSucceeds(howManyFailures = 5)
+      Push.pushSite
+      verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
+    }
   }
 
   "push with CloudFront" should {
@@ -185,6 +200,18 @@ class S3WebsiteSpec extends Specification {
       )
       Push.pushSite must equalTo(1)
     }
+
+    "be 0 if upload retry succeeds" in new SiteDirectory with MockAWS {
+      implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
+      uploadFailsAndThenSucceeds(howManyFailures = 1)
+      Push.pushSite must equalTo(0)
+    }
+
+    "be 1 if delete retry fails" in new SiteDirectory with MockAWS {
+      implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
+      uploadFailsAndThenSucceeds(howManyFailures = 6)
+      Push.pushSite must equalTo(1)
+    }
   }
 
   "s3_website.yml file" should {
@@ -323,8 +350,10 @@ class S3WebsiteSpec extends Specification {
   
   trait MockCloudFront {
     val amazonCloudFrontClient = mock(classOf[AmazonCloudFront])
-    implicit val cfClientProvider: CloudFrontClientProvider = _ => amazonCloudFrontClient
-    implicit val cloudFrontSleepTimeUnit: TimeUnit = MILLISECONDS
+    implicit val cfSettings: CloudFrontSettings = CloudFrontSettings(
+      cfClient = _ => amazonCloudFrontClient,
+      cloudFrontSleepTimeUnit = MICROSECONDS
+    )
 
     def sentInvalidationRequests: Seq[CreateInvalidationRequest] = {
       val createInvalidationReq = ArgumentCaptor.forClass(classOf[CreateInvalidationRequest])
@@ -359,7 +388,10 @@ class S3WebsiteSpec extends Specification {
   
   trait MockS3 {
     val amazonS3Client = mock(classOf[AmazonS3])
-    implicit val s3ClientProvider: S3ClientProvider = _ => amazonS3Client
+    implicit val s3Settings: S3Settings = S3Settings(
+      s3Client = _ => amazonS3Client,
+      retrySleepTimeUnit = MICROSECONDS
+    )
     val s3ObjectListing = new ObjectListing
     when(amazonS3Client.listObjects(Matchers.any(classOf[ListObjectsRequest]))).thenReturn(s3ObjectListing)
 
@@ -376,6 +408,32 @@ class S3WebsiteSpec extends Specification {
 
     val s3 = new S3()
 
+    def uploadFailsAndThenSucceeds(howManyFailures: Int) {
+      var callCount = 0
+      doAnswer(new Answer[PutObjectResult] {
+        override def answer(invocation: InvocationOnMock) = {
+          callCount += 1
+          if (callCount <= howManyFailures)
+            throw new AmazonServiceException("AWS is temporarily down")
+          else
+            mock(classOf[PutObjectResult])
+        }
+      }).when(amazonS3Client).putObject(Matchers.anyObject())
+    }
+
+    def deleteFailsAndThenSucceeds(howManyFailures: Int) {
+      var callCount = 0
+      doAnswer(new Answer[DeleteObjectRequest] {
+        override def answer(invocation: InvocationOnMock) = {
+          callCount += 1
+          if (callCount <= howManyFailures)
+            throw new AmazonServiceException("AWS is temporarily down")
+          else
+            mock(classOf[DeleteObjectRequest])
+        }
+      }).when(amazonS3Client).deleteObject(Matchers.anyString(), Matchers.anyString())
+    }
+
     def asSeenByS3Client(upload: Upload)(implicit config: Config): PutObjectRequest = {
       Await.ready(s3.upload(upload withUploadType NewFile), Duration("1 s"))
       val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_website_monadic
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-06 00:00:00.000000000 Z
+date: 2014-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk