s3_website_monadic 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b120ee4e68407bd3c41617d5f2fd2ef96142f3c
-  data.tar.gz: 3db76e44515ec798bc4bab1907fa612aefef9e64
+  metadata.gz: 714e16805c804d5b86d6097a85ba4ec85a4dee14
+  data.tar.gz: fd750b7d7de92913e893104d5945cfb78184d511
 SHA512:
-  metadata.gz: 7aa168c9af8a8e74261a76b829300d488e4f7765682504ac8d86cbc963059c11952a6b448e911724d4c5ab52b90eb72f0f278cdf1928485a7146fe558250aab6
-  data.tar.gz: 7b1800e6dcfa02bbed5e9b4034f59db180e4864b0dadacebaac526da23e88ecc4a1053addbea5e513e937de613795fe275aaa78634418ab4cebc8e190713a12d
+  metadata.gz: 403cb29f93d49f16f36dd52e72f4f7f6fd897137f8b485cd7940eedbc59940528b054a051b971e9e35098155dfb4afc79cc3f284b15b6395736a7bd250152d42
+  data.tar.gz: 1c10547952fa577f6d574d6b08aad9849a3746a723952c1262a53db05c27dcb0851c682cbc505630964840494bf513337c2b6c448292417d6bf4f73d4a14a930

data/s3_website.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "s3_website_monadic"
-  s.version     = "0.0.6"
+  s.version     = "0.0.7"
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Lauri Lehmijoki"]
   s.email       = ["lauri.lehmijoki@iki.fi"]

data/src/main/scala/s3/website/Diff.scala

@@ -2,6 +2,7 @@ package s3.website
 
 import s3.website.model._
 import s3.website.Ruby.rubyRegexMatches
+import s3.website._
 
 object Diff {
 
@@ -17,7 +18,7 @@ object Diff {
   }
 
   def resolveUploads(localFiles: Seq[LocalFile], s3Files: Seq[S3File])(implicit config: Config):
-  Stream[Either[Error, Upload with UploadTypeResolved]] = {
+  Stream[Either[ErrorReport, Upload with UploadTypeResolved]] = {
     val remoteS3KeysIndex = s3Files.map(_.s3Key).toSet
     val remoteMd5Index = s3Files.map(_.md5).toSet
     localFiles
@@ -36,7 +37,7 @@ object Diff {
   def isUpdate(remoteS3KeysIndex: Set[String], remoteMd5Index: Set[String])(u: Upload) =
     remoteS3KeysIndex.exists(_ == u.s3Key) && !remoteMd5Index.exists(remoteMd5 => u.essence.right.exists(_.md5 == remoteMd5))
 
-  def resolveUploadSource(localFile: LocalFile)(implicit config: Config): Either[Error, Upload] =
+  def resolveUploadSource(localFile: LocalFile)(implicit config: Config): Either[ErrorReport, Upload] =
     for (upload <- LocalFile.toUpload(localFile).right)
     yield upload
 }

data/src/main/scala/s3/website/Push.scala

@@ -23,6 +23,7 @@ import s3.website.Logger._
 import s3.website.S3.SuccessfulDelete
 import s3.website.CloudFront.SuccessfulInvalidation
 import s3.website.S3.S3Settings
+import s3.website.CloudFront.CloudFrontSettings
 import s3.website.S3.SuccessfulUpload
 import s3.website.CloudFront.FailedInvalidation
 
@@ -37,19 +38,21 @@ object Push {
     info(s"Deploying ${site.rootDirectory}/* to ${site.config.s3_bucket}")
     val utils: Utils = new Utils
 
+    val redirects = Redirect.resolveRedirects
+    val redirectResults = redirects.map(new S3() upload)
+
     val errorsOrReports = for {
-      s3Files    <- resolveS3Files.right
+      s3Files    <- Await.result(resolveS3Files(), 1 minutes).right
       localFiles <- resolveLocalFiles.right
     } yield {
-      val redirects = Redirect.resolveRedirects
       val deleteReports: PushReports = utils toParSeq resolveDeletes(localFiles, s3Files, redirects)
         .map { s3File => new S3() delete s3File.s3Key }
         .map { Right(_) } // To make delete reports type-compatible with upload reports
-      val uploadReports: PushReports = utils toParSeq (redirects.toStream.map(Right(_)) ++ resolveUploads(localFiles, s3Files))
-        .map { errorOrUpload => errorOrUpload.right.map(new S3() upload) }
-      uploadReports ++ deleteReports
+      val uploadReports: PushReports = utils toParSeq resolveUploads(localFiles, s3Files)
+        .map { _.right.map(new S3() upload) }
+      uploadReports ++ deleteReports ++ redirectResults.map(Right(_))
     }
-    val errorsOrFinishedPushOps: Either[Error, FinishedPushOperations] = errorsOrReports.right map {
+    val errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations] = errorsOrReports.right map {
       uploadReports => awaitForUploads(uploadReports)
     }
     val invalidationSucceeded = invalidateCloudFrontItems(errorsOrFinishedPushOps)
@@ -58,7 +61,7 @@ object Push {
   }
   
   def invalidateCloudFrontItems
-    (errorsOrFinishedPushOps: Either[Error, FinishedPushOperations])
+    (errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations])
     (implicit config: Config, cloudFrontSettings: CloudFrontSettings): Option[InvalidationSucceeded] = {
     config.cloudfront_distribution_id.map {
       distributionId =>
@@ -91,17 +94,17 @@ object Push {
 
   type InvalidationSucceeded = Boolean
 
-  def afterPushFinished(errorsOrFinishedUploads: Either[Error, FinishedPushOperations], invalidationSucceeded: Option[Boolean])(implicit config: Config): ExitCode = {
+  def afterPushFinished(errorsOrFinishedUploads: Either[ErrorReport, FinishedPushOperations], invalidationSucceeded: Option[Boolean])(implicit config: Config): ExitCode = {
     errorsOrFinishedUploads.right.foreach { finishedUploads =>
       val pushCounts = pushCountsToString(resolvePushCounts(finishedUploads))
       info(s"Summary: $pushCounts")
     }
-    errorsOrFinishedUploads.left foreach (err => fail(s"Encountered error: ${err.message}"))
+    errorsOrFinishedUploads.left foreach (err => fail(s"Encountered an error: ${err.reportMessage}"))
     val exitCode = errorsOrFinishedUploads.fold(
       _ => 1,
       finishedUploads => finishedUploads.foldLeft(0) { (memo, finishedUpload) =>
         memo + finishedUpload.fold(
-          (error: Error) => 1,
+          (error: ErrorReport) => 1,
           (failedOrSucceededUpload: Either[PushFailureReport, PushSuccessReport]) =>
             if (failedOrSucceededUpload.isLeft) 1 else 0
         )
@@ -124,7 +127,7 @@ object Push {
 
   def resolvePushCounts(implicit finishedOperations: FinishedPushOperations) = finishedOperations.foldLeft(PushCounts()) {
     (counts: PushCounts, uploadReport) => uploadReport.fold(
-      (error: model.Error) => counts.copy(failures = counts.failures + 1),
+      (error: ErrorReport) => counts.copy(failures = counts.failures + 1),
       failureOrSuccess => failureOrSuccess.fold(
         (failureReport: PushFailureReport) => counts.copy(failures = counts.failures + 1),
         (successReport: PushSuccessReport) => successReport match {
@@ -161,8 +164,8 @@ object Push {
                          redirects: Int = 0, 
                          deletes: Int = 0
                          )
-  type FinishedPushOperations = ParSeq[Either[model.Error, Either[PushFailureReport, PushSuccessReport]]]
-  type PushReports = ParSeq[Either[model.Error, Future[Either[PushFailureReport, PushSuccessReport]]]]
+  type FinishedPushOperations = ParSeq[Either[ErrorReport, Either[PushFailureReport, PushSuccessReport]]]
+  type PushReports = ParSeq[Either[ErrorReport, Future[Either[PushFailureReport, PushSuccessReport]]]]
   case class PushResult(threadPool: ExecutorService, uploadReports: PushReports)
   type ExitCode = Int
 
@@ -188,7 +191,7 @@ object Push {
         threadPool.shutdownNow()
         pushStatus
     }
-    errorOrPushStatus.left foreach (err => fail(s"Could not load the site: ${err.message}"))
+    errorOrPushStatus.left foreach (err => fail(s"Could not load the site: ${err.reportMessage}"))
     System.exit(errorOrPushStatus.fold(_ => 1, pushStatus => pushStatus))
   }
 }

data/src/main/scala/s3/website/S3.scala

@@ -5,68 +5,45 @@ import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}
 import com.amazonaws.auth.BasicAWSCredentials
 import com.amazonaws.services.s3.model._
 import scala.collection.JavaConversions._
-import scala.util.Try
-import com.amazonaws.AmazonClientException
 import scala.concurrent.{ExecutionContextExecutor, Future}
 import s3.website.S3._
 import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
 import s3.website.Logger._
 import s3.website.Utils._
+import scala.concurrent.duration.{Duration, TimeUnit}
+import java.util.concurrent.TimeUnit.SECONDS
 import s3.website.S3.SuccessfulUpload
 import s3.website.S3.SuccessfulDelete
 import s3.website.S3.FailedUpload
-import scala.util.Failure
 import scala.Some
 import s3.website.S3.FailedDelete
-import s3.website.model.IOError
-import scala.util.Success
-import s3.website.model.UserError
-import scala.concurrent.duration.{TimeUnit, Duration}
-import java.util.concurrent.TimeUnit.SECONDS
+import s3.website.S3.S3Settings
+import s3.website.model.Error.isClientError
 
 class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
 
-  def upload(upload: Upload with UploadTypeResolved)(implicit a: Attempt = 1, config: Config): S3PushResult =
+  def upload(upload: Upload with UploadTypeResolved)(implicit a: Attempt = 1, config: Config): Future[Either[FailedUpload, SuccessfulUpload]] =
     Future {
       s3Settings.s3Client(config) putObject toPutObjectRequest(upload)
       val report = SuccessfulUpload(upload)
       info(report)
       Right(report)
     } recoverWith retry(
-      createReport = error => FailedUpload(upload.s3Key, error),
+      createFailureReport = error => FailedUpload(upload.s3Key, error),
       retryAction  = newAttempt => this.upload(upload)(newAttempt, config)
     )
 
-  def delete(s3Key: String)(implicit a: Attempt = 1, config: Config): S3PushResult =
+  def delete(s3Key: String)(implicit a: Attempt = 1, config: Config): Future[Either[FailedDelete, SuccessfulDelete]] =
     Future {
       s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
       val report = SuccessfulDelete(s3Key)
       info(report)
       Right(report)
     } recoverWith retry(
-      createReport = error => FailedDelete(s3Key, error),
+      createFailureReport = error => FailedDelete(s3Key, error),
       retryAction  = newAttempt => this.delete(s3Key)(newAttempt, config)
     )
       
-  type S3PushResult = Future[Either[PushFailureReport, PushSuccessReport]]
-  
-  type Attempt = Int
-  
-  def retry(createReport: (Throwable) => PushFailureReport, retryAction: (Attempt) => S3PushResult)(implicit attempt: Attempt):
-  PartialFunction[Throwable, S3PushResult] = {
-    case error: Throwable =>
-      val failureReport = createReport(error)
-      if (attempt == 6) {
-        info(failureReport)
-        Future(Left(failureReport))
-      } else {
-        val sleepDuration = Duration(fibs.drop(attempt + 1).head, s3Settings.retrySleepTimeUnit)
-        pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
-        Thread.sleep(sleepDuration.toMillis)
-        retryAction(attempt + 1)
-      }
-  }
-
   def toPutObjectRequest(upload: Upload)(implicit config: Config) =
     upload.essence.fold(
       redirect => {
@@ -99,36 +76,49 @@ class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
 object S3 {
   def awsS3Client(config: Config) = new AmazonS3Client(new BasicAWSCredentials(config.s3_id, config.s3_secret))
 
-  def resolveS3Files(implicit config: Config, s3Settings: S3Settings): Either[Error, Stream[S3File]] = Try {
-    objectSummaries()
-  } match {
-    case Success(remoteFiles) =>
-      Right(remoteFiles)
-    case Failure(error) if error.isInstanceOf[AmazonClientException] =>
-      Left(UserError(error.getMessage))
-    case Failure(error) =>
-      Left(IOError(error))
-  }
-
-  def objectSummaries(nextMarker: Option[String] = None)(implicit config: Config, s3Settings: S3Settings): Stream[S3File] = {
+  def resolveS3Files(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil)
+                    (implicit attempt: Attempt = 1, config: Config, s3Settings: S3Settings, ec: ExecutionContextExecutor): ObjectListingResult = Future {
+    nextMarker.foreach(m => info(s"Fetching the next part of the object listing from S3 (starting from $m)"))
     val objects: ObjectListing = s3Settings.s3Client(config).listObjects({
       val req = new ListObjectsRequest()
       req.setBucketName(config.s3_bucket)
       nextMarker.foreach(req.setMarker)
       req
     })
-    val summaries = (objects.getObjectSummaries map (S3File(_))).toStream
-    if (objects.isTruncated)
-      summaries #::: objectSummaries(Some(objects.getNextMarker)) // Call the next Get Bucket request lazily
-    else
-      summaries
-  }
+    objects
+  } flatMap { objects =>
+    val s3Files = alreadyResolved ++ (objects.getObjectSummaries.toIndexedSeq.toSeq map (S3File(_)))
+    Option(objects.getNextMarker)
+      .fold(Future(Right(s3Files)): ObjectListingResult)(nextMarker => resolveS3Files(Some(nextMarker), s3Files))
+  } recoverWith retry(
+    createFailureReport = error => UserError(s"Failed to fetch an object listing (${error.getMessage})"),
+    retryAction = nextAttempt => resolveS3Files(nextMarker, alreadyResolved)(nextAttempt, config, s3Settings, ec)
+  )
+
+  type ObjectListingResult = Future[Either[ErrorReport, Seq[S3File]]]
 
   sealed trait PushFailureReport extends FailureReport
   sealed trait PushSuccessReport extends SuccessReport {
     def s3Key: String
   }
 
+  def retry[L <: Report, R](createFailureReport: (Throwable) => L, retryAction: (Attempt) => Future[Either[L, R]])
+           (implicit attempt: Attempt, s3Settings: S3Settings, ec: ExecutionContextExecutor):
+  PartialFunction[Throwable, Future[Either[L, R]]] = {
+    case error: Throwable if attempt == 6 || isClientError(error) =>
+      val failureReport = createFailureReport(error)
+      fail(failureReport.reportMessage)
+      Future(Left(failureReport))
+    case error: Throwable =>
+      val failureReport = createFailureReport(error)
+      val sleepDuration = Duration(fibs.drop(attempt + 1).head, s3Settings.retrySleepTimeUnit)
+      pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
+      Thread.sleep(sleepDuration.toMillis)
+      retryAction(attempt + 1)
+  }
+
+  type Attempt = Int
+
   case class SuccessfulUpload(upload: Upload with UploadTypeResolved) extends PushSuccessReport {
     def reportMessage =
       upload.uploadType match {

data/src/main/scala/s3/website/model/Config.scala

@@ -3,6 +3,7 @@ package s3.website.model
 import scala.util.{Failure, Try}
 import scala.collection.JavaConversions._
 import s3.website.Ruby.rubyRuntime
+import s3.website.ErrorReport
 
 case class Config(
   s3_id:                      String,
@@ -22,7 +23,7 @@ case class Config(
 )
 
 object Config {
-  def loadOptionalBooleanOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Either[Boolean, Seq[String]]]] = {
+  def loadOptionalBooleanOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Boolean, Seq[String]]]] = {
     val yamlValue = for {
       optionalValue <- loadOptionalValue(key)
     } yield {
@@ -35,7 +36,7 @@ object Config {
     yamlValue getOrElse Left(UserError(s"The key $key has to have a boolean or [string] value"))
   }
 
-  def loadOptionalStringOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Either[String, Seq[String]]]] = {
+  def loadOptionalStringOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[String, Seq[String]]]] = {
     val yamlValue = for {
       valueOption <- loadOptionalValue(key)
     } yield {
@@ -48,7 +49,7 @@ object Config {
     yamlValue getOrElse Left(UserError(s"The key $key has to have a string or [string] value"))
   }
 
-  def loadMaxAge(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Either[Int, Map[String, Int]]]] = {
+  def loadMaxAge(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Int, Map[String, Int]]]] = {
     val key = "max_age"
     val yamlValue = for {
       maxAgeOption <- loadOptionalValue(key)
@@ -62,7 +63,7 @@ object Config {
     yamlValue getOrElse Left(UserError(s"The key $key has to have an int or (string -> int) value"))
   }
 
-  def loadEndpoint(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[S3Endpoint]] =
+  def loadEndpoint(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[S3Endpoint]] =
     loadOptionalString("s3_endpoint").right flatMap { endpointString =>
       endpointString.map(S3Endpoint.forString) match {
         case Some(Right(endpoint)) => Right(Some(endpoint))
@@ -71,17 +72,17 @@ object Config {
       }
     }
 
-  def loadRedirects(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Map[String, String]]] = {
+  def loadRedirects(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Map[String, String]]] = {
     val key = "redirects"
     val yamlValue = for {
       redirectsOption <- loadOptionalValue(key)
       redirects <- Try(redirectsOption.map(_.asInstanceOf[java.util.Map[String,String]].toMap))
     } yield Right(redirects)
 
-    yamlValue getOrElse Left(UserError(s"The key $key has to have an int or (string -> int) value"))
+    yamlValue getOrElse Left(UserError(s"The key $key has to have a (string -> string) value"))
   }
 
-  def loadRequiredString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, String] = {
+  def loadRequiredString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, String] = {
     val yamlValue = for {
       valueOption <- loadOptionalValue(key)
       stringValue <- Try(valueOption.asInstanceOf[Option[String]].get)
@@ -94,7 +95,7 @@ object Config {
     }
   }
 
-  def loadOptionalString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[String]] = {
+  def loadOptionalString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[String]] = {
     val yamlValueOption = for {
       valueOption <- loadOptionalValue(key)
       optionalString <- Try(valueOption.asInstanceOf[Option[String]])
@@ -107,7 +108,7 @@ object Config {
     }
   }
 
-  def loadOptionalBoolean(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Boolean]] = {
+  def loadOptionalBoolean(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Boolean]] = {
     val yamlValueOption = for {
       valueOption <- loadOptionalValue(key)
       optionalBoolean <- Try(valueOption.asInstanceOf[Option[Boolean]])
@@ -120,7 +121,7 @@ object Config {
     }
   }
 
-  def loadOptionalInt(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Int]] = {
+  def loadOptionalInt(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Int]] = {
     val yamlValueOption = for {
       valueOption <- loadOptionalValue(key)
       optionalInt <- Try(valueOption.asInstanceOf[Option[Int]])
@@ -139,12 +140,16 @@ object Config {
     }
 
   def erbEval(erbString: String): Try[String] = Try {
-    val erbStringWithoutComments = erbString.replaceAll("#.*", "")
+    val erbStringWithoutComments = erbString.replaceAll("^\\s*#.*", "")
     rubyRuntime.evalScriptlet(
       s"""
-        require 'erb'
-        ERB.new("$erbStringWithoutComments").result
-      """
+        |require 'erb'
+        |
+        |str = <<-ERBSTR
+        |$erbStringWithoutComments
+        |ERBSTR
+        |ERB.new(str).result
+      """.stripMargin
     ).asJavaString()
   }
 

data/src/main/scala/s3/website/model/Site.scala

@@ -10,13 +10,14 @@ import s3.website.Logger._
 import scala.util.Failure
 import s3.website.model.Config.UnsafeYaml
 import scala.util.Success
+import s3.website.ErrorReport
 
 case class Site(rootDirectory: String, config: Config) {
   def resolveS3Key(file: File) = file.getAbsolutePath.replace(rootDirectory, "").replaceFirst("^/", "")
 }
 
 object Site {
-  def loadSite(yamlConfigPath: String, siteRootDirectory: String): Either[Error, Site] = {
+  def loadSite(yamlConfigPath: String, siteRootDirectory: String): Either[ErrorReport, Site] = {
     val yamlObjectTry = for {
       yamlString <- Try(fromFile(new File(yamlConfigPath)).mkString)
       yamlWithErbEvaluated <- erbEval(yamlString)
@@ -25,7 +26,7 @@ object Site {
     yamlObjectTry match {
       case Success(yamlObject) =>
         implicit val unsafeYaml = UnsafeYaml(yamlObject)
-        val config: Either[Error, Config] = for {
+        val config: Either[ErrorReport, Config] = for {
           s3_id <- loadRequiredString("s3_id").right
           s3_secret <- loadRequiredString("s3_secret").right
           s3_bucket <- loadRequiredString("s3_bucket").right

data/src/main/scala/s3/website/model/errors.scala

@@ -1,11 +1,16 @@
 package s3.website.model
 
-trait Error {
-  def message: String
+import com.amazonaws.AmazonServiceException
+import com.amazonaws.AmazonServiceException.ErrorType.Client
+import s3.website.ErrorReport
+
+object Error {
+  def isClientError(error: Throwable) =
+    error.isInstanceOf[AmazonServiceException] && error.asInstanceOf[AmazonServiceException].getErrorType == Client
 }
 
-case class UserError(message: String) extends Error
+case class UserError(reportMessage: String) extends ErrorReport
 
-case class IOError(exception: Throwable) extends Error {
-  def message = exception.getMessage
+case class IOError(exception: Throwable) extends ErrorReport {
+  def reportMessage = exception.getMessage
 }

data/src/main/scala/s3/website/model/push.scala

@@ -9,6 +9,7 @@ import java.util.zip.GZIPOutputStream
 import org.apache.commons.io.IOUtils
 import org.apache.tika.Tika
 import s3.website.Ruby._
+import s3.website._
 import s3.website.model.Encoding.Gzip
 import scala.util.Failure
 import scala.Some
@@ -59,7 +60,7 @@ case class LocalFile(
 ) extends S3KeyProvider
 
 object LocalFile {
-  def toUpload(localFile: LocalFile)(implicit config: Config): Either[Error, Upload] = Try {
+  def toUpload(localFile: LocalFile)(implicit config: Config): Either[ErrorReport, Upload] = Try {
     def fis(file: File): InputStream = new FileInputStream(file)
     def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
     val sourceFile: File = localFile
@@ -107,7 +108,7 @@ object LocalFile {
 
   lazy val tika = new Tika()
 
-  def resolveLocalFiles(implicit site: Site): Either[Error, Seq[LocalFile]] = Try {
+  def resolveLocalFiles(implicit site: Site): Either[ErrorReport, Seq[LocalFile]] = Try {
     val files = recursiveListFiles(new File(site.rootDirectory)).filterNot(_.isDirectory)
     files map { file =>
       val s3Key = site.resolveS3Key(file)

data/src/main/scala/s3/website/package.scala

@@ -7,4 +7,6 @@ package object website {
   trait SuccessReport extends Report
 
   trait FailureReport extends Report
+
+  trait ErrorReport extends Report
 }

data/src/test/scala/s3/website/S3WebsiteSpec.scala

@@ -18,13 +18,15 @@ import s3.website.S3.{S3Settings, S3ClientProvider}
 import scala.collection.JavaConversions._
 import s3.website.model.NewFile
 import scala.Some
-import com.amazonaws.AmazonServiceException
+import com.amazonaws.{AmazonClientException, AmazonServiceException}
 import org.apache.commons.codec.digest.DigestUtils.md5Hex
 import s3.website.CloudFront.{CloudFrontSettings, CloudFrontClientProvider}
 import com.amazonaws.services.cloudfront.AmazonCloudFront
 import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, CreateInvalidationRequest, TooManyInvalidationsInProgressException}
 import org.mockito.stubbing.Answer
 import org.mockito.invocation.InvocationOnMock
+import com.amazonaws.AmazonServiceException.ErrorType
+import java.util.concurrent.atomic.AtomicInteger
 
 class S3WebsiteSpec extends Specification {
 
@@ -101,6 +103,17 @@ class S3WebsiteSpec extends Specification {
       verify(amazonS3Client, times(6)).putObject(Matchers.any(classOf[PutObjectRequest]))
     }
 
+    "not try again if the upload fails on because of the client" in new SiteDirectory with MockAWS {
+      implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
+      when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow {
+        val e = new AmazonServiceException("your credentials are incorrect")
+        e.setErrorType(ErrorType.Client)
+        e
+      }
+      Push.pushSite
+      verify(amazonS3Client, times(1)).putObject(Matchers.any(classOf[PutObjectRequest]))
+    }
+
     "try again if the delete fails" in new SiteDirectory with MockAWS {
       implicit val site = buildSite()
       setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
@@ -108,6 +121,14 @@ class S3WebsiteSpec extends Specification {
       Push.pushSite
       verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
     }
+
+    "try again if the object listing fails" in new SiteDirectory with MockAWS {
+      implicit val site = buildSite()
+      setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
+      objectListingFailsAndThenSucceeds(howManyFailures = 5)
+      Push.pushSite
+      verify(amazonS3Client, times(6)).listObjects(Matchers.any(classOf[ListObjectsRequest]))
+    }
   }
 
   "push with CloudFront" should {
@@ -184,6 +205,12 @@ class S3WebsiteSpec extends Specification {
       Push.pushSite must equalTo(1)
     }
 
+    "be 1 if any of the redirects fails" in new SiteDirectory with MockAWS {
+      implicit val site = buildSite(defaultConfig.copy(redirects = Some(Map("index.php" -> "/index.html"))))
+      when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow(new AmazonServiceException("AWS failed"))
+      Push.pushSite must equalTo(1)
+    }
+
     "be 0 if CloudFront invalidations and uploads succeed"in new SiteDirectory with MockAWS {
       implicit val site = siteWithFiles(
         config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
@@ -212,6 +239,13 @@ class S3WebsiteSpec extends Specification {
       uploadFailsAndThenSucceeds(howManyFailures = 6)
       Push.pushSite must equalTo(1)
     }
+
+    "be 1 if an object listing fails" in new SiteDirectory with MockAWS {
+      implicit val site = buildSite()
+      setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
+      objectListingFailsAndThenSucceeds(howManyFailures = 6)
+      Push.pushSite must equalTo(1)
+    }
   }
 
   "s3_website.yml file" should {
@@ -408,30 +442,32 @@ class S3WebsiteSpec extends Specification {
 
     val s3 = new S3()
 
-    def uploadFailsAndThenSucceeds(howManyFailures: Int) {
-      var callCount = 0
-      doAnswer(new Answer[PutObjectResult] {
-        override def answer(invocation: InvocationOnMock) = {
-          callCount += 1
-          if (callCount <= howManyFailures)
-            throw new AmazonServiceException("AWS is temporarily down")
-          else
-            mock(classOf[PutObjectResult])
-        }
-      }).when(amazonS3Client).putObject(Matchers.anyObject())
+    def uploadFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
+      doAnswer(temporaryFailure(classOf[PutObjectResult]))
+        .when(amazonS3Client)
+        .putObject(Matchers.anyObject())
     }
 
-    def deleteFailsAndThenSucceeds(howManyFailures: Int) {
-      var callCount = 0
-      doAnswer(new Answer[DeleteObjectRequest] {
-        override def answer(invocation: InvocationOnMock) = {
-          callCount += 1
-          if (callCount <= howManyFailures)
-            throw new AmazonServiceException("AWS is temporarily down")
-          else
-            mock(classOf[DeleteObjectRequest])
-        }
-      }).when(amazonS3Client).deleteObject(Matchers.anyString(), Matchers.anyString())
+    def deleteFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
+      doAnswer(temporaryFailure(classOf[DeleteObjectRequest]))
+        .when(amazonS3Client)
+        .deleteObject(Matchers.anyString(), Matchers.anyString())
+    }
+
+    def objectListingFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
+      doAnswer(temporaryFailure(classOf[ObjectListing]))
+        .when(amazonS3Client)
+        .listObjects(Matchers.any(classOf[ListObjectsRequest]))
+    }
+
+    def temporaryFailure[T](clazz: Class[T])(implicit callCount: AtomicInteger, howManyFailures: Int) = new Answer[T] {
+      def answer(invocation: InvocationOnMock) = {
+        callCount.incrementAndGet()
+        if (callCount.get() <= howManyFailures)
+          throw new AmazonServiceException("AWS is temporarily down")
+        else
+          mock(clazz)
+      }
     }
 
     def asSeenByS3Client(upload: Upload)(implicit config: Config): PutObjectRequest = {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_website_monadic
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-07 00:00:00.000000000 Z
+date: 2014-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk