s3_website_monadic 0.0.31 → 0.0.32

data/src/main/scala/s3/website/Push.scala

@@ -4,15 +4,13 @@ import s3.website.model.Site._
 import scala.concurrent.{ExecutionContextExecutor, Future, Await}
 import scala.concurrent.duration._
 import scala.language.postfixOps
-import s3.website.Diff.{resolveNewFiles, resolveDeletes}
+import s3.website.Diff.{resolveDeletes, resolveDiff}
 import s3.website.S3._
 import scala.concurrent.ExecutionContext.fromExecutor
 import java.util.concurrent.Executors.newFixedThreadPool
-import s3.website.model.LocalFile.resolveLocalFiles
-import scala.collection.parallel.ParSeq
 import java.util.concurrent.ExecutorService
 import s3.website.model._
-import s3.website.model.Update
+import s3.website.model.FileUpdate
 import s3.website.model.NewFile
 import s3.website.S3.PushSuccessReport
 import scala.collection.mutable.ArrayBuffer
@@ -27,6 +25,7 @@ import scala.Int
 import java.io.File
 import com.lexicalscope.jewel.cli.CliFactory.parseArguments
 import s3.website.ByteHelper.humanReadableByteCount
+import s3.website.S3.SuccessfulUpload.humanizeUploadSpeed
 
 object Push {
 
@@ -83,27 +82,32 @@ object Push {
                 pushMode: PushMode
                 ): ExitCode = {
     logger.info(s"${Deploy.renderVerb} ${site.rootDirectory}/* to ${site.config.s3_bucket}")
-    val utils = new Utils
-
     val redirects = Redirect.resolveRedirects
-    val redirectResults = redirects.map(new S3() upload(_))
+    val s3FilesFuture = resolveS3Files()
+    val redirectReports: PushReports = redirects.map(S3 uploadRedirect _) map (Right(_))
 
-    val errorsOrReports = for {
-      localFiles        <- resolveLocalFiles.right
-      errorOrS3FilesAndUpdateFutures <- Await.result(resolveS3FilesAndUpdates(localFiles)(), 7 days).right
-      s3Files <- errorOrS3FilesAndUpdateFutures._1.right
+    val errorsOrReports: Either[ErrorReport, PushReports] = for {
+      diff <- resolveDiff(s3FilesFuture).right
     } yield {
-      val updateReports: PushReports = errorOrS3FilesAndUpdateFutures._2.par
-      val deleteReports: PushReports = utils toParSeq resolveDeletes(localFiles, s3Files, redirects)
-        .map { s3File => new S3() delete s3File.s3Key }
-        .map { Right(_) } // To make delete reports type-compatible with upload reports
-      val uploadReports: PushReports = utils toParSeq resolveNewFiles(localFiles, s3Files)
-        .map { _.right.map(new S3() upload(_)) }
-      uploadReports ++ deleteReports ++ updateReports ++ redirectResults.map(Right(_))
-    }
-    val errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations] = errorsOrReports.right map {
-      uploadReports => awaitForUploads(uploadReports)
+      val newOrChangedReports: PushReports = diff.uploads.map { uploadBatch =>
+        uploadBatch.map(_.right.map(_.map(S3 uploadFile _)))
+      }.map (Await.result(_, 1 day)).foldLeft(Seq(): PushReports) { (memo: PushReports, res: Either[ErrorReport, Seq[Future[PushErrorOrSuccess]]]) =>
+        res.fold(
+          error => memo :+ Left(error),
+          (pushResults: Seq[Future[PushErrorOrSuccess]]) => memo ++ (pushResults map (Right(_)))
+        )
+      }
+      val deleteReports =
+        Await.result(resolveDeletes(diff, s3FilesFuture, redirects), 1 day).right.map { keysToDelete =>
+          keysToDelete map (S3 delete _)
+        }.fold(
+          error => Left(error) :: Nil,
+          (pushResults: Seq[Future[PushErrorOrSuccess]]) => pushResults map (Right(_))
+        )
+      val diffErrorReport: PushReports = Await.result(diff.persistenceError, 1 day).fold(Nil: PushReports)(Left(_) :: Nil)
+      newOrChangedReports ++ deleteReports ++ redirectReports ++ diffErrorReport
     }
+    val errorsOrFinishedPushOps = errorsOrReports.right map awaitForResults
     val invalidationSucceeded = invalidateCloudFrontItems(errorsOrFinishedPushOps)
     
     afterPushFinished(errorsOrFinishedPushOps, invalidationSucceeded)
@@ -112,40 +116,37 @@ object Push {
   def invalidateCloudFrontItems
     (errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations])
     (implicit config: Config, cloudFrontSettings: CloudFrontSetting, ec: ExecutionContextExecutor, logger: Logger, pushMode: PushMode):
-  Option[InvalidationSucceeded] = {
-    config.cloudfront_distribution_id.map {
-      distributionId =>
-        val pushSuccessReports = errorsOrFinishedPushOps.fold(
-          errors => Nil,
-          finishedPushOps => {
-            finishedPushOps.map {
-              ops =>
-                for {
-                  failedOrSucceededPushes <- ops.right
-                  successfulPush <- failedOrSucceededPushes.right
-                } yield successfulPush
-            }.foldLeft(Seq(): Seq[PushSuccessReport]) {
-              (reports, failOrSucc) =>
-                failOrSucc.fold(
-                  _ => reports,
-                  (pushSuccessReport: PushSuccessReport) => reports :+ pushSuccessReport
-                )
-            }
-          }
-        )
-        val invalidationResults: Seq[Either[FailedInvalidation, SuccessfulInvalidation]] =
-          toInvalidationBatches(pushSuccessReports) map { invalidationBatch =>
-            Await.result(
-              new CloudFront().invalidate(invalidationBatch, distributionId),
-              atMost = 1 day
-            )
+  Option[InvalidationSucceeded] =
+    config.cloudfront_distribution_id.map { distributionId =>
+      val pushSuccessReports = errorsOrFinishedPushOps.fold(
+        errors => Nil,
+        finishedPushOps =>
+          finishedPushOps.map {
+            ops =>
+              for {
+                failedOrSucceededPushes <- ops.right
+                successfulPush <- failedOrSucceededPushes.right
+              } yield successfulPush
+          }.foldLeft(Seq(): Seq[PushSuccessReport]) {
+            (reports, failOrSucc) =>
+              failOrSucc.fold(
+                _ => reports,
+                (pushSuccessReport: PushSuccessReport) => reports :+ pushSuccessReport
+              )
           }
-        if (invalidationResults.exists(_.isLeft))
-          false // If one of the invalidations failed, mark the whole process as failed
-        else
-          true
+      )
+      val invalidationResults: Seq[Either[FailedInvalidation, SuccessfulInvalidation]] =
+        toInvalidationBatches(pushSuccessReports) map { invalidationBatch =>
+          Await.result(
+            CloudFront.invalidate(invalidationBatch, distributionId),
+            atMost = 1 day
+          )
+        }
+      if (invalidationResults.exists(_.isLeft))
+        false // If one of the invalidations failed, mark the whole process as failed
+      else
+        true
     }
-  }
 
   type InvalidationSucceeded = Boolean
 
@@ -177,7 +178,7 @@ object Push {
     exitCode
   }
 
-  def awaitForUploads(uploadReports: PushReports)(implicit executor: ExecutionContextExecutor): FinishedPushOperations =
+  def awaitForResults(uploadReports: PushReports)(implicit executor: ExecutionContextExecutor): FinishedPushOperations =
     uploadReports map (_.right.map {
       rep => Await.result(rep, 1 day)
     })
@@ -188,12 +189,13 @@ object Push {
         (error: ErrorReport) => counts.copy(failures = counts.failures + 1),
         failureOrSuccess => failureOrSuccess.fold(
           (failureReport: PushFailureReport) => counts.copy(failures = counts.failures + 1),
-          (successReport: PushSuccessReport) => successReport match {
-            case succ: SuccessfulUpload => succ.upload.uploadType match {
-              case NewFile => counts.copy(newFiles = counts.newFiles + 1).addTransferStats(succ) // TODO nasty repetition here
-              case Update => counts.copy(updates = counts.updates + 1).addTransferStats(succ)
-              case Redirect => counts.copy(redirects = counts.redirects + 1).addTransferStats(succ)
-            }
+          (successReport: PushSuccessReport) =>
+            successReport match {
+              case succ: SuccessfulUpload => succ.source.fold(_.uploadType, _.uploadType) match {
+                case NewFile      => counts.copy(newFiles = counts.newFiles + 1).addTransferStats(succ) // TODO nasty repetition here
+                case FileUpdate   => counts.copy(updates = counts.updates + 1).addTransferStats(succ)
+                case RedirectFile => counts.copy(redirects = counts.redirects + 1).addTransferStats(succ)
+              }
             case SuccessfulDelete(_) => counts.copy(deletes = counts.deletes + 1)
           }
         )
@@ -205,7 +207,7 @@ object Push {
       case PushCounts(updates, newFiles, failures, redirects, deletes, _, _)
         if updates == 0 && newFiles == 0 && failures == 0 && redirects == 0 && deletes == 0 =>
           PushNothing.renderVerb
-      case PushCounts(updates, newFiles, failures, redirects, deletes, uploadedBytes, uploadDurationAndFrequency) =>
+      case PushCounts(updates, newFiles, failures, redirects, deletes, uploadedBytes, uploadDurations) =>
         val reportClauses: scala.collection.mutable.ArrayBuffer[String] = ArrayBuffer()
         if (updates > 0)       reportClauses += s"${Updated.renderVerb} ${updates ofType "file"}."
         if (newFiles > 0)      reportClauses += s"${Created.renderVerb} ${newFiles ofType "file"}."
@@ -213,14 +215,7 @@ object Push {
         if (redirects > 0)     reportClauses += s"${Applied.renderVerb} ${redirects ofType "redirect"}."
         if (deletes > 0)       reportClauses += s"${Deleted.renderVerb} ${deletes ofType "file"}."
         if (uploadedBytes > 0) {
-          val transferSuffix =
-            if (uploadDurationAndFrequency._1.getStandardSeconds > 0)
-              s", ${humanReadableByteCount(
-                (uploadedBytes / uploadDurationAndFrequency._1.getMillis * 1000) * uploadDurationAndFrequency._2
-              )}/s."
-            else
-              "."
-
+          val transferSuffix = humanizeUploadSpeed(uploadedBytes, uploadDurations: _*).fold(".")(speed => s", $speed.")
           reportClauses += s"${Transferred.renderVerb} ${humanReadableByteCount(uploadedBytes)}$transferSuffix"
         }
         reportClauses.mkString(" ")
@@ -233,22 +228,19 @@ object Push {
                          redirects: Int = 0, 
                          deletes: Int = 0,
                          uploadedBytes: Long = 0,
-                         uploadDurationAndFrequency: (org.joda.time.Duration, Int) = (new org.joda.time.Duration(0), 0)
+                         uploadDurations: Seq[org.joda.time.Duration] = Nil
                          ) {
     val thereWasSomethingToPush = updates + newFiles + redirects + deletes > 0
 
-    def addTransferStats(successfulUpload: SuccessfulUpload): PushCounts = {
+    def addTransferStats(successfulUpload: SuccessfulUpload): PushCounts =
       copy(
         uploadedBytes = uploadedBytes + (successfulUpload.uploadSize getOrElse 0L),
-        uploadDurationAndFrequency = successfulUpload.uploadDuration.fold(uploadDurationAndFrequency)(
-          dur => (uploadDurationAndFrequency._1.plus(dur), uploadDurationAndFrequency._2 + 1)
-        )
+        uploadDurations = uploadDurations ++ successfulUpload.uploadDuration
       )
-    }
   }
 
-  type FinishedPushOperations = ParSeq[Either[ErrorReport, PushErrorOrSuccess]]
-  type PushReports = ParSeq[Either[ErrorReport, Future[PushErrorOrSuccess]]]
+  type FinishedPushOperations = Seq[Either[ErrorReport, PushErrorOrSuccess]]
+  type PushReports = Seq[Either[ErrorReport, Future[PushErrorOrSuccess]]]
   case class PushResult(threadPool: ExecutorService, uploadReports: PushReports)
   type ExitCode = Int
 }

data/src/main/scala/s3/website/S3.scala

@@ -19,26 +19,38 @@ import scala.concurrent.duration.TimeUnit
 import java.util.concurrent.TimeUnit
 import scala.concurrent.duration.TimeUnit
 import java.util.concurrent.TimeUnit.SECONDS
+import s3.website.S3.SuccessfulUpload.humanizeUploadSpeed
+import java.io.FileInputStream
 
-class S3(implicit s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger) {
+object S3 {
+
+  def uploadRedirect(redirect: Redirect, a: Attempt = 1)
+            (implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger) =
+    upload(Right(redirect))
+
+  def uploadFile(localFile: LocalFileFromDisk, a: Attempt = 1)
+                    (implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger) =
+    upload(Left(localFile))
 
-  def upload(upload: Upload with UploadTypeResolved, a: Attempt = 1)
-            (implicit config: Config): Future[Either[FailedUpload, SuccessfulUpload]] =
+  def upload(source: Either[LocalFileFromDisk, Redirect], a: Attempt = 1)
+            (implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger):
+  Future[Either[FailedUpload, SuccessfulUpload]] =
     Future {
-      val putObjectRequest = toPutObjectRequest(upload)
+      val putObjectRequest = toPutObjectRequest(source)
       val uploadDuration =
         if (pushMode.dryRun) None
         else recordUploadDuration(putObjectRequest, s3Settings.s3Client(config) putObject putObjectRequest)
-      val report = SuccessfulUpload(upload, putObjectRequest, uploadDuration)
+      val report = SuccessfulUpload(source, putObjectRequest, uploadDuration)
       logger.info(report)
       Right(report)
     } recoverWith retry(a)(
-      createFailureReport = error => FailedUpload(upload.s3Key, error),
-      retryAction  = newAttempt => this.upload(upload, newAttempt)
+      createFailureReport = error => FailedUpload(source.fold(_.s3Key, _.s3Key), error),
+      retryAction  = newAttempt => this.upload(source, newAttempt)
     )
 
-  def delete(s3Key: String,  a: Attempt = 1)
-            (implicit config: Config): Future[Either[FailedDelete, SuccessfulDelete]] =
+  def delete(s3Key: S3Key,  a: Attempt = 1)
+            (implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger):
+  Future[Either[FailedDelete, SuccessfulDelete]] =
     Future {
       if (!pushMode.dryRun) s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
       val report = SuccessfulDelete(s3Key)
@@ -49,10 +61,27 @@ class S3(implicit s3Settings: S3Setting, pushMode: PushMode, executor: Execution
       retryAction  = newAttempt => this.delete(s3Key, newAttempt)
     )
 
-  def toPutObjectRequest(upload: Upload)(implicit config: Config) =
-    upload.essence.fold(
+  def toPutObjectRequest(source: Either[LocalFileFromDisk, Redirect])(implicit config: Config) =
+    source.fold(
+      localFile => {
+        val md = new ObjectMetadata()
+        md setContentLength localFile.uploadFile.length
+        md setContentType localFile.contentType
+        localFile.encodingOnS3.map(_ => "gzip") foreach md.setContentEncoding
+        localFile.maxAge foreach { seconds =>
+          md.setCacheControl(
+            if (seconds == 0)
+              s"no-cache; max-age=$seconds"
+            else
+              s"max-age=$seconds"
+          )
+        }
+        val req = new PutObjectRequest(config.s3_bucket, localFile.s3Key, new FileInputStream(localFile.uploadFile), md)
+        config.s3_reduced_redundancy.filter(_ == true) foreach (_ => req setStorageClass ReducedRedundancy)
+        req
+      },
       redirect => {
-        val req = new PutObjectRequest(config.s3_bucket, upload.s3Key, redirect.redirectTarget)
+        val req = new PutObjectRequest(config.s3_bucket, redirect.s3Key, redirect.redirectTarget)
         req.setMetadata({
           val md = new ObjectMetadata()
           md.setContentLength(0) // Otherwise the AWS SDK will log a warning
@@ -64,23 +93,6 @@ class S3(implicit s3Settings: S3Setting, pushMode: PushMode, executor: Execution
           md
         })
         req
-      },
-      uploadBody => {
-        val md = new ObjectMetadata()
-        md setContentLength uploadBody.contentLength
-        md setContentType uploadBody.contentType
-        uploadBody.contentEncoding foreach md.setContentEncoding
-        uploadBody.maxAge foreach { seconds =>
-          md.setCacheControl(
-            if (seconds == 0)
-              s"no-cache; max-age=$seconds"
-            else
-              s"max-age=$seconds"
-          )
-        }
-        val req = new PutObjectRequest(config.s3_bucket, upload.s3Key, uploadBody.openInputStream(), md)
-        config.s3_reduced_redundancy.filter(_ == true) foreach (_ => req setStorageClass ReducedRedundancy)
-        req
       }
     )
 
@@ -92,15 +104,12 @@ class S3(implicit s3Settings: S3Setting, pushMode: PushMode, executor: Execution
     else
       None // We are not interested in tracking durations of PUT requests that don't contain data. Redirect is an example of such request.
   }
-}
 
-object S3 {
   def awsS3Client(config: Config) = new AmazonS3Client(new BasicAWSCredentials(config.s3_id, config.s3_secret))
 
-  def resolveS3FilesAndUpdates(localFiles: Seq[LocalFile])
-                              (nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil,  attempt: Attempt = 1, onFlightUpdateFutures: UpdateFutures = Nil)
+  def resolveS3Files(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil,  attempt: Attempt = 1)
                               (implicit config: Config, s3Settings: S3Setting, ec: ExecutionContextExecutor, logger: Logger, pushMode: PushMode):
-  ErrorOrS3FilesAndUpdates = Future {
+  Future[Either[ErrorReport, Seq[S3File]]] = Future {
     logger.debug(nextMarker.fold
       ("Querying S3 files")
       {m => s"Querying more S3 files (starting from $m)"}
@@ -111,35 +120,17 @@ object S3 {
       nextMarker.foreach(req.setMarker)
       req
     })
-    val summaryIndex = objects.getObjectSummaries.map { summary => (summary.getETag, summary.getKey) }.toSet // Index to avoid O(n^2) lookups 
-    def shouldUpdate(lf: LocalFile) =
-      summaryIndex.exists((md5AndS3Key) => 
-        md5AndS3Key._1 != lf.md5 && md5AndS3Key._2 == lf.s3Key
-      )
-    val updateFutures: UpdateFutures = localFiles.collect {
-      case lf: LocalFile if shouldUpdate(lf) =>
-        val errorOrUpdate = LocalFile
-          .toUpload(lf)
-          .right
-          .map { (upload: Upload) =>
-            upload.withUploadType(Update)
-          }
-        errorOrUpdate.right.map(update => new S3 upload update)
-    }
-
-    (objects, onFlightUpdateFutures ++ updateFutures)
-  } flatMap { (objectsAndUpdateFutures) =>
-    val objects: ObjectListing = objectsAndUpdateFutures._1
-    val updateFutures: UpdateFutures = objectsAndUpdateFutures._2
+    objects
+  } flatMap { (objects: ObjectListing) =>
     val s3Files = alreadyResolved ++ (objects.getObjectSummaries.toIndexedSeq.toSeq map (S3File(_)))
     Option(objects.getNextMarker)
-      .fold(Future(Right((Right(s3Files), updateFutures))): ErrorOrS3FilesAndUpdates) // We've received all the S3 keys from the bucket
+      .fold(Future(Right(s3Files)): Future[Either[ErrorReport, Seq[S3File]]]) // We've received all the S3 keys from the bucket
       { nextMarker => // There are more S3 keys on the bucket. Fetch them.
-        resolveS3FilesAndUpdates(localFiles)(Some(nextMarker), s3Files, attempt = attempt, updateFutures)
+        resolveS3Files(Some(nextMarker), s3Files, attempt = attempt)
       }
   } recoverWith retry(attempt)(
-    createFailureReport = error => ClientError(s"Failed to fetch an object listing (${error.getMessage})"),
-    retryAction = nextAttempt => resolveS3FilesAndUpdates(localFiles)(nextMarker, alreadyResolved, nextAttempt, onFlightUpdateFutures)
+    createFailureReport = error => ErrorReport(s"Failed to fetch an object listing (${error.getMessage})"),
+    retryAction = nextAttempt => resolveS3Files(nextMarker, alreadyResolved, nextAttempt)
   )
 
   type S3FilesAndUpdates = (ErrorOrS3Files, UpdateFutures)
@@ -153,13 +144,13 @@ object S3 {
     def s3Key: String
   }
 
-  case class SuccessfulUpload(upload: Upload with UploadTypeResolved, putObjectRequest: PutObjectRequest, uploadDuration: Option[Duration])
+  case class SuccessfulUpload(source: Either[LocalFileFromDisk, Redirect], putObjectRequest: PutObjectRequest, uploadDuration: Option[Duration])
                              (implicit pushMode: PushMode, logger: Logger) extends PushSuccessReport {
     def reportMessage =
-      upload.uploadType match {
-        case NewFile  => s"${Created.renderVerb} $s3Key ($reportDetails)"
-        case Update   => s"${Updated.renderVerb} $s3Key ($reportDetails)"
-        case Redirect => s"${Redirected.renderVerb} ${upload.essence.left.get.key} to ${upload.essence.left.get.redirectTarget}"
+      source.fold(_.uploadType, (redirect: Redirect) => redirect) match {
+        case NewFile                         => s"${Created.renderVerb} $s3Key ($reportDetails)"
+        case FileUpdate                      => s"${Updated.renderVerb} $s3Key ($reportDetails)"
+        case Redirect(s3Key, redirectTarget) => s"${Redirected.renderVerb} $s3Key to $redirectTarget"
       }
 
     def reportDetails = {
@@ -177,24 +168,38 @@ object S3 {
       }.mkString(" | ")
     }
 
+    def s3Key = source.fold(_.s3Key, _.s3Key)
+
     lazy val uploadSize: Option[Long] =
-      upload.essence.fold(
-        (redirect: Redirect) => None,
-        uploadBody           => Some(uploadBody.contentLength)
-      )
+      source.fold(
+        (localFile: LocalFileFromDisk) => Some(localFile.uploadFile.length()),
+        (redirect: Redirect)           => None
+    )
 
     lazy val uploadSizeForHumans: Option[String] = uploadSize filter (_ => logger.verboseOutput) map humanReadableByteCount
 
-    lazy val uploadSpeed: Option[Long] = for {
-      dataSize <- uploadSize
-      duration <- uploadDuration
-    } yield (dataSize / (duration.getMillis max 1)) * 1000 // Precision tweaking and avoidance of divide-by-zero
-
-    lazy val uploadSpeedForHumans: Option[String] = uploadSpeed filter (_ => logger.verboseOutput) map {
-      bytesPerSecond => s"${humanReadableByteCount(bytesPerSecond)}/s"
+    lazy val uploadSpeedForHumans: Option[String] =
+      (for {
+        dataSize <- uploadSize
+        duration <- uploadDuration
+      } yield {
+        humanizeUploadSpeed(dataSize, duration)
+      }) flatMap identity filter (_ => logger.verboseOutput)
+  }
+  
+  object SuccessfulUpload {
+    def humanizeUploadSpeed(uploadedBytes: Long, uploadDurations: Duration*): Option[String] = {
+      val totalDurationMillis = uploadDurations.foldLeft(new org.joda.time.Duration(0)){ (memo, duration) =>
+        memo.plus(duration)
+      }.getMillis // retain precision by using milliseconds
+      if (totalDurationMillis > 0) {
+        val bytesPerMillisecond = uploadedBytes / totalDurationMillis
+        val bytesPerSecond = bytesPerMillisecond * 1000 * uploadDurations.length
+        Some(humanReadableByteCount(bytesPerSecond) + "/s")  
+      } else {
+        None
+      }
     }
-
-    def s3Key = upload.s3Key
   }
 
   case class SuccessfulDelete(s3Key: String)(implicit pushMode: PushMode) extends PushSuccessReport {

data/src/main/scala/s3/website/model/Config.scala

@@ -33,7 +33,7 @@ object Config {
       })
     }
 
-    yamlValue getOrElse Left(ClientError(s"The key $key has to have a boolean or [string] value"))
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have a boolean or [string] value"))
   }
 
   def loadOptionalStringOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[String, Seq[String]]]] = {
@@ -46,7 +46,7 @@ object Config {
       })
     }
 
-    yamlValue getOrElse Left(ClientError(s"The key $key has to have a string or [string] value"))
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have a string or [string] value"))
   }
 
   def loadMaxAge(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Int, Map[String, Int]]]] = {
@@ -60,7 +60,7 @@ object Config {
       })
     }
 
-    yamlValue getOrElse Left(ClientError(s"The key $key has to have an int or (string -> int) value"))
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have an int or (string -> int) value"))
   }
 
   def loadEndpoint(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[S3Endpoint]] =
@@ -79,7 +79,7 @@ object Config {
       redirects <- Try(redirectsOption.map(_.asInstanceOf[java.util.Map[String,String]].toMap))
     } yield Right(redirects)
 
-    yamlValue getOrElse Left(ClientError(s"The key $key has to have a (string -> string) value"))
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have a (string -> string) value"))
   }
 
   def loadRequiredString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, String] = {
@@ -91,7 +91,7 @@ object Config {
     }
 
     yamlValue getOrElse {
-      Left(ClientError(s"The key $key has to have a string value"))
+      Left(ErrorReport(s"The key $key has to have a string value"))
     }
   }
 
@@ -104,7 +104,7 @@ object Config {
     }
 
     yamlValueOption getOrElse {
-      Left(ClientError(s"The key $key has to have a string value"))
+      Left(ErrorReport(s"The key $key has to have a string value"))
     }
   }
 
@@ -117,7 +117,7 @@ object Config {
     }
 
     yamlValueOption getOrElse {
-      Left(ClientError(s"The key $key has to have a boolean value"))
+      Left(ErrorReport(s"The key $key has to have a boolean value"))
     }
   }
 
@@ -130,7 +130,7 @@ object Config {
     }
 
     yamlValueOption getOrElse {
-      Left(ClientError(s"The key $key has to have an integer value"))
+      Left(ErrorReport(s"The key $key has to have an integer value"))
     }
   }
 

data/src/main/scala/s3/website/model/S3Endpoint.scala

@@ -1,5 +1,7 @@
 package s3.website.model
 
+import s3.website.ErrorReport
+
 case class S3Endpoint(
   s3WebsiteHostname: String,
   s3Hostname: String
@@ -8,7 +10,7 @@ case class S3Endpoint(
 object S3Endpoint {
   val defaultEndpoint = S3Endpoint("s3-website-us-east-1.amazonaws.com", "s3.amazonaws.com")
 
-  def forString(locationConstraint: String): Either[ClientError, S3Endpoint] = locationConstraint match {
+  def forString(locationConstraint: String): Either[ErrorReport, S3Endpoint] = locationConstraint match {
     case "EU" | "eu-west-1" => Right(S3Endpoint("s3-website-eu-west-1.amazonaws.com",      "s3-eu-west-1.amazonaws.com"))
     case "us-east-1" =>        Right(defaultEndpoint)
     case "us-west-1" =>        Right(S3Endpoint("s3-website-us-west-1.amazonaws.com",      "s3-us-west-1.amazonaws.com"))
@@ -17,6 +19,6 @@ object S3Endpoint {
     case "ap-southeast-2" =>   Right(S3Endpoint("s3-website-ap-southeast-2.amazonaws.com", "s3-ap-southeast-2.amazonaws.com"))
     case "ap-northeast-1" =>   Right(S3Endpoint("s3-website-ap-northeast-1.amazonaws.com", "s3-ap-northeast-1.amazonaws.com"))
     case "sa-east-1" =>        Right(S3Endpoint("s3-website-sa-east-1.amazonaws.com",      "s3-sa-east-1.amazonaws.com"))
-    case _ =>                  Left(ClientError(s"Unrecognised endpoint: $locationConstraint"))
+    case _ =>                  Left(ErrorReport(s"Unrecognised endpoint: $locationConstraint"))
   }
 }

data/src/main/scala/s3/website/model/Site.scala

@@ -6,14 +6,17 @@ import org.yaml.snakeyaml.Yaml
 import s3.website.model.Config._
 import scala.io.Source.fromFile
 import scala.language.postfixOps
-import s3.website.Logger
+import s3.website.{S3Key, Logger, ErrorReport}
 import scala.util.Failure
 import s3.website.model.Config.UnsafeYaml
 import scala.util.Success
-import s3.website.ErrorReport
 
 case class Site(rootDirectory: String, config: Config) {
   def resolveS3Key(file: File) = file.getAbsolutePath.replace(rootDirectory, "").replaceFirst("^/", "")
+
+  def resolveFile(s3File: S3File): File = resolveFile(s3File.s3Key)
+
+  def resolveFile(s3Key: S3Key): File = new File(s"$rootDirectory/$s3Key")
 }
 
 object Site {
@@ -68,7 +71,7 @@ object Site {
 
         config.right.map(Site(siteRootDirectory, _))
       case Failure(error) =>
-        Left(IOError(error))
+        Left(ErrorReport(error))
     }
   }
 }