s3_website_revived 4.0.0

data/src/main/scala/s3/website/S3.scala

@@ -0,0 +1,239 @@
+package s3.website
+
+import s3.website.ErrorReport.errorMessage
+import s3.website.model._
+import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}
+import com.amazonaws.services.s3.model._
+import scala.collection.JavaConversions._
+import scala.concurrent.{ExecutionContextExecutor, Future}
+import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
+import s3.website.ByteHelper.humanReadableByteCount
+import scala.concurrent.duration.TimeUnit
+import java.util.concurrent.TimeUnit.SECONDS
+import s3.website.S3.SuccessfulUpload.humanizeUploadSpeed
+import java.io.FileInputStream
+import s3.website.model.Config.awsCredentials
+import scala.util.Try
+
+object S3 {
+
+  def uploadRedirect(redirect: Redirect, a: Attempt = 1)
+            (implicit config: Config, s3Settings: S3Setting, pushOptions: PushOptions, executor: ExecutionContextExecutor, logger: Logger) =
+    uploadToS3(Right(redirect))
+
+  def uploadFile(up: Upload, a: Attempt = 1)
+                    (implicit config: Config, s3Settings: S3Setting, pushOptions: PushOptions, executor: ExecutionContextExecutor, logger: Logger) =
+    uploadToS3(Left(up))
+
+  def uploadToS3(source: Either[Upload, Redirect], a: Attempt = 1)
+            (implicit config: Config, s3Settings: S3Setting, pushOptions: PushOptions, executor: ExecutionContextExecutor, logger: Logger):
+  Future[Either[FailedUpload, SuccessfulUpload]] =
+    Future {
+      val putObjectRequest = toPutObjectRequest(source).get
+      val uploadDuration =
+        if (pushOptions.dryRun) None
+        else Some(recordUploadDuration(putObjectRequest, s3Settings.s3Client(config) putObject putObjectRequest))
+      val report = SuccessfulUpload(
+        source.fold(_.s3Key, _.s3Key),
+        source.fold(
+          upload => Left(SuccessfulNewOrCreatedDetails(upload.uploadType, upload.uploadFile.get.length(), uploadDuration)),
+          redirect  => Right(SuccessfulRedirectDetails(redirect.uploadType, redirect.redirectTarget))
+        ),
+        putObjectRequest
+      )
+      logger.info(report)
+      Right(report)
+    } recoverWith retry(a)(
+      createFailureReport = error => FailedUpload(source.fold(_.s3Key, _.s3Key), error),
+      retryAction  = newAttempt => this.uploadToS3(source, newAttempt)
+    )
+
+  def delete(s3Key: S3Key,  a: Attempt = 1)
+            (implicit config: Config, s3Settings: S3Setting, pushOptions: PushOptions, executor: ExecutionContextExecutor, logger: Logger):
+  Future[Either[FailedDelete, SuccessfulDelete]] =
+    Future {
+      if (!pushOptions.dryRun) s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key.key)
+      val report = SuccessfulDelete(s3Key)
+      logger.info(report)
+      Right(report)
+    } recoverWith retry(a)(
+      createFailureReport = error => FailedDelete(s3Key, error),
+      retryAction  = newAttempt => this.delete(s3Key, newAttempt)
+    )
+
+  def toPutObjectRequest(source: Either[Upload, Redirect])(implicit config: Config, logger: Logger): Try[PutObjectRequest] =
+    source.fold(
+      upload =>
+        for {
+          uploadFile <- upload.uploadFile
+          contentType <- upload.contentType
+        } yield {
+          val md = new ObjectMetadata()
+          md setContentLength uploadFile.length
+          md setContentType contentType
+          upload.encodingOnS3.map(_ => "gzip") foreach md.setContentEncoding
+          val cacheControl: Option[String] = (upload.maxAge, upload.cacheControl) match {
+            case (maxAge: Some[Int], cacheCtrl: Some[String]) =>
+              logger.warn("Overriding the max_age setting with the cache_control setting")
+              cacheCtrl
+            case (_, cacheCtrl: Some[String]) =>
+              cacheCtrl
+            case (maxAgeSeconds: Some[int], None) =>
+              maxAgeSeconds.map({
+                case seconds if seconds == 0 => s"no-cache, max-age=0"
+                case seconds                 => s"max-age=$seconds"
+              })
+            case (None, None) => None
+          }
+          cacheControl foreach { md.setCacheControl }
+          val req = new PutObjectRequest(config.s3_bucket, upload.s3Key.key, new FileInputStream(uploadFile), md)
+          config.s3_reduced_redundancy.filter(_ == true) foreach (_ => req setStorageClass ReducedRedundancy)
+          req
+        }
+      ,
+      redirect => {
+        val req = new PutObjectRequest(config.s3_bucket, redirect.s3Key.key, redirect.redirectTarget)
+        req.setMetadata({
+          val md = new ObjectMetadata()
+          md.setContentLength(0) // Otherwise the AWS SDK will log a warning
+          /*
+           * Instruct HTTP clients to always re-check the redirect. The 301 status code may override this, though.
+           * This is for the sake of simplicity.
+           */
+          md.setCacheControl("max-age=0, no-cache")
+          md
+        })
+        Try(req)
+      }
+    )
+
+  def recordUploadDuration(putObjectRequest: PutObjectRequest, f: => Unit): UploadDuration = {
+    val start = System.currentTimeMillis()
+    f
+    System.currentTimeMillis - start
+  }
+
+  def awsS3Client(config: Config) = new AmazonS3Client(awsCredentials(config))
+
+  def resolveS3Files(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil,  attempt: Attempt = 1)
+                              (implicit site: Site, s3Settings: S3Setting, ec: ExecutionContextExecutor, logger: Logger, pushOptions: PushOptions):
+  Future[Either[ErrorReport, Seq[S3File]]] = Future {
+    logger.debug(nextMarker.fold
+      ("Querying S3 files")
+      {m => s"Querying more S3 files (starting from $m)"}
+    )
+    val objects: ObjectListing = s3Settings.s3Client(site.config).listObjects({
+      val req = new ListObjectsRequest()
+      req.setBucketName(site.config.s3_bucket)
+      nextMarker.foreach(req.setMarker)
+      req
+    })
+    objects
+  } flatMap { (objects: ObjectListing) =>
+
+    /**
+     * We could filter the keys by prefix already on S3, but unfortunately s3_website test infrastructure does not currently support testing of that.
+     * Hence fetch all the keys from S3 and then filter by s3_key_prefix.
+     */
+    def matchesPrefix(os: S3ObjectSummary) = site.config.s3_key_prefix.fold(true)(prefix => os.getKey.startsWith(prefix))
+
+    val s3Files = alreadyResolved ++ (objects.getObjectSummaries.filter(matchesPrefix).toIndexedSeq.toSeq map (S3File(_)))
+    Option(objects.getNextMarker)
+      .fold(Future(Right(s3Files)): Future[Either[ErrorReport, Seq[S3File]]]) // We've received all the S3 keys from the bucket
+      { nextMarker => // There are more S3 keys on the bucket. Fetch them.
+        resolveS3Files(Some(nextMarker), s3Files, attempt = attempt)
+      }
+  } recoverWith retry(attempt)(
+    createFailureReport = error => ErrorReport(s"Failed to fetch an object listing (${error.getMessage})"),
+    retryAction = nextAttempt => resolveS3Files(nextMarker, alreadyResolved, nextAttempt)
+  )
+
+  type S3FilesAndUpdates = (ErrorOrS3Files, UpdateFutures)
+  type S3FilesAndUpdatesFuture = Future[S3FilesAndUpdates]
+  type ErrorOrS3FilesAndUpdates = Future[Either[ErrorReport, S3FilesAndUpdates]]
+  type UpdateFutures = Seq[Either[ErrorReport, Future[PushErrorOrSuccess]]]
+  type ErrorOrS3Files = Either[ErrorReport, Seq[S3File]]
+
+  sealed trait PushFailureReport extends ErrorReport
+  sealed trait PushSuccessReport extends SuccessReport {
+    def s3Key: S3Key
+  }
+
+  case class SuccessfulRedirectDetails(uploadType: UploadType, redirectTarget: String)
+  case class SuccessfulNewOrCreatedDetails(uploadType: UploadType, uploadSize: Long, uploadDuration: Option[Long])
+
+  case class SuccessfulUpload(s3Key: S3Key,
+                              details: Either[SuccessfulNewOrCreatedDetails, SuccessfulRedirectDetails],
+                              putObjectRequest: PutObjectRequest)
+                             (implicit pushOptions: PushOptions, logger: Logger) extends PushSuccessReport {
+    def reportMessage =
+      details.fold(
+        newOrCreatedDetails => s"${newOrCreatedDetails.uploadType.pushAction.renderVerb} $s3Key ($reportDetails)",
+        redirectDetails     => s"${redirectDetails.uploadType.pushAction.renderVerb} $s3Key to ${redirectDetails.redirectTarget}"
+      )
+
+    def reportDetails = {
+      val md = putObjectRequest.getMetadata
+      val detailFragments: Seq[Option[String]] =
+        (
+          md.getCacheControl ::
+          md.getContentType ::
+          md.getContentEncoding ::
+          putObjectRequest.getStorageClass ::
+          Nil map (Option(_)) // AWS SDK may return nulls
+        ) :+ uploadSizeForHumans :+ uploadSpeedForHumans
+      detailFragments.collect {
+        case Some(detailFragment) => detailFragment
+      }.mkString(" | ")
+    }
+
+    lazy val uploadSize = details.fold(
+      newOrCreatedDetails => Some(newOrCreatedDetails.uploadSize),
+      redirectDetails     => None
+    )
+
+    lazy val uploadSizeForHumans: Option[String] = uploadSize filter (_ => logger.verboseOutput) map humanReadableByteCount
+
+    lazy val uploadSpeedForHumans: Option[String] =
+      (for {
+        dataSize <- uploadSize
+        duration <- details.left.map(_.uploadDuration).left.toOption.flatten
+      } yield {
+        humanizeUploadSpeed(dataSize, duration)
+      }) flatMap identity filter (_ => logger.verboseOutput)
+  }
+  
+  object SuccessfulUpload {
+    def humanizeUploadSpeed(uploadedBytes: Long, uploadDurations: UploadDuration*): Option[String] = {
+      val totalDurationMillis = uploadDurations.foldLeft(0L){ (memo, duration) =>
+        memo + duration
+      }
+      if (totalDurationMillis > 0) {
+        val bytesPerMillisecond = uploadedBytes / totalDurationMillis
+        val bytesPerSecond = bytesPerMillisecond * 1000 * uploadDurations.length
+        Some(humanReadableByteCount(bytesPerSecond) + "/s")  
+      } else {
+        None
+      }
+    }
+  }
+
+  case class SuccessfulDelete(s3Key: S3Key)(implicit pushOptions: PushOptions) extends PushSuccessReport {
+    def reportMessage = s"${Deleted.renderVerb} $s3Key"
+  }
+
+  case class FailedUpload(s3Key: S3Key, error: Throwable)(implicit logger: Logger) extends PushFailureReport {
+    def reportMessage = errorMessage(s"Failed to upload $s3Key", error)
+  }
+
+  case class FailedDelete(s3Key: S3Key, error: Throwable)(implicit logger: Logger) extends PushFailureReport {
+    def reportMessage = errorMessage(s"Failed to delete $s3Key", error)
+  }
+
+  type S3ClientProvider = (Config) => AmazonS3
+
+  case class S3Setting(
+    s3Client: S3ClientProvider = S3.awsS3Client,
+    retryTimeUnit: TimeUnit = SECONDS
+  ) extends RetrySetting
+}

data/src/main/scala/s3/website/UploadHelper.scala

@@ -0,0 +1,76 @@
+package s3.website
+
+import s3.website.S3Key.isIgnoredBecauseOfPrefix
+import s3.website.model.Files.listSiteFiles
+import s3.website.model._
+import s3.website.Ruby.rubyRegexMatches
+import scala.concurrent.{ExecutionContextExecutor, Future}
+import scala.util.{Failure, Success, Try}
+import java.io.File
+
+object UploadHelper {
+
+  type FutureUploads = Future[Either[ErrorReport, Seq[Upload]]]
+
+  def resolveUploads(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
+                 (implicit site: Site, pushOptions: PushOptions, logger: Logger, executor: ExecutionContextExecutor): FutureUploads =
+    resolveUploadsAgainstGetBucketResponse(s3FilesFuture)
+
+  private def resolveUploadsAgainstGetBucketResponse(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
+                                                    (implicit site: Site,
+                                                     pushOptions: PushOptions,
+                                                     logger: Logger,
+                                                     executor: ExecutionContextExecutor): FutureUploads =
+    s3FilesFuture.map { errorOrS3Files =>
+      errorOrS3Files.right.flatMap { s3Files =>
+        Try {
+          val s3KeyIndex = s3Files.map(_.s3Key).toSet
+          val s3Md5Index = s3Files.map(s3File => (s3File.s3Key, s3File.md5)).toSet
+          val siteFiles = listSiteFiles
+          val existsOnS3 = (f: File) => s3KeyIndex contains site.resolveS3Key(f)
+          val isChangedOnS3 = (upload: Upload) => !(s3Md5Index contains (upload.s3Key, upload.md5.get))
+          val newUploads = siteFiles collect {
+            case file if !existsOnS3(file) => Upload(file, NewFile)
+          }
+          val changedUploads = siteFiles collect {
+            case file if existsOnS3(file) => Upload(file, FileUpdate)
+          } filter (if (pushOptions.force) selectAllFiles else isChangedOnS3)
+          newUploads ++ changedUploads
+        } match {
+          case Success(ok) => Right(ok)
+          case Failure(err) => Left(ErrorReport(err))
+        }
+      }
+    }
+
+  val selectAllFiles = (upload: Upload) => true
+
+  def resolveDeletes(s3Files: Future[Either[ErrorReport, Seq[S3File]]], redirects: Seq[Redirect])
+                    (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Future[Either[ErrorReport, Seq[S3Key]]] =
+    if (site.config.ignore_on_server exists (
+      ignoreRegexes => ignoreRegexes.s3KeyRegexes exists( regex => regex matches S3Key.build(DELETE_NOTHING_MAGIC_WORD, site.config.s3_key_prefix))
+    )) {
+      logger.debug(s"Ignoring all files on the bucket, since the setting $DELETE_NOTHING_MAGIC_WORD is on.")
+      Future(Right(Nil))
+    } else {
+      val localS3Keys = listSiteFiles.map(site resolveS3Key)
+
+      s3Files map { s3Files: Either[ErrorReport, Seq[S3File]] =>
+        for {
+          remoteS3Keys <- s3Files.right.map(_ map (_.s3Key)).right
+        } yield {
+          val keysIgnoredBecauseOf_s3_key_prefix = remoteS3Keys.filterNot(isIgnoredBecauseOfPrefix)
+          val keysToRetain = (
+            localS3Keys ++ (redirects map { _.s3Key }) ++ keysIgnoredBecauseOf_s3_key_prefix
+          ).toSet
+          remoteS3Keys filterNot { s3Key =>
+            val ignoreOnServer = site.config.ignore_on_server.exists(_ matches s3Key)
+            if (ignoreOnServer) logger.debug(s"Ignoring $s3Key on server")
+            (keysToRetain contains s3Key) || ignoreOnServer
+          }
+        }
+      }
+    }
+
+  val DELETE_NOTHING_MAGIC_WORD = "_DELETE_NOTHING_ON_THE_S3_BUCKET_"
+}

data/src/main/scala/s3/website/model/Config.scala

@@ -0,0 +1,249 @@
+package s3.website.model
+
+import java.io.File
+import java.util
+
+import scala.util.matching.Regex
+import scala.util.{Failure, Try}
+import scala.collection.JavaConversions._
+import s3.website.Ruby.rubyRuntime
+import s3.website._
+import com.amazonaws.auth.{AWSCredentialsProvider, BasicAWSCredentials, BasicSessionCredentials, AWSStaticCredentialsProvider, DefaultAWSCredentialsProviderChain, STSAssumeRoleSessionCredentialsProvider}
+import com.amazonaws.auth.profile.ProfileCredentialsProvider
+
+case class Config(
+  s3_id:                                  Option[String], // If undefined, use IAM Roles (http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-roles.html)
+  s3_secret:                              Option[String], // If undefined, use IAM Roles (http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-roles.html)
+  session_token:                          Option[String], // If defined, the AWS Security Token Service session token (http://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html)
+  profile:                                Option[String], // If defined, the AWS profile to use for credentials
+  profile_assume_role_arn:                Option[String], // If defined, the ARN of the role to assume
+  s3_bucket:                              String,
+  s3_endpoint:                            S3Endpoint,
+  site:                                   Option[String],
+  max_age:                                Option[Either[Int, S3KeyGlob[Int]]],
+  cache_control:                          Option[Either[String, S3KeyGlob[String]]],
+  gzip:                                   Option[Either[Boolean, Seq[String]]],
+  gzip_zopfli:                            Option[Boolean],
+  s3_key_prefix:                          Option[String],
+  ignore_on_server:                       Option[S3KeyRegexes],
+  exclude_from_upload:                    Option[S3KeyRegexes],
+  s3_reduced_redundancy:                  Option[Boolean],
+  cloudfront_distribution_id:             Option[String],
+  cloudfront_invalidate_root:             Option[Boolean],
+  content_type:                           Option[S3KeyGlob[String]],
+  redirects:                              Option[Map[S3Key, String]],
+  concurrency_level:                      Int,
+  cloudfront_wildcard_invalidation:       Option[Boolean],
+  treat_zero_length_objects_as_redirects: Option[Boolean]
+)
+
+object Config {
+
+  def awsCredentials(config: Config): AWSCredentialsProvider = {
+    val credentialsFromConfigFile: Option[AWSCredentialsProvider] =
+      (
+        for {
+          s3_id <- config.s3_id
+          s3_secret <- config.s3_secret
+          session_token <- config.session_token
+        } yield new AWSStaticCredentialsProvider(new BasicSessionCredentials(s3_id, s3_secret, session_token))
+      ) orElse (
+        for {
+          s3_id <- config.s3_id
+          s3_secret <- config.s3_secret
+        } yield new AWSStaticCredentialsProvider(new BasicAWSCredentials(s3_id, s3_secret))
+      ) orElse (
+        for {
+          profile <- config.profile
+          profile_assume_role_arn <- config.profile_assume_role_arn
+        } yield new STSAssumeRoleSessionCredentialsProvider.Builder(profile_assume_role_arn, "s3_website_assume_role_session")
+          .withLongLivedCredentialsProvider(new ProfileCredentialsProvider(profile)).build()
+      ) orElse (
+        for {
+          profile <- config.profile
+        } yield new ProfileCredentialsProvider(profile)
+      )
+    credentialsFromConfigFile getOrElse new DefaultAWSCredentialsProviderChain
+  }
+
+  def loadOptionalBooleanOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Boolean, Seq[String]]]] = {
+    val yamlValue = for {
+      optionalValue <- loadOptionalValue(key)
+    } yield {
+      Right(optionalValue.map {
+        case value if value.isInstanceOf[Boolean] => Left(value.asInstanceOf[Boolean])
+        case value if value.isInstanceOf[java.util.List[_]] => Right(value.asInstanceOf[java.util.List[String]].toIndexedSeq)
+      })
+    }
+
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have a boolean or [string] value"))
+  }
+
+  def loadOptionalS3KeyRegexes(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[S3KeyRegexes]] = {
+    val yamlValue = for {
+      valueOption <- loadOptionalValue(key)
+    } yield {
+      def toS3KeyRegexes(xs: Seq[String]) = S3KeyRegexes(xs map (str => str.r) map S3KeyRegex)
+      Right(valueOption.map {
+        case value if value.isInstanceOf[String] =>
+          toS3KeyRegexes(value.asInstanceOf[String] :: Nil)
+        case value if value.isInstanceOf[java.util.List[_]] =>
+          toS3KeyRegexes(value.asInstanceOf[java.util.List[String]].toIndexedSeq)
+      })
+    }
+
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have a string or [string] value"))
+  }
+
+  def loadMaxAge(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Int, S3KeyGlob[Int]]]] = {
+    val key = "max_age"
+    val yamlValue = for {
+      maxAgeOption <- loadOptionalValue(key)
+    } yield {
+        // TODO below we are using an unsafe call to asInstance of – we should implement error handling
+        Right(maxAgeOption.map {
+          case maxAge if maxAge.isInstanceOf[Int] =>
+            Left(maxAge.asInstanceOf[Int])
+          case maxAge if maxAge.isInstanceOf[java.util.Map[_,_]] =>
+            val globs: Map[String, Int] = maxAge.asInstanceOf[util.Map[String, Int]].toMap
+            Right(S3KeyGlob(globs))
+        })
+      }
+
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have an int or (string -> int) value"))
+  }
+
+  def loadCacheControl(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[String, S3KeyGlob[String]]]] = {
+    val key = "cache_control"
+    val yamlValue = for {
+      cacheControlOption <- loadOptionalValue(key)
+    } yield {
+        // TODO below we are using an unsafe call to asInstance of – we should implement error handling
+        Right(cacheControlOption.map {
+          case cacheControl if cacheControl.isInstanceOf[String] =>
+            Left(cacheControl.asInstanceOf[String])
+          case cacheControl if cacheControl.isInstanceOf[java.util.Map[_,_]] =>
+            val globs: Map[String, String] = cacheControl.asInstanceOf[util.Map[String, String]].toMap
+            Right(S3KeyGlob(globs))
+        })
+      }
+
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have a string or (string -> string) value"))
+  }
+
+  def loadContentType(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[S3KeyGlob[String]]] = {
+    val key = "content_type"
+    val yamlValue = for {
+      contentTypeOption <- loadOptionalValue(key)
+    } yield {
+      // TODO below we are using an unsafe call to asInstance of – we should implement error handling
+      Right(contentTypeOption.map { xs =>
+          val globs: Map[String, String] = xs.asInstanceOf[util.Map[String, String]].toMap
+          S3KeyGlob(globs)
+        }
+      )
+    }
+
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have a string or (string -> string) value"))
+  }
+
+
+  def loadEndpoint(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[S3Endpoint]] =
+    loadOptionalString("s3_endpoint").right map { endpointString =>
+      endpointString.map(S3Endpoint.fromString)
+    }
+
+  def loadRedirects(s3_key_prefix: Option[String])(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Map[S3Key, String]]] = {
+    val key = "redirects"
+    val yamlValue = for {
+      redirectsOption <- loadOptionalValue(key)
+      redirectsOption <- Try(redirectsOption.map(_.asInstanceOf[java.util.Map[String,String]].toMap))
+    } yield Right(redirectsOption.map(
+        redirects => redirects.map(
+          ((key: String, value: String) => (S3Key.build(key, s3_key_prefix), value)).tupled
+        )
+      ))
+
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have a (string -> string) value"))
+  }
+
+  def loadRequiredString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, String] = {
+    val yamlValue = for {
+      valueOption <- loadOptionalValue(key)
+      stringValue <- Try(valueOption.asInstanceOf[Option[String]].get)
+    } yield {
+      Right(stringValue)
+    }
+
+    yamlValue getOrElse {
+      Left(ErrorReport(s"The key $key has to have a string value"))
+    }
+  }
+
+  def loadOptionalString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[String]] = {
+    val yamlValueOption = for {
+      valueOption <- loadOptionalValue(key)
+      optionalString <- Try(valueOption.asInstanceOf[Option[String]])
+    } yield {
+      Right(optionalString)
+    }
+
+    yamlValueOption getOrElse {
+      Left(ErrorReport(s"The key $key has to have a string value"))
+    }
+  }
+
+  def loadOptionalBoolean(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Boolean]] = {
+    val yamlValueOption = for {
+      valueOption <- loadOptionalValue(key)
+      optionalBoolean <- Try(valueOption.asInstanceOf[Option[Boolean]])
+    } yield {
+      Right(optionalBoolean)
+    }
+
+    yamlValueOption getOrElse {
+      Left(ErrorReport(s"The key $key has to have a boolean value"))
+    }
+  }
+
+  def loadOptionalInt(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Int]] = {
+    val yamlValueOption = for {
+      valueOption <- loadOptionalValue(key)
+      optionalInt <- Try(valueOption.asInstanceOf[Option[Int]])
+    } yield {
+      Right(optionalInt)
+    }
+
+    yamlValueOption getOrElse {
+      Left(ErrorReport(s"The key $key has to have an integer value"))
+    }
+  }
+
+  def loadOptionalValue(key: String)(implicit unsafeYaml: UnsafeYaml): Try[Option[_]] =
+    Try {
+      unsafeYaml.yamlObject.asInstanceOf[java.util.Map[String, _]].toMap get key
+    }
+
+  def erbEval(erbString: String, yamlConfig: S3_website_yml): Try[String] = Try {
+    val erbStringWithoutComments = erbString.replaceAll("^\\s*#.*", "")
+    rubyRuntime.evalScriptlet(
+      s"""|# encoding: utf-8
+        |require 'erb'
+        |
+        |str = <<-ERBSTR
+        |$erbStringWithoutComments
+        |ERBSTR
+        |ERB.new(str).result
+      """.stripMargin
+    ).asJavaString()
+  } match {
+    case Failure(err) => Failure(new RuntimeException(s"Failed to parse ERB in $yamlConfig:\n${err.getMessage}"))
+    case x => x
+  }
+
+  case class UnsafeYaml(yamlObject: AnyRef)
+
+  case class S3_website_yml(file: File) {
+    override def toString = file.getPath
+  }
+}

data/src/main/scala/s3/website/model/S3Endpoint.scala

@@ -0,0 +1,35 @@
+package s3.website.model
+
+case class S3Endpoint(
+  s3WebsiteHostname: String
+)
+
+object S3Endpoint {
+  def defaultEndpoint = S3Endpoint.fromString("us-east-1")
+
+  val oldRegions = Seq(
+    "us-east-1",
+    "us-west-1",
+    "us-west-2",
+    "ap-southeast-1",
+    "ap-southeast-2",
+    "ap-northeast-1",
+    "eu-west-1",
+    "sa-east-1"
+  )
+
+  def fromString(region: String): S3Endpoint = {
+    if (region == "EU") {
+      return S3Endpoint.fromString("eu-west-1")
+    }
+
+    val isOldRegion = oldRegions.contains(region)
+    val s3WebsiteHostname =
+      if (isOldRegion)
+        s"s3-website-$region.amazonaws.com"
+      else
+        s"s3-website.$region.amazonaws.com"
+
+    S3Endpoint(s3WebsiteHostname = s3WebsiteHostname)
+  }
+}