s3_website 1.8.2 → 2.0.0

data/src/main/scala/s3/website/model/Config.scala

@@ -0,0 +1,175 @@
+package s3.website.model
+
+import scala.util.{Failure, Try}
+import scala.collection.JavaConversions._
+import s3.website.Ruby.rubyRuntime
+import s3.website.ErrorReport
+import com.amazonaws.auth.{AWSCredentialsProvider, BasicAWSCredentials, DefaultAWSCredentialsProviderChain}
+
+case class Config(
+  s3_id:                      Option[String], // If undefined, use IAM Roles (http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-roles.html)
+  s3_secret:                  Option[String], // If undefined, use IAM Roles (http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-roles.html)
+  s3_bucket:                  String,
+  s3_endpoint:                S3Endpoint,
+  max_age:                    Option[Either[Int, Map[String, Int]]],
+  gzip:                       Option[Either[Boolean, Seq[String]]],
+  gzip_zopfli:                Option[Boolean],
+  ignore_on_server:           Option[Either[String, Seq[String]]],
+  exclude_from_upload:        Option[Either[String, Seq[String]]],
+  s3_reduced_redundancy:      Option[Boolean],
+  cloudfront_distribution_id: Option[String],
+  cloudfront_invalidate_root: Option[Boolean],
+  redirects:                  Option[Map[String, String]],
+  concurrency_level:          Int
+)
+
+object Config {
+
+  def awsCredentials(config: Config): AWSCredentialsProvider = {
+    val credentialsFromConfigFile = for {
+      s3_id <- config.s3_id
+      s3_secret <- config.s3_secret
+    } yield new BasicAWSCredentials(s3_id, s3_secret)
+    credentialsFromConfigFile.fold(new DefaultAWSCredentialsProviderChain: AWSCredentialsProvider)(credentials =>
+      new AWSCredentialsProvider {
+        def getCredentials = credentials
+        def refresh() = {}
+      }
+    )
+  }
+
+  def loadOptionalBooleanOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Boolean, Seq[String]]]] = {
+    val yamlValue = for {
+      optionalValue <- loadOptionalValue(key)
+    } yield {
+      Right(optionalValue.map {
+        case value if value.isInstanceOf[Boolean] => Left(value.asInstanceOf[Boolean])
+        case value if value.isInstanceOf[java.util.List[_]] => Right(value.asInstanceOf[java.util.List[String]].toIndexedSeq)
+      })
+    }
+
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have a boolean or [string] value"))
+  }
+
+  def loadOptionalStringOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[String, Seq[String]]]] = {
+    val yamlValue = for {
+      valueOption <- loadOptionalValue(key)
+    } yield {
+      Right(valueOption.map {
+        case value if value.isInstanceOf[String] => Left(value.asInstanceOf[String])
+        case value if value.isInstanceOf[java.util.List[_]] => Right(value.asInstanceOf[java.util.List[String]].toIndexedSeq)
+      })
+    }
+
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have a string or [string] value"))
+  }
+
+  def loadMaxAge(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Int, Map[String, Int]]]] = {
+    val key = "max_age"
+    val yamlValue = for {
+      maxAgeOption <- loadOptionalValue(key)
+    } yield {
+      Right(maxAgeOption.map {
+        case maxAge if maxAge.isInstanceOf[Int] => Left(maxAge.asInstanceOf[Int])
+        case maxAge if maxAge.isInstanceOf[java.util.Map[_,_]] => Right(maxAge.asInstanceOf[java.util.Map[String,Int]].toMap)
+      })
+    }
+
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have an int or (string -> int) value"))
+  }
+
+  def loadEndpoint(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[S3Endpoint]] =
+    loadOptionalString("s3_endpoint").right flatMap { endpointString =>
+      endpointString.map(S3Endpoint.forString) match {
+        case Some(Right(endpoint)) => Right(Some(endpoint))
+        case Some(Left(endpointError)) => Left(endpointError)
+        case None => Right(None)
+      }
+    }
+
+  def loadRedirects(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Map[String, String]]] = {
+    val key = "redirects"
+    val yamlValue = for {
+      redirectsOption <- loadOptionalValue(key)
+      redirects <- Try(redirectsOption.map(_.asInstanceOf[java.util.Map[String,String]].toMap))
+    } yield Right(redirects)
+
+    yamlValue getOrElse Left(ErrorReport(s"The key $key has to have a (string -> string) value"))
+  }
+
+  def loadRequiredString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, String] = {
+    val yamlValue = for {
+      valueOption <- loadOptionalValue(key)
+      stringValue <- Try(valueOption.asInstanceOf[Option[String]].get)
+    } yield {
+      Right(stringValue)
+    }
+
+    yamlValue getOrElse {
+      Left(ErrorReport(s"The key $key has to have a string value"))
+    }
+  }
+
+  def loadOptionalString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[String]] = {
+    val yamlValueOption = for {
+      valueOption <- loadOptionalValue(key)
+      optionalString <- Try(valueOption.asInstanceOf[Option[String]])
+    } yield {
+      Right(optionalString)
+    }
+
+    yamlValueOption getOrElse {
+      Left(ErrorReport(s"The key $key has to have a string value"))
+    }
+  }
+
+  def loadOptionalBoolean(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Boolean]] = {
+    val yamlValueOption = for {
+      valueOption <- loadOptionalValue(key)
+      optionalBoolean <- Try(valueOption.asInstanceOf[Option[Boolean]])
+    } yield {
+      Right(optionalBoolean)
+    }
+
+    yamlValueOption getOrElse {
+      Left(ErrorReport(s"The key $key has to have a boolean value"))
+    }
+  }
+
+  def loadOptionalInt(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Int]] = {
+    val yamlValueOption = for {
+      valueOption <- loadOptionalValue(key)
+      optionalInt <- Try(valueOption.asInstanceOf[Option[Int]])
+    } yield {
+      Right(optionalInt)
+    }
+
+    yamlValueOption getOrElse {
+      Left(ErrorReport(s"The key $key has to have an integer value"))
+    }
+  }
+
+  def loadOptionalValue(key: String)(implicit unsafeYaml: UnsafeYaml): Try[Option[_]] =
+    Try {
+      unsafeYaml.yamlObject.asInstanceOf[java.util.Map[String, _]].toMap get key
+    }
+
+  def erbEval(erbString: String, yamlConfigPath: String): Try[String] = Try {
+    val erbStringWithoutComments = erbString.replaceAll("^\\s*#.*", "")
+    rubyRuntime.evalScriptlet(
+      s"""
+        |require 'erb'
+        |
+        |str = <<-ERBSTR
+        |$erbStringWithoutComments
+        |ERBSTR
+        |ERB.new(str).result
+      """.stripMargin
+    ).asJavaString()
+  } match {
+    case Failure(err) => Failure(new RuntimeException(s"Failed to parse ERB in $yamlConfigPath:\n${err.getMessage}"))
+    case x => x
+  }
+
+  case class UnsafeYaml(yamlObject: AnyRef)
+}

data/src/main/scala/s3/website/model/S3Endpoint.scala

@@ -0,0 +1,24 @@
+package s3.website.model
+
+import s3.website.ErrorReport
+
+case class S3Endpoint(
+  s3WebsiteHostname: String,
+  s3Hostname: String
+)
+
+object S3Endpoint {
+  val defaultEndpoint = S3Endpoint("s3-website-us-east-1.amazonaws.com", "s3.amazonaws.com")
+
+  def forString(locationConstraint: String): Either[ErrorReport, S3Endpoint] = locationConstraint match {
+    case "EU" | "eu-west-1" => Right(S3Endpoint("s3-website-eu-west-1.amazonaws.com",      "s3-eu-west-1.amazonaws.com"))
+    case "us-east-1" =>        Right(defaultEndpoint)
+    case "us-west-1" =>        Right(S3Endpoint("s3-website-us-west-1.amazonaws.com",      "s3-us-west-1.amazonaws.com"))
+    case "us-west-2" =>        Right(S3Endpoint("s3-website-us-west-2.amazonaws.com",      "s3-us-west-2.amazonaws.com"))
+    case "ap-southeast-1" =>   Right(S3Endpoint("s3-website-ap-southeast-1.amazonaws.com", "s3-ap-southeast-1.amazonaws.com"))
+    case "ap-southeast-2" =>   Right(S3Endpoint("s3-website-ap-southeast-2.amazonaws.com", "s3-ap-southeast-2.amazonaws.com"))
+    case "ap-northeast-1" =>   Right(S3Endpoint("s3-website-ap-northeast-1.amazonaws.com", "s3-ap-northeast-1.amazonaws.com"))
+    case "sa-east-1" =>        Right(S3Endpoint("s3-website-sa-east-1.amazonaws.com",      "s3-sa-east-1.amazonaws.com"))
+    case _ =>                  Left(ErrorReport(s"Unrecognised endpoint: $locationConstraint"))
+  }
+}

data/src/main/scala/s3/website/model/Site.scala

@@ -0,0 +1,80 @@
+package s3.website.model
+
+import java.io.File
+import scala.util.Try
+import org.yaml.snakeyaml.Yaml
+import s3.website.model.Config._
+import scala.io.Source.fromFile
+import scala.language.postfixOps
+import s3.website.{S3Key, Logger, ErrorReport}
+import scala.util.Failure
+import s3.website.model.Config.UnsafeYaml
+import scala.util.Success
+
+case class Site(rootDirectory: String, config: Config) {
+  def resolveS3Key(file: File) = file.getAbsolutePath.replace(rootDirectory, "").replaceFirst("^/", "")
+
+  def resolveFile(s3File: S3File): File = resolveFile(s3File.s3Key)
+
+  def resolveFile(s3Key: S3Key): File = new File(s"$rootDirectory/$s3Key")
+}
+
+object Site {
+  def loadSite(yamlConfigPath: String, siteRootDirectory: String)
+              (implicit logger: Logger): Either[ErrorReport, Site] = {
+    val yamlObjectTry = for {
+      yamlString <- Try(fromFile(new File(yamlConfigPath)).mkString)
+      yamlWithErbEvaluated <- erbEval(yamlString, yamlConfigPath)
+      yamlObject <- Try(new Yaml() load yamlWithErbEvaluated)
+    } yield yamlObject
+    yamlObjectTry match {
+      case Success(yamlObject) =>
+        implicit val unsafeYaml = UnsafeYaml(yamlObject)
+        val config: Either[ErrorReport, Config] = for {
+          s3_id <- loadOptionalString("s3_id").right
+          s3_secret <- loadOptionalString("s3_secret").right
+          s3_bucket <- loadRequiredString("s3_bucket").right
+          s3_endpoint <- loadEndpoint.right
+          max_age <- loadMaxAge.right
+          gzip <- loadOptionalBooleanOrStringSeq("gzip").right
+          gzip_zopfli <- loadOptionalBoolean("gzip_zopfli").right
+          extensionless_mime_type <- loadOptionalString("extensionless_mime_type").right
+          ignore_on_server <- loadOptionalStringOrStringSeq("ignore_on_server").right
+          exclude_from_upload <- loadOptionalStringOrStringSeq("exclude_from_upload").right
+          s3_reduced_redundancy <- loadOptionalBoolean("s3_reduced_redundancy").right
+          cloudfront_distribution_id <- loadOptionalString("cloudfront_distribution_id").right
+          cloudfront_invalidate_root <- loadOptionalBoolean("cloudfront_invalidate_root").right
+          concurrency_level <- loadOptionalInt("concurrency_level").right
+          redirects <- loadRedirects.right
+        } yield {
+          gzip_zopfli.foreach(_ => logger.info(
+            """|Zopfli is not currently supported. Falling back to regular gzip.
+               |If you find a stable Java implementation for zopfli, please send an email to lauri.lehmijoki@iki.fi about it."""
+            .stripMargin))
+          extensionless_mime_type.foreach(_ => logger.info(
+            s"Ignoring the extensionless_mime_type setting in $yamlConfigPath. Counting on Apache Tika to infer correct mime types.")
+          )
+          Config(
+            s3_id,
+            s3_secret,
+            s3_bucket,
+            s3_endpoint getOrElse S3Endpoint.defaultEndpoint,
+            max_age,
+            gzip,
+            gzip_zopfli,
+            ignore_on_server = ignore_on_server,
+            exclude_from_upload = exclude_from_upload,
+            s3_reduced_redundancy,
+            cloudfront_distribution_id,
+            cloudfront_invalidate_root,
+            redirects,
+            concurrency_level.fold(20)(_ max 20) // At minimum, run 20 concurrent operations
+          )
+        }
+
+        config.right.map(Site(siteRootDirectory, _))
+      case Failure(error) =>
+        Left(ErrorReport(error))
+    }
+  }
+}

data/src/main/scala/s3/website/model/push.scala

@@ -0,0 +1,167 @@
+package s3.website.model
+
+import com.amazonaws.services.s3.model.S3ObjectSummary
+import java.io._
+import org.apache.commons.codec.digest.DigestUtils
+import java.util.zip.GZIPOutputStream
+import org.apache.tika.Tika
+import s3.website.Ruby._
+import s3.website._
+import s3.website.model.LocalFile.tika
+import s3.website.model.Encoding.encodingOnS3
+import java.io.File.createTempFile
+import org.apache.commons.io.IOUtils.copy
+import scala.util.Try
+
+object Encoding {
+
+  val defaultGzipExtensions = ".html" :: ".css" :: ".js" :: ".txt" :: Nil
+
+  case class Gzip()
+  case class Zopfli()
+
+  def encodingOnS3(s3Key: String)(implicit config: Config): Option[Either[Gzip, Zopfli]] =
+    config.gzip.flatMap { (gzipSetting: Either[Boolean, Seq[String]]) =>
+      val shouldZipThisFile = gzipSetting.fold(
+        shouldGzip => defaultGzipExtensions exists s3Key.endsWith,
+        fileExtensions => fileExtensions exists s3Key.endsWith
+      )
+      if (shouldZipThisFile && config.gzip_zopfli.isDefined)
+        Some(Right(Zopfli()))
+      else if (shouldZipThisFile)
+        Some(Left(Gzip()))
+      else
+        None
+    }
+}
+
+sealed trait UploadType {
+  val pushAction: PushAction
+}
+
+case object NewFile extends UploadType {
+  val pushAction = Created
+}
+case object FileUpdate extends UploadType {
+  val pushAction = Updated
+}
+
+case object RedirectFile extends UploadType {
+  val pushAction = Redirected
+}
+
+case class LocalFile(originalFile: File, uploadType: UploadType)(implicit site: Site) {
+  lazy val s3Key = site.resolveS3Key(originalFile)
+
+  lazy val encodingOnS3 = Encoding.encodingOnS3(s3Key)
+
+  /**
+   * This is the file we should upload, because it contains the potentially gzipped contents of the original file.
+   *
+   * May throw an exception, so remember to call this in a Try or Future monad
+   */
+  lazy val uploadFile: Try[File] = LocalFile uploadFile originalFile
+
+  lazy val contentType: Try[String] = tika map { tika =>
+    val mimeType = tika.detect(originalFile)
+    if (mimeType.startsWith("text/") || mimeType == "application/json")
+      mimeType + "; charset=utf-8"
+    else
+      mimeType
+  }
+
+  lazy val maxAge: Option[Int] = {
+    type GlobsMap = Map[String, Int]
+    site.config.max_age.flatMap { (intOrGlobs: Either[Int, GlobsMap]) =>
+      type GlobsSeq = Seq[(String, Int)]
+      def respectMostSpecific(globs: GlobsMap): GlobsSeq = globs.toSeq.sortBy(_._1.length).reverse
+      intOrGlobs
+        .right.map(respectMostSpecific)
+        .fold(
+          (seconds: Int) => Some(seconds),
+          (globs: GlobsSeq) =>
+            globs.find { globAndInt =>
+              (rubyRuntime evalScriptlet s"File.fnmatch('${globAndInt._1}', '$s3Key')")
+                .toJava(classOf[Boolean])
+                .asInstanceOf[Boolean]
+            } map (_._2)
+        )
+    }
+  }
+
+  /**
+   * May throw an exception, so remember to call this in a Try or Future monad
+   */
+  lazy val md5 = LocalFile md5 originalFile
+}
+
+object LocalFile {
+  lazy val tika = Try(new Tika())
+
+  def md5(originalFile: File)(implicit site: Site): Try[MD5] =
+    uploadFile(originalFile) map { file =>
+      using(fis { file }) { DigestUtils.md5Hex }
+    }
+
+  def uploadFile(originalFile: File)(implicit site: Site): Try[File] =
+    encodingOnS3(site resolveS3Key originalFile)
+      .fold(Try(originalFile))(algorithm =>
+        Try {
+          val tempFile = createTempFile(originalFile.getName, "gzip")
+          tempFile.deleteOnExit()
+          using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
+            copy(fis(originalFile), stream)
+          }
+          tempFile
+        }
+      )
+
+  private[this] def fis(file: File): InputStream = new FileInputStream(file)
+  private[this] def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
+}
+
+object Files {
+  def recursiveListFiles(f: File): Seq[File] = {
+    val these = f.listFiles
+    these ++ these.filter(_.isDirectory).flatMap(recursiveListFiles)
+  }
+
+  def listSiteFiles(implicit site: Site, logger: Logger) = {
+    def excludeFromUpload(s3Key: String) = {
+      val excludeByConfig = site.config.exclude_from_upload exists {
+        _.fold(
+          // For backward compatibility, use Ruby regex matching
+          (exclusionRegex: String) => rubyRegexMatches(s3Key, exclusionRegex),
+          (exclusionRegexes: Seq[String]) => exclusionRegexes exists (rubyRegexMatches(s3Key, _))
+        )
+      }
+      val neverUpload = "s3_website.yml" :: ".env" :: Nil
+      val doNotUpload = excludeByConfig || (neverUpload contains s3Key)
+      if (doNotUpload) logger.debug(s"Excluded $s3Key from upload")
+      doNotUpload
+    }
+    recursiveListFiles(new File(site.rootDirectory))
+      .filterNot(_.isDirectory)
+      .filterNot(f => excludeFromUpload(site.resolveS3Key(f)))
+  }
+}
+
+case class Redirect(s3Key: String, redirectTarget: String) {
+  def uploadType = RedirectFile
+}
+
+object Redirect {
+  def resolveRedirects(implicit config: Config): Seq[Redirect] =
+    config.redirects.fold(Nil: Seq[Redirect]) { sourcesToTargets =>
+      sourcesToTargets.foldLeft(Seq(): Seq[Redirect]) {
+        (redirects, sourceToTarget) =>
+          redirects :+ Redirect(sourceToTarget._1, sourceToTarget._2)
+      }
+  }
+}
+
+case class S3File(s3Key: String, md5: MD5)
+
+object S3File {
+  def apply(summary: S3ObjectSummary): S3File = S3File(summary.getKey, summary.getETag)
+}