s3_website 2.14.1 → 2.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3bbec93340ef359945d7d3cfbe80883d68375991
-  data.tar.gz: 5bdcd5868373b3864e4dd5e5632cf3776a4c2486
+  metadata.gz: 0ec6e16da212e3d7ca9874580d8dd9be94860ad9
+  data.tar.gz: 52622ca62f5343414315af55e456dbb4b03a2230
 SHA512:
-  metadata.gz: 8c1ce8a06185665f9c5069ef5a116c596928bed0b0045e8ab8fb4ca0a2e83c40897229b8581705f15907840cca5d2bbffc97d6d9735d1f1444645fcbd7b44cf9
-  data.tar.gz: 626d786f95ed0678bc13fddc544c37a8775f7e3856d7dbb9e158da8ac06fee042288323f4b5296e24ee517ea04ca8f93f98486abd698fef407902762df74c674
+  metadata.gz: 4aaf8cf3d256762c897c777f175c6f5e549ece335a0ed45005980c1676d6876a85f91d6dbf1d6b9e038bbd28e3bb846d8bb6e8d31afebcced7320805176df2d0
+  data.tar.gz: 510fe2386587a3b07e08e9fb017c47a9a7abb01ddb2eccc51f0ba9c099b875ac451cc5e03291247e72d12998a35e85ebfcdba8acae67a2335f3991f1ee36522a

data/changelog.md

@@ -2,6 +2,12 @@
 
 This project uses [Semantic Versioning](http://semver.org).
 
+## 2.14.2
+
+* Apply correct mime type on already-gzipped files
+
+  See <https://github.com/laurilehmijoki/s3_website/issues/229#issuecomment-237229421> for discussion  
+
 ## 2.14.1
 
 * Do not gzip a file that is already gzipped

data/lib/s3_website/version.rb

@@ -1,3 +1,3 @@
 module S3Website
-  VERSION = '2.14.1'
+  VERSION = '2.14.2'
 end

data/src/main/scala/s3/website/model/push.scala

@@ -9,12 +9,12 @@ import java.util.zip.{GZIPInputStream, GZIPOutputStream}
 import org.apache.tika.Tika
 import s3.website.Ruby._
 import s3.website._
-import s3.website.model.Upload.tika
-import s3.website.model.Encoding.encodingOnS3
+import s3.website.model.Upload.{amountOfMagicGzipBytes, tika}
+import s3.website.model.Encoding.{Gzip, Zopfli, encodingOnS3}
 import java.io.File.createTempFile
 
-import org.apache.commons.io.FileUtils
-import org.apache.commons.io.IOUtils.copy
+import org.apache.commons.io.{FileUtils, IOUtils}
+import org.apache.commons.io.IOUtils._
 
 import scala.concurrent.{ExecutionContextExecutor, Future}
 import scala.util.Try
@@ -60,16 +60,21 @@ case class Upload(originalFile: File, uploadType: UploadType)(implicit site: Sit
   lazy val s3Key = site.resolveS3Key(originalFile)
 
   lazy val encodingOnS3 = Encoding.encodingOnS3(s3Key)
-
-  /**
-   * This is the file we should upload, because it contains the potentially gzipped contents of the original file.
-   *
-   * May throw an exception, so remember to call this in a Try or Future monad
-   */
-  lazy val uploadFile: Try[File] = Upload uploadFile originalFile
+  lazy val gzipEnabledByConfig: Boolean = encodingOnS3.fold(false)((algorithm: Either[Gzip, Zopfli]) => true)
 
   lazy val contentType: Try[String] = tika map { tika =>
-    val mimeType = tika.detect(originalFile)
+    val file = // This file contains the data that the user should see after decoding the the transport protocol (HTTP) encoding (practically: after ungzipping)
+      if (fileIsGzippedByExternalBuildTool) {
+        val unzippedFile = createTempFile(originalFile.getName, "unzipped")
+        unzippedFile.deleteOnExit()
+        using(new GZIPInputStream(fis(originalFile))) { stream =>
+          IOUtils.copy(stream, new FileOutputStream(unzippedFile))
+        }
+        unzippedFile
+      } else {
+        originalFile
+      }
+    val mimeType = tika.detect(file)
     if (mimeType.startsWith("text/") || mimeType == "application/json")
       mimeType + "; charset=utf-8"
     else
@@ -95,50 +100,52 @@ case class Upload(originalFile: File, uploadType: UploadType)(implicit site: Sit
   /**
    * May throw an exception, so remember to call this in a Try or Future monad
    */
-  lazy val md5 = Upload md5 originalFile
-}
-
-object Upload {
-  lazy val tika = Try(new Tika())
-
-  def md5(originalFile: File)(implicit site: Site, logger: Logger): Try[MD5] =
-    uploadFile(originalFile) map { file =>
-      using(fis { file }) { DigestUtils.md5Hex }
-    }
+  lazy val md5 = uploadFile map { file =>
+    using(fis { file }) { DigestUtils.md5Hex }
+  }
 
-  def uploadFile(originalFile: File)(implicit site: Site, logger: Logger): Try[File] =
-    encodingOnS3(site resolveS3Key originalFile)
-      .fold(Try(originalFile))(algorithm =>
-        Try {
-          val isAlreadyGzipped =
-            if (originalFile.length() < 2) {
-              false
-            } else {
-              val fis = new FileInputStream(originalFile)
-              val amountOfMagicGzipBytes = 2
-              val firstTwoBytes = Array.fill[Byte](amountOfMagicGzipBytes)(0)
-              fis.read(firstTwoBytes, 0, amountOfMagicGzipBytes)
-              val head = firstTwoBytes(0) & 0xff | (firstTwoBytes(1) << 8) & 0xff00
-              head == GZIPInputStream.GZIP_MAGIC
-            }
-          if (isAlreadyGzipped) {
-            logger.debug(s"File ${originalFile.getAbsolutePath} is already gzipped. Skipping gzip.")
-            originalFile
-          } else {
-            val tempFile = createTempFile(originalFile.getName, "gzip")
-            tempFile.deleteOnExit()
-            using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
-              copy(fis(originalFile), stream)
-            }
-            tempFile
+  // This is the file we should try to upload
+  lazy val uploadFile: Try[File] =
+    if (gzipEnabledByConfig)
+      Try {
+        if (fileIsGzippedByExternalBuildTool) {
+          logger.debug(s"File ${originalFile.getAbsolutePath} is already gzipped. Skipping gzip.")
+          originalFile
+        } else {
+          logger.debug(s"Gzipping file ${originalFile.getName}")
+          val tempFile = createTempFile(originalFile.getName, "gzip")
+          tempFile.deleteOnExit()
+          using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
+            IOUtils.copy(fis(originalFile), stream)
           }
+          tempFile
         }
-      )
+      }
+    else
+      Try(originalFile)
+
+  private lazy val fileIsGzippedByExternalBuildTool = gzipEnabledByConfig && originalFileIsGzipped
+
+  private lazy val originalFileIsGzipped =
+    if (originalFile.length() < amountOfMagicGzipBytes) {
+      false
+    } else {
+      val fis = new FileInputStream(originalFile)
+      val firstTwoBytes = Array.fill[Byte](amountOfMagicGzipBytes)(0)
+      fis.read(firstTwoBytes, 0, amountOfMagicGzipBytes)
+      val head = firstTwoBytes(0) & 0xff | (firstTwoBytes(1) << 8) & 0xff00
+      head == GZIPInputStream.GZIP_MAGIC
+    }
 
   private[this] def fis(file: File): InputStream = new FileInputStream(file)
   private[this] def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
 }
 
+object Upload {
+  lazy val tika = Try(new Tika())
+  private val amountOfMagicGzipBytes = 2
+}
+
 object Files {
   def recursiveListFiles(f: File): Seq[File] = {
     val these = f.listFiles

data/src/test/scala/s3/website/S3WebsiteSpec.scala

@@ -44,11 +44,10 @@ class S3WebsiteSpec extends Specification {
       sentPutObjectRequest.getKey must equalTo("styles.css")
     }
 
-    "gzips a file" in new BasicSetup {
+    "gzip a file" in new BasicSetup {
       val htmlString = "<h1>hi again</h1>"
-      val gzippedBytes = gzip(htmlString.getBytes(StandardCharsets.UTF_8))
       config = "gzip: true"
-      setLocalFileWithContent("index.html", gzippedBytes)
+      setLocalFileWithContent(("index.html", htmlString))
       setS3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */)
       val putObjectRequestCaptor = ArgumentCaptor.forClass(classOf[PutObjectRequest])
       push()
@@ -62,6 +61,15 @@ class S3WebsiteSpec extends Specification {
       unzippedString must equalTo(htmlString)
     }
 
+    "apply the correct mime type on an already gzipped file" in new BasicSetup {
+      val htmlString = "<html><body><h1>Hello world</h1></body></html>"
+      val gzippedHtml = gzip(htmlString.getBytes(StandardCharsets.UTF_8))
+      config = "gzip: true"
+      setLocalFileWithContent("index.html", gzippedHtml)
+      push()
+      sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
+    }
+
     "not gzip the file if it's already gzipped" in new BasicSetup {
       config = "gzip: true"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_website
 version: !ruby/object:Gem::Version
-  version: 2.14.1
+  version: 2.14.2
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-02 00:00:00.000000000 Z
+date: 2016-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor