s3_website 2.14.0 → 2.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed60d795c8498eaa376cf3db07a1d150e664fb94
-  data.tar.gz: eb092f1524a34883201452b5ed013a864901d866
+  metadata.gz: 3bbec93340ef359945d7d3cfbe80883d68375991
+  data.tar.gz: 5bdcd5868373b3864e4dd5e5632cf3776a4c2486
 SHA512:
-  metadata.gz: 08ec5fdda103891fef1dd85b5319cd5572fd021048cceb1aae4fee438800462e072b6bcac45408fa8110f8dc59845cc98ed9eaa42a0229d0eec2ce8936dcd806
-  data.tar.gz: aab6bd5d66dafc38a200756c26a5da9e5334a35ec26f2947c1db1f6fc08dd91ed3054afc86057e126136adb63da34d673ee9dbe05da20f23ffeb06812909e3d7
+  metadata.gz: 8c1ce8a06185665f9c5069ef5a116c596928bed0b0045e8ab8fb4ca0a2e83c40897229b8581705f15907840cca5d2bbffc97d6d9735d1f1444645fcbd7b44cf9
+  data.tar.gz: 626d786f95ed0678bc13fddc544c37a8775f7e3856d7dbb9e158da8ac06fee042288323f4b5296e24ee517ea04ca8f93f98486abd698fef407902762df74c674

data/README.md

@@ -72,6 +72,11 @@ in the project's root directory to take advantage of this feature. Please have
 a look at [dotenv's usage guide](https://github.com/bkeepers/dotenv#usage) for
 syntax information.
 
+Your `.env` file should containing the following variables:
+
+    AWS_ACCESS_KEY_ID=FOO
+    AWS_SECRET_ACCESS_KEY=BAR
+
 ## Project goals
 
 * Provide a command-line interface tool for deploying and managing S3 websites
@@ -169,6 +174,9 @@ not the pre-processed extensions.
 
 After changing the `gzip` setting, push with the `--force` option.
 
+s3_website will not gzip a file that is already gzipped. This is useful in the
+situations where your build tools gzip a file before you invoke `s3_website push`.
+
 ### Using non-standard AWS regions
 
 By default, `s3_website` uses the US Standard Region. You can upload your
@@ -185,7 +193,10 @@ s3_endpoint: ap-northeast-1
 The valid `s3_endpoint` values consist of the [S3 location constraint
 values](http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region).
 
-Note that at the moment s3_website does not support the *eu-central-1* region.
+Note that at the moment s3_website does not support any region that
+supports only a V4 Signature and does not support V2
+ (i.e. *eu-central-1*, *ap-south-1*, *ap-northeast-2*).
+This support can be tracked in [issue #126](https://github.com/laurilehmijoki/s3_website/issues/126).
 
 ### Ignoring files you want to keep on AWS
 

data/changelog.md

@@ -2,6 +2,12 @@
 
 This project uses [Semantic Versioning](http://semver.org).
 
+## 2.14.1
+
+* Do not gzip a file that is already gzipped
+
+  See <https://github.com/laurilehmijoki/s3_website/issues/229> for discussion
+
 ## 2.14.0
 
 * Add support for CloudFront wildcard invalidations

data/lib/s3_website/version.rb

@@ -1,3 +1,3 @@
 module S3Website
-  VERSION = '2.14.0'
+  VERSION = '2.14.1'
 end

data/resources/s3_website.jar.md5

@@ -1 +1 @@
-6736177c885c454c20caadf2396980f0
+53be05bebd606f3d1e2443beb13130a5

data/src/main/scala/s3/website/model/push.scala

@@ -2,15 +2,20 @@ package s3.website.model
 
 import com.amazonaws.services.s3.model.S3ObjectSummary
 import java.io._
+
 import org.apache.commons.codec.digest.DigestUtils
-import java.util.zip.GZIPOutputStream
+import java.util.zip.{GZIPInputStream, GZIPOutputStream}
+
 import org.apache.tika.Tika
 import s3.website.Ruby._
 import s3.website._
 import s3.website.model.Upload.tika
 import s3.website.model.Encoding.encodingOnS3
 import java.io.File.createTempFile
+
+import org.apache.commons.io.FileUtils
 import org.apache.commons.io.IOUtils.copy
+
 import scala.concurrent.{ExecutionContextExecutor, Future}
 import scala.util.Try
 
@@ -51,7 +56,7 @@ case object RedirectFile extends UploadType {
   val pushAction = Redirected
 }
 
-case class Upload(originalFile: File, uploadType: UploadType)(implicit site: Site) {
+case class Upload(originalFile: File, uploadType: UploadType)(implicit site: Site, logger: Logger) {
   lazy val s3Key = site.resolveS3Key(originalFile)
 
   lazy val encodingOnS3 = Encoding.encodingOnS3(s3Key)
@@ -96,21 +101,37 @@ case class Upload(originalFile: File, uploadType: UploadType)(implicit site: Sit
 object Upload {
   lazy val tika = Try(new Tika())
 
-  def md5(originalFile: File)(implicit site: Site): Try[MD5] =
+  def md5(originalFile: File)(implicit site: Site, logger: Logger): Try[MD5] =
     uploadFile(originalFile) map { file =>
       using(fis { file }) { DigestUtils.md5Hex }
     }
 
-  def uploadFile(originalFile: File)(implicit site: Site): Try[File] =
+  def uploadFile(originalFile: File)(implicit site: Site, logger: Logger): Try[File] =
     encodingOnS3(site resolveS3Key originalFile)
       .fold(Try(originalFile))(algorithm =>
         Try {
-          val tempFile = createTempFile(originalFile.getName, "gzip")
-          tempFile.deleteOnExit()
-          using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
-            copy(fis(originalFile), stream)
+          val isAlreadyGzipped =
+            if (originalFile.length() < 2) {
+              false
+            } else {
+              val fis = new FileInputStream(originalFile)
+              val amountOfMagicGzipBytes = 2
+              val firstTwoBytes = Array.fill[Byte](amountOfMagicGzipBytes)(0)
+              fis.read(firstTwoBytes, 0, amountOfMagicGzipBytes)
+              val head = firstTwoBytes(0) & 0xff | (firstTwoBytes(1) << 8) & 0xff00
+              head == GZIPInputStream.GZIP_MAGIC
+            }
+          if (isAlreadyGzipped) {
+            logger.debug(s"File ${originalFile.getAbsolutePath} is already gzipped. Skipping gzip.")
+            originalFile
+          } else {
+            val tempFile = createTempFile(originalFile.getName, "gzip")
+            tempFile.deleteOnExit()
+            using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
+              copy(fis(originalFile), stream)
+            }
+            tempFile
           }
-          tempFile
         }
       )
 

data/src/test/scala/s3/website/S3WebsiteSpec.scala

@@ -1,7 +1,9 @@
 package s3.website
 
-import java.io.File
+import java.io._
+import java.nio.charset.StandardCharsets
 import java.util.concurrent.atomic.AtomicInteger
+import java.util.zip.{GZIPInputStream, GZIPOutputStream}
 
 import com.amazonaws.AmazonServiceException
 import com.amazonaws.services.cloudfront.AmazonCloudFront
@@ -9,8 +11,9 @@ import com.amazonaws.services.cloudfront.model.{CreateInvalidationRequest, Creat
 import com.amazonaws.services.s3.AmazonS3
 import com.amazonaws.services.s3.model._
 import org.apache.commons.codec.digest.DigestUtils._
-import org.apache.commons.io.FileUtils
 import org.apache.commons.io.FileUtils._
+import org.apache.commons.io.IOUtils.{write => _}
+import org.apache.commons.io.{FileUtils, IOUtils}
 import org.mockito.Mockito._
 import org.mockito.invocation.InvocationOnMock
 import org.mockito.stubbing.Answer
@@ -18,9 +21,9 @@ import org.mockito.{ArgumentCaptor, Matchers, Mockito}
 import org.specs2.mutable.{BeforeAfter, Specification}
 import org.specs2.specification.Scope
 import s3.website.CloudFront.CloudFrontSetting
-import s3.website.UploadHelper.DELETE_NOTHING_MAGIC_WORD
-import s3.website.Push.{CliArgs}
+import s3.website.Push.CliArgs
 import s3.website.S3.S3Setting
+import s3.website.UploadHelper.DELETE_NOTHING_MAGIC_WORD
 import s3.website.model.Config.S3_website_yml
 import s3.website.model.Ssg.automaticallySupportedSiteGenerators
 import s3.website.model._
@@ -41,6 +44,52 @@ class S3WebsiteSpec extends Specification {
       sentPutObjectRequest.getKey must equalTo("styles.css")
     }
 
+    "gzips a file" in new BasicSetup {
+      val htmlString = "<h1>hi again</h1>"
+      val gzippedBytes = gzip(htmlString.getBytes(StandardCharsets.UTF_8))
+      config = "gzip: true"
+      setLocalFileWithContent("index.html", gzippedBytes)
+      setS3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */)
+      val putObjectRequestCaptor = ArgumentCaptor.forClass(classOf[PutObjectRequest])
+      push()
+      sentPutObjectRequest.getKey must equalTo("index.html")
+      verify(amazonS3Client).putObject(putObjectRequestCaptor.capture())
+
+      val bytesToS3: InputStream = putObjectRequestCaptor.getValue.getInputStream
+      val unzippedBytesToS3 = new GZIPInputStream(bytesToS3)
+      val unzippedString = IOUtils.toString(unzippedBytesToS3, StandardCharsets.UTF_8)
+
+      unzippedString must equalTo(htmlString)
+    }
+
+    "not gzip the file if it's already gzipped" in new BasicSetup {
+      config = "gzip: true"
+
+      val cssString = "body { color: red }"
+      val gzippedCss = gzip(cssString.getBytes(StandardCharsets.UTF_8))
+      setLocalFileWithContent("styles.css", gzippedCss)
+      val putObjectRequestCaptor = ArgumentCaptor.forClass(classOf[PutObjectRequest])
+      push()
+      sentPutObjectRequest.getKey must equalTo("styles.css")
+      verify(amazonS3Client).putObject(putObjectRequestCaptor.capture())
+
+      val bytesToS3: InputStream = putObjectRequestCaptor.getValue.getInputStream
+      val unzippedBytesToS3 = new GZIPInputStream(bytesToS3)
+      val unzippedString = IOUtils.toString(unzippedBytesToS3, StandardCharsets.UTF_8)
+
+      unzippedString must equalTo(cssString)
+    }
+
+    def gzip(data: Array[Byte]): Array[Byte] = {
+      def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
+
+      val gzippedOutputStream: ByteArrayOutputStream = new ByteArrayOutputStream
+      using(new GZIPOutputStream(gzippedOutputStream)) { stream =>
+        IOUtils.copy(new ByteArrayInputStream(data), stream)
+      }
+      gzippedOutputStream.toByteArray
+    }
+
     "not update a gzipped S3 object if the contents has not changed" in new BasicSetup {
       config = "gzip: true"
       setLocalFileWithContent(("styles.css", "<h1>hi</h1>"))
@@ -1343,6 +1392,13 @@ class S3WebsiteSpec extends Specification {
       write(file, fileNameAndContent._2)
     }
 
+    def setLocalFileWithContent(fileName: String, contents: Array[Byte]) = {
+      val file = new File(siteDirectory, fileName)
+      forceMkdir(file.getParentFile)
+      file.createNewFile()
+      FileUtils.writeByteArrayToFile(file, contents)
+    }
+
     var config = ""
     val baseConfig =
     """

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_website
 version: !ruby/object:Gem::Version
-  version: 2.14.0
+  version: 2.14.1
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-07 00:00:00.000000000 Z
+date: 2016-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor