s3_website_monadic 0.0.30 → 0.0.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a489b6afb931e8384aa6d656acd8bf23a0c366be
-  data.tar.gz: 12234bf6f3689a83ad37a61c7029ce80900e2a67
+  metadata.gz: 240e4d2a14000e5eed029857d69e426cf7439c4f
+  data.tar.gz: ad0d71c36c1f0a8add0cca08f1de9f49467c2019
 SHA512:
-  metadata.gz: bbc33e009ccbe805fe72697433d59fd4c82691cc9edd3e95d4e8277fceb159715f8b0d4566fa2c1e85d092c20cab2eb37178e0cfde0db1fc2103682f90653a67
-  data.tar.gz: 6c8b915eab3220b1a7148d0912c267dfe7b05ac653367b033e83c9c6144d027d22d18ae2ec820314415b46433b0ec5335616a8f2cda730c46bab28577dbc8528
+  metadata.gz: 17edd14e1510e3b82604e6942a6bca4dc92b3bc34e6189fb4db5f62f2f930b32ea385794f19c1032d53f6d8c8d4600b5bd6cfc8f3ffd08f22f75e05b4b43544f
+  data.tar.gz: b3b818d8d18f40f36cf8b1661785da1fd38b8f4e9afeb78a33a37e39c5cd016bbd0824429330729e65fe01351506c044e51c9c80e372875fd4debebeaf83539f

data/build.sbt

@@ -26,6 +26,10 @@ libraryDependencies += "org.apache.tika" % "tika-core" % "1.4"
 
 libraryDependencies += "com.lexicalscope.jewelcli" % "jewelcli" % "0.8.9"
 
+libraryDependencies += "joda-time" % "joda-time" % "2.3"
+
+libraryDependencies += "org.joda" % "joda-convert" % "1.2"
+
 libraryDependencies += "org.specs2" %% "specs2" % "2.3.11" % "test"
 
 resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots"

data/changelog.md

@@ -10,6 +10,16 @@ This project uses [Semantic Versioning](http://semver.org).
 
 * Support CloudFront invalidations when the site contains over 3000 files
 
+* Display transferred bytes
+
+* Display upload speed
+
+* `push --verbose` switch
+
+### Bug fixes
+
+* Fault tolerance – do not crash if one of the uploads fails
+
 ### Java is now required
 
 * The `push` command is now written in Scala. This means that you need Java 1.6

data/lib/s3_website/version.rb

@@ -1,3 +1,3 @@
 module S3Website
-  VERSION = '0.0.30'
+  VERSION = '0.0.31'
 end

data/src/main/java/s3/website/ByteHelper.java

@@ -0,0 +1,14 @@
+package s3.website;
+
+public class ByteHelper {
+
+    // Adapted from http://stackoverflow.com/a/3758880/219947
+    public static String humanReadableByteCount(long bytes) {
+        boolean si = true;
+        int unit = si ? 1000 : 1024;
+        if (bytes < unit) return bytes + " B";
+        int exp = (int) (Math.log(bytes) / Math.log(unit));
+        String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp-1) + (si ? "" : "i");
+        return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
+    }
+}

data/src/main/scala/s3/website/CloudFront.scala

@@ -121,7 +121,7 @@ object CloudFront {
 
 
   def needsInvalidation: PartialFunction[PushSuccessReport, Boolean] = {
-    case SuccessfulUpload(upload, _) => upload.uploadType match {
+    case SuccessfulUpload(upload, _, _) => upload.uploadType match {
       case Update => true
       case _ => false
     }

data/src/main/scala/s3/website/Diff.scala

@@ -6,13 +6,15 @@ import s3.website._
 
 object Diff {
 
-  def resolveDeletes(localFiles: Seq[LocalFile], s3Files: Seq[S3File], redirects: Seq[Upload with UploadTypeResolved])(implicit config: Config): Seq[S3File] = {
+  def resolveDeletes(localFiles: Seq[LocalFile], s3Files: Seq[S3File], redirects: Seq[Upload with UploadTypeResolved])
+                    (implicit config: Config, logger: Logger): Seq[S3File] = {
     val keysNotToBeDeleted: Set[String] = (localFiles ++ redirects).map(_.s3Key).toSet
     s3Files.filterNot { s3File =>
       val ignoreOnServer = config.ignore_on_server.exists(_.fold(
         (ignoreRegex: String)        => rubyRegexMatches(s3File.s3Key, ignoreRegex),
         (ignoreRegexes: Seq[String]) => ignoreRegexes.exists(rubyRegexMatches(s3File.s3Key, _))
       ))
+      if (ignoreOnServer) logger.debug(s"Ignoring ${s3File.s3Key} on server")
       keysNotToBeDeleted.exists(_ == s3File.s3Key) || ignoreOnServer
     }
   }

data/src/main/scala/s3/website/Push.scala

@@ -3,7 +3,6 @@ package s3.website
 import s3.website.model.Site._
 import scala.concurrent.{ExecutionContextExecutor, Future, Await}
 import scala.concurrent.duration._
-import com.lexicalscope.jewel.cli.CliFactory
 import scala.language.postfixOps
 import s3.website.Diff.{resolveNewFiles, resolveDeletes}
 import s3.website.S3._
@@ -27,9 +26,54 @@ import s3.website.CloudFront.FailedInvalidation
 import scala.Int
 import java.io.File
 import com.lexicalscope.jewel.cli.CliFactory.parseArguments
+import s3.website.ByteHelper.humanReadableByteCount
 
 object Push {
 
+  def main(args: Array[String]) {
+    implicit val cliArgs = parseArguments(classOf[CliArgs], args:_*)
+    implicit val s3Settings = S3Setting()
+    implicit val cloudFrontSettings = CloudFrontSetting()
+    implicit val workingDirectory = new File(System.getProperty("user.dir")).getAbsoluteFile
+    System exit push
+  }
+
+  trait CliArgs {
+    import com.lexicalscope.jewel.cli.Option
+
+    @Option(defaultToNull = true) def site: String
+    @Option(longName = Array("config-dir"), defaultToNull = true) def configDir: String
+    @Option def verbose: Boolean
+    @Option(longName = Array("dry-run")) def dryRun: Boolean
+  }
+
+  def push(implicit cliArgs: CliArgs, s3Settings: S3Setting, cloudFrontSettings: CloudFrontSetting, workingDirectory: File): ExitCode = {
+    implicit val logger: Logger = new Logger(cliArgs.verbose)
+    implicit val pushMode = new PushMode {
+      def dryRun = cliArgs.dryRun
+    }
+
+    val errorOrSiteDir: ErrorOrFile =
+      Option(cliArgs.site).fold(Ssg.findSiteDirectory(workingDirectory))(siteDirFromCli => Right(new File(siteDirFromCli)))
+    def errorOrSite(siteInDirectory: File): Either[ErrorReport, Site] =
+      loadSite(Option(cliArgs.configDir).getOrElse(workingDirectory.getPath) + "/s3_website.yml", siteInDirectory.getAbsolutePath)
+
+    val errorOrPushStatus = for {
+      siteInDirectory <- errorOrSiteDir.right
+      loadedSite <- errorOrSite(siteInDirectory).right
+    } yield {
+      implicit val site = loadedSite
+      val threadPool = newFixedThreadPool(site.config.concurrency_level)
+      implicit val executor = fromExecutor(threadPool)
+      val pushStatus = pushSite
+      threadPool.shutdownNow()
+      pushStatus
+    }
+
+    errorOrPushStatus.left foreach (err => logger.fail(s"Could not load the site: ${err.reportMessage}"))
+    errorOrPushStatus fold((err: ErrorReport) => 1, pushStatus => pushStatus)
+  }
+
   def pushSite(
                 implicit site: Site,
                 executor: ExecutionContextExecutor,
@@ -139,34 +183,46 @@ object Push {
     })
 
   def resolvePushCounts(implicit finishedOperations: FinishedPushOperations) = finishedOperations.foldLeft(PushCounts()) {
-    (counts: PushCounts, uploadReport) => uploadReport.fold(
-      (error: ErrorReport) => counts.copy(failures = counts.failures + 1),
-      failureOrSuccess => failureOrSuccess.fold(
-        (failureReport: PushFailureReport) => counts.copy(failures = counts.failures + 1),
-        (successReport: PushSuccessReport) => successReport match {
-          case SuccessfulUpload(upload, _) => upload.uploadType match {
-            case NewFile  => counts.copy(newFiles = counts.newFiles + 1)
-            case Update   => counts.copy(updates = counts.updates + 1)
-            case Redirect => counts.copy(redirects = counts.redirects + 1)
+    (counts: PushCounts, uploadReport) =>
+      uploadReport.fold(
+        (error: ErrorReport) => counts.copy(failures = counts.failures + 1),
+        failureOrSuccess => failureOrSuccess.fold(
+          (failureReport: PushFailureReport) => counts.copy(failures = counts.failures + 1),
+          (successReport: PushSuccessReport) => successReport match {
+            case succ: SuccessfulUpload => succ.upload.uploadType match {
+              case NewFile => counts.copy(newFiles = counts.newFiles + 1).addTransferStats(succ) // TODO nasty repetition here
+              case Update => counts.copy(updates = counts.updates + 1).addTransferStats(succ)
+              case Redirect => counts.copy(redirects = counts.redirects + 1).addTransferStats(succ)
+            }
+            case SuccessfulDelete(_) => counts.copy(deletes = counts.deletes + 1)
           }
-          case SuccessfulDelete(_) => counts.copy(deletes = counts.deletes + 1)
-        }
+        )
       )
-    )
   }
 
   def pushCountsToString(pushCounts: PushCounts)(implicit pushMode: PushMode): String =
     pushCounts match {
-      case PushCounts(updates, newFiles, failures, redirects, deletes)
+      case PushCounts(updates, newFiles, failures, redirects, deletes, _, _)
         if updates == 0 && newFiles == 0 && failures == 0 && redirects == 0 && deletes == 0 =>
           PushNothing.renderVerb
-      case PushCounts(updates, newFiles, failures, redirects, deletes) =>
+      case PushCounts(updates, newFiles, failures, redirects, deletes, uploadedBytes, uploadDurationAndFrequency) =>
         val reportClauses: scala.collection.mutable.ArrayBuffer[String] = ArrayBuffer()
-        if (updates > 0)   reportClauses += s"${Updated.renderVerb} ${updates ofType "file"}."
-        if (newFiles > 0)  reportClauses += s"${Created.renderVerb} ${newFiles ofType "file"}."
-        if (failures > 0)  reportClauses += s"${failures ofType "operation"} failed." // This includes both failed uploads and deletes.
-        if (redirects > 0) reportClauses += s"${Applied.renderVerb} ${redirects ofType "redirect"}."
-        if (deletes > 0)   reportClauses += s"${Deleted.renderVerb} ${deletes ofType "file"}."
+        if (updates > 0)       reportClauses += s"${Updated.renderVerb} ${updates ofType "file"}."
+        if (newFiles > 0)      reportClauses += s"${Created.renderVerb} ${newFiles ofType "file"}."
+        if (failures > 0)      reportClauses += s"${failures ofType "operation"} failed." // This includes both failed uploads and deletes.
+        if (redirects > 0)     reportClauses += s"${Applied.renderVerb} ${redirects ofType "redirect"}."
+        if (deletes > 0)       reportClauses += s"${Deleted.renderVerb} ${deletes ofType "file"}."
+        if (uploadedBytes > 0) {
+          val transferSuffix =
+            if (uploadDurationAndFrequency._1.getStandardSeconds > 0)
+              s", ${humanReadableByteCount(
+                (uploadedBytes / uploadDurationAndFrequency._1.getMillis * 1000) * uploadDurationAndFrequency._2
+              )}/s."
+            else
+              "."
+
+          reportClauses += s"${Transferred.renderVerb} ${humanReadableByteCount(uploadedBytes)}$transferSuffix"
+        }
         reportClauses.mkString(" ")
     }
 
@@ -175,56 +231,24 @@ object Push {
                          newFiles: Int = 0, 
                          failures: Int = 0, 
                          redirects: Int = 0, 
-                         deletes: Int = 0
+                         deletes: Int = 0,
+                         uploadedBytes: Long = 0,
+                         uploadDurationAndFrequency: (org.joda.time.Duration, Int) = (new org.joda.time.Duration(0), 0)
                          ) {
     val thereWasSomethingToPush = updates + newFiles + redirects + deletes > 0
+
+    def addTransferStats(successfulUpload: SuccessfulUpload): PushCounts = {
+      copy(
+        uploadedBytes = uploadedBytes + (successfulUpload.uploadSize getOrElse 0L),
+        uploadDurationAndFrequency = successfulUpload.uploadDuration.fold(uploadDurationAndFrequency)(
+          dur => (uploadDurationAndFrequency._1.plus(dur), uploadDurationAndFrequency._2 + 1)
+        )
+      )
+    }
   }
+
   type FinishedPushOperations = ParSeq[Either[ErrorReport, PushErrorOrSuccess]]
   type PushReports = ParSeq[Either[ErrorReport, Future[PushErrorOrSuccess]]]
   case class PushResult(threadPool: ExecutorService, uploadReports: PushReports)
   type ExitCode = Int
-
-  trait CliArgs {
-    import com.lexicalscope.jewel.cli.Option
-
-    @Option(defaultToNull = true) def site: String
-    @Option(longName = Array("config-dir"), defaultToNull = true) def configDir: String
-    @Option def verbose: Boolean
-    @Option(longName = Array("dry-run")) def dryRun: Boolean
-  }
-
-  def main(args: Array[String]) {
-    implicit val cliArgs = parseArguments(classOf[CliArgs], args:_*)
-    implicit val s3Settings = S3Setting()
-    implicit val cloudFrontSettings = CloudFrontSetting()
-    implicit val workingDirectory = new File(System.getProperty("user.dir")).getAbsoluteFile
-    System exit push
-  }
-
-  def push(implicit cliArgs: CliArgs, s3Settings: S3Setting, cloudFrontSettings: CloudFrontSetting, workingDirectory: File): ExitCode = {
-    implicit val logger: Logger = new Logger(cliArgs.verbose)
-    implicit val pushMode = new PushMode {
-      def dryRun = cliArgs.dryRun
-    }
-
-    val errorOrSiteDir: ErrorOrFile =
-      Option(cliArgs.site).fold(Ssg.findSiteDirectory(workingDirectory))(siteDirFromCli => Right(new File(siteDirFromCli)))
-    def errorOrSite(siteInDirectory: File): Either[ErrorReport, Site] =
-      loadSite(Option(cliArgs.configDir).getOrElse(workingDirectory.getPath) + "/s3_website.yml", siteInDirectory.getAbsolutePath)
-
-    val errorOrPushStatus = for {
-      siteInDirectory <- errorOrSiteDir.right
-      loadedSite <- errorOrSite(siteInDirectory).right
-    } yield {
-      implicit val site = loadedSite
-      val threadPool = newFixedThreadPool(site.config.concurrency_level)
-      implicit val executor = fromExecutor(threadPool)
-      val pushStatus = pushSite
-      threadPool.shutdownNow()
-      pushStatus
-    }
-
-    errorOrPushStatus.left foreach (err => logger.fail(s"Could not load the site: ${err.reportMessage}"))
-    errorOrPushStatus fold((err: ErrorReport) => 1, pushStatus => pushStatus)
-  }
 }

data/src/main/scala/s3/website/S3.scala

@@ -7,23 +7,29 @@ import com.amazonaws.services.s3.model._
 import scala.collection.JavaConversions._
 import scala.concurrent.{ExecutionContextExecutor, Future}
 import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
-import scala.concurrent.duration.TimeUnit
-import java.util.concurrent.TimeUnit.SECONDS
 import s3.website.S3.SuccessfulUpload
 import s3.website.S3.SuccessfulDelete
 import s3.website.S3.FailedUpload
 import scala.Some
 import s3.website.S3.FailedDelete
 import s3.website.S3.S3Setting
+import s3.website.ByteHelper.humanReadableByteCount
+import org.joda.time.{Seconds, Duration, Interval}
+import scala.concurrent.duration.TimeUnit
+import java.util.concurrent.TimeUnit
+import scala.concurrent.duration.TimeUnit
+import java.util.concurrent.TimeUnit.SECONDS
 
-class S3(implicit s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor) {
+class S3(implicit s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger) {
 
   def upload(upload: Upload with UploadTypeResolved, a: Attempt = 1)
-            (implicit config: Config, logger: Logger): Future[Either[FailedUpload, SuccessfulUpload]] =
+            (implicit config: Config): Future[Either[FailedUpload, SuccessfulUpload]] =
     Future {
       val putObjectRequest = toPutObjectRequest(upload)
-      if (!pushMode.dryRun) s3Settings.s3Client(config) putObject putObjectRequest
-      val report = SuccessfulUpload(upload, putObjectRequest)
+      val uploadDuration =
+        if (pushMode.dryRun) None
+        else recordUploadDuration(putObjectRequest, s3Settings.s3Client(config) putObject putObjectRequest)
+      val report = SuccessfulUpload(upload, putObjectRequest, uploadDuration)
       logger.info(report)
       Right(report)
     } recoverWith retry(a)(
@@ -32,7 +38,7 @@ class S3(implicit s3Settings: S3Setting, pushMode: PushMode, executor: Execution
     )
 
   def delete(s3Key: String,  a: Attempt = 1)
-            (implicit config: Config, logger: Logger): Future[Either[FailedDelete, SuccessfulDelete]] =
+            (implicit config: Config): Future[Either[FailedDelete, SuccessfulDelete]] =
     Future {
       if (!pushMode.dryRun) s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
       val report = SuccessfulDelete(s3Key)
@@ -42,7 +48,7 @@ class S3(implicit s3Settings: S3Setting, pushMode: PushMode, executor: Execution
       createFailureReport = error => FailedDelete(s3Key, error),
       retryAction  = newAttempt => this.delete(s3Key, newAttempt)
     )
-      
+
   def toPutObjectRequest(upload: Upload)(implicit config: Config) =
     upload.essence.fold(
       redirect => {
@@ -77,6 +83,15 @@ class S3(implicit s3Settings: S3Setting, pushMode: PushMode, executor: Execution
         req
       }
     )
+
+  def recordUploadDuration(putObjectRequest: PutObjectRequest, f: => Unit): Option[Duration] = {
+    val start = System.currentTimeMillis()
+    f
+    if (putObjectRequest.getMetadata.getContentLength > 0)
+      Some(new Duration(start, System.currentTimeMillis))
+    else
+      None // We are not interested in tracking durations of PUT requests that don't contain data. Redirect is an example of such request.
+  }
 }
 
 object S3 {
@@ -138,21 +153,47 @@ object S3 {
     def s3Key: String
   }
 
-  case class SuccessfulUpload(upload: Upload with UploadTypeResolved, putObjectRequest: PutObjectRequest)
-                             (implicit pushMode: PushMode) extends PushSuccessReport {
-    val metadata = putObjectRequest.getMetadata
-    def metadataReport =
-      (metadata.getCacheControl :: metadata.getContentType :: metadata.getContentEncoding :: putObjectRequest.getStorageClass :: Nil)
-        .filterNot(_ == null)
-        .mkString(" | ")
-
+  case class SuccessfulUpload(upload: Upload with UploadTypeResolved, putObjectRequest: PutObjectRequest, uploadDuration: Option[Duration])
+                             (implicit pushMode: PushMode, logger: Logger) extends PushSuccessReport {
     def reportMessage =
       upload.uploadType match {
-        case NewFile  => s"${Created.renderVerb} ${upload.s3Key} ($metadataReport)"
-        case Update   => s"${Updated.renderVerb} ${upload.s3Key} ($metadataReport)"
+        case NewFile  => s"${Created.renderVerb} $s3Key ($reportDetails)"
+        case Update   => s"${Updated.renderVerb} $s3Key ($reportDetails)"
         case Redirect => s"${Redirected.renderVerb} ${upload.essence.left.get.key} to ${upload.essence.left.get.redirectTarget}"
       }
 
+    def reportDetails = {
+      val md = putObjectRequest.getMetadata
+      val detailFragments: Seq[Option[String]] =
+        (
+          md.getCacheControl ::
+          md.getContentType ::
+          md.getContentEncoding ::
+          putObjectRequest.getStorageClass ::
+          Nil map (Option(_)) // AWS SDK may return nulls
+        ) :+ uploadSizeForHumans :+ uploadSpeedForHumans
+      detailFragments.collect {
+        case Some(detailFragment) => detailFragment
+      }.mkString(" | ")
+    }
+
+    lazy val uploadSize: Option[Long] =
+      upload.essence.fold(
+        (redirect: Redirect) => None,
+        uploadBody           => Some(uploadBody.contentLength)
+      )
+
+    lazy val uploadSizeForHumans: Option[String] = uploadSize filter (_ => logger.verboseOutput) map humanReadableByteCount
+
+    lazy val uploadSpeed: Option[Long] = for {
+      dataSize <- uploadSize
+      duration <- uploadDuration
+    } yield (dataSize / (duration.getMillis max 1)) * 1000 // Precision tweaking and avoidance of divide-by-zero
+
+    lazy val uploadSpeedForHumans: Option[String] = uploadSpeed filter (_ => logger.verboseOutput) map {
+      bytesPerSecond => s"${humanReadableByteCount(bytesPerSecond)}/s"
+    }
+
     def s3Key = upload.s3Key
   }
 

data/src/main/scala/s3/website/Utils.scala

@@ -16,7 +16,7 @@ object Utils {
   lazy val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map { n => n._1 + n._2 }
 }
 
-class Logger(verboseOutput: Boolean, logMessage: (String) => Unit = println) {
+class Logger(val verboseOutput: Boolean, logMessage: (String) => Unit = println) {
   import Rainbow._
   def debug(msg: String) = if (verboseOutput) log(Debug, msg)
   def info(msg: String) = log(Info, msg)

data/src/main/scala/s3/website/model/push.scala

@@ -137,17 +137,19 @@ object LocalFile {
     }
   }
 
-  def resolveLocalFiles(implicit site: Site): Either[ErrorReport, Seq[LocalFile]] = Try {
+  def resolveLocalFiles(implicit site: Site, logger: Logger): Either[ErrorReport, Seq[LocalFile]] = Try {
     val files = recursiveListFiles(new File(site.rootDirectory)).filterNot(_.isDirectory)
     files map { file =>
       val s3Key = site.resolveS3Key(file)
       LocalFile(s3Key, file, encodingOnS3(s3Key))
     } filterNot { file =>
-      site.config.exclude_from_upload exists { _.fold(
+      val excludeFile = site.config.exclude_from_upload exists { _.fold(
         // For backward compatibility, use Ruby regex matching
         (exclusionRegex: String) => rubyRegexMatches(file.s3Key, exclusionRegex),
         (exclusionRegexes: Seq[String]) => exclusionRegexes exists (rubyRegexMatches(file.s3Key, _))
       ) }
+      if (excludeFile) logger.debug(s"Excluded ${file.s3Key} from upload")
+      excludeFile
     } filterNot { _.originalFile.getName == "s3_website.yml" } // For security reasons, the s3_website.yml should never be pushed
   } match {
     case Success(localFiles) =>

data/src/main/scala/s3/website/package.scala

@@ -39,6 +39,7 @@ package object website {
   case object Updated     extends PushAction
   case object Redirected  extends PushAction
   case object Deleted     extends PushAction
+  case object Transferred extends PushAction
   case object Invalidated extends PushAction
   case object Applied     extends PushAction
   case object PushNothing extends PushAction {

data/src/test/scala/s3/website/S3WebsiteSpec.scala

@@ -23,7 +23,7 @@ import org.mockito.invocation.InvocationOnMock
 import java.util.concurrent.atomic.AtomicInteger
 import org.apache.commons.io.FileUtils.{forceDelete, forceMkdir, write}
 import scala.collection.mutable
-import s3.website.Push.CliArgs
+import s3.website.Push.{push, CliArgs}
 
 class S3WebsiteSpec extends Specification {
 
@@ -32,7 +32,7 @@ class S3WebsiteSpec extends Specification {
       config = "gzip: true"
       setLocalFileWithContent(("styles.css", "<h1>hi again</h1>"))
       setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
-      Push.push
+      push
       sentPutObjectRequest.getKey must equalTo("styles.css")
     }
 
@@ -40,7 +40,7 @@ class S3WebsiteSpec extends Specification {
       config = "gzip: true"
       setLocalFileWithContent(("styles.css", "<h1>hi</h1>"))
       setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
-      Push.push
+      push
       noUploadsOccurred must beTrue
     }
   }
@@ -56,7 +56,7 @@ class S3WebsiteSpec extends Specification {
       """.stripMargin
       setLocalFileWithContent(("file.xml", "<h1>hi again</h1>"))
       setS3Files(S3File("file.xml", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
-      Push.push
+      push
       sentPutObjectRequest.getKey must equalTo("file.xml")
     }
   }
@@ -65,33 +65,33 @@ class S3WebsiteSpec extends Specification {
     "not upload a file if it has not changed" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFileWithContent(("index.html", "<div>hello</div>"))
       setS3Files(S3File("index.html", md5Hex("<div>hello</div>")))
-      Push.push
+      push
       noUploadsOccurred must beTrue
     }
 
     "update a file if it has changed" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFileWithContent(("index.html", "<h1>old text</h1>"))
       setS3Files(S3File("index.html", md5Hex("<h1>new text</h1>")))
-      Push.push
+      push
       sentPutObjectRequest.getKey must equalTo("index.html")
     }
 
     "create a file if does not exist on S3" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile("index.html")
-      Push.push
+      push
       sentPutObjectRequest.getKey must equalTo("index.html")
     }
 
     "delete files that are on S3 but not on local file system" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
-      Push.push
+      push
       sentDelete must equalTo("old.html")
     }
 
     "try again if the upload fails" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile("index.html")
       uploadFailsAndThenSucceeds(howManyFailures = 5)
-      Push.push
+      push
       verify(amazonS3Client, times(6)).putObject(Matchers.any(classOf[PutObjectRequest]))
     }
 
@@ -102,21 +102,21 @@ class S3WebsiteSpec extends Specification {
         e.setStatusCode(403)
         e
       }
-      Push.push
+      push
       verify(amazonS3Client, times(1)).putObject(Matchers.any(classOf[PutObjectRequest]))
     }
 
     "try again if the delete fails" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
       deleteFailsAndThenSucceeds(howManyFailures = 5)
-      Push.push
+      push
       verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
     }
 
     "try again if the object listing fails" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
       objectListingFailsAndThenSucceeds(howManyFailures = 5)
-      Push.push
+      push
       verify(amazonS3Client, times(6)).listObjects(Matchers.any(classOf[ListObjectsRequest]))
     }
   }
@@ -126,14 +126,14 @@ class S3WebsiteSpec extends Specification {
       config = "cloudfront_distribution_id: EGM1J2JJX9Z"
       setLocalFiles("css/test.css", "articles/index.html")
       setOutdatedS3Keys("css/test.css", "articles/index.html")
-      Push.push
+      push
       sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/css/test.css" :: "/articles/index.html" :: Nil).sorted)
     }
 
     "not send CloudFront invalidation requests on new objects"  in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       config = "cloudfront_distribution_id: EGM1J2JJX9Z"
       setLocalFile("newfile.js")
-      Push.push
+      push
       noInvalidationsOccurred must beTrue
     }
 
@@ -143,7 +143,7 @@ class S3WebsiteSpec extends Specification {
         |redirects:
         |  /index.php: index.html
       """.stripMargin
-      Push.push
+      push
       noInvalidationsOccurred must beTrue
     }
 
@@ -152,7 +152,7 @@ class S3WebsiteSpec extends Specification {
       config = "cloudfront_distribution_id: EGM1J2JJX9Z"
       setLocalFile("test.css")
       setOutdatedS3Keys("test.css")
-      Push.push must equalTo(0) // The retries should finally result in a success
+      push must equalTo(0) // The retries should finally result in a success
       sentInvalidationRequests.length must equalTo(4)
     }
 
@@ -161,7 +161,7 @@ class S3WebsiteSpec extends Specification {
       config = "cloudfront_distribution_id: EGM1J2JJX9Z"
       setLocalFile("test.css")
       setOutdatedS3Keys("test.css")
-      Push.push
+      push
       sentInvalidationRequests.length must equalTo(6)
     }
 
@@ -169,7 +169,7 @@ class S3WebsiteSpec extends Specification {
       config = "cloudfront_distribution_id: EGM1J2JJX9Z"
       setLocalFile("articles/arnold's file.html")
       setOutdatedS3Keys("articles/arnold's file.html")
-      Push.push
+      push
       sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/articles/arnold's%20file.html" :: Nil).sorted)
     }
 
@@ -177,7 +177,7 @@ class S3WebsiteSpec extends Specification {
       config = "cloudfront_distribution_id: EGM1J2JJX9Z"
       setLocalFile("maybe-index.html")
       setOutdatedS3Keys("maybe-index.html")
-      Push.push
+      push
       sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/" :: "/maybe-index.html" :: Nil).sorted)
     }
   }
@@ -190,7 +190,7 @@ class S3WebsiteSpec extends Specification {
       """.stripMargin
       setLocalFile("articles/index.html")
       setOutdatedS3Keys("articles/index.html")
-      Push.push
+      push
       sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/articles/" :: Nil).sorted)
     }
   }
@@ -201,7 +201,7 @@ class S3WebsiteSpec extends Specification {
       config = "cloudfront_distribution_id: EGM1J2JJX9Z"
       setLocalFiles(files:_*)
       setOutdatedS3Keys(files:_*)
-      Push.push
+      push
       sentInvalidationRequests.length must equalTo(2)
       sentInvalidationRequests(0).getInvalidationBatch.getPaths.getItems.length must equalTo(1000)
       sentInvalidationRequests(1).getInvalidationBatch.getPaths.getItems.length must equalTo(2)
@@ -211,13 +211,13 @@ class S3WebsiteSpec extends Specification {
   "push exit status" should {
     "be 0 all uploads succeed" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFiles("file.txt")
-      Push.push must equalTo(0)
+      push must equalTo(0)
     }
 
     "be 1 if any of the uploads fails" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFiles("file.txt")
       when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow(new AmazonServiceException("AWS failed"))
-      Push.push must equalTo(1)
+      push must equalTo(1)
     }
 
     "be 1 if any of the redirects fails" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
@@ -226,14 +226,14 @@ class S3WebsiteSpec extends Specification {
         |  index.php: /index.html
       """.stripMargin
       when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow(new AmazonServiceException("AWS failed"))
-      Push.push must equalTo(1)
+      push must equalTo(1)
     }
 
     "be 0 if CloudFront invalidations and uploads succeed"in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       config = "cloudfront_distribution_id: EGM1J2JJX9Z"
       setLocalFile("test.css")
       setOutdatedS3Keys("test.css")
-      Push.push must equalTo(0)
+      push must equalTo(0)
     }
 
     "be 1 if CloudFront is unreachable or broken"in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
@@ -241,32 +241,32 @@ class S3WebsiteSpec extends Specification {
       config = "cloudfront_distribution_id: EGM1J2JJX9Z"
       setLocalFile("test.css")
       setOutdatedS3Keys("test.css")
-      Push.push must equalTo(1)
+      push must equalTo(1)
     }
 
     "be 0 if upload retry succeeds" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile("index.html")
       uploadFailsAndThenSucceeds(howManyFailures = 1)
-      Push.push must equalTo(0)
+      push must equalTo(0)
     }
 
     "be 1 if delete retry fails" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile("index.html")
       uploadFailsAndThenSucceeds(howManyFailures = 6)
-      Push.push must equalTo(1)
+      push must equalTo(1)
     }
 
     "be 1 if an object listing fails" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
       objectListingFailsAndThenSucceeds(howManyFailures = 6)
-      Push.push must equalTo(1)
+      push must equalTo(1)
     }
   }
 
   "s3_website.yml file" should {
     "never be uploaded" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile("s3_website.yml")
-      Push.push
+      push
       noUploadsOccurred must beTrue
     }
   }
@@ -275,7 +275,7 @@ class S3WebsiteSpec extends Specification {
     "result in matching files not being uploaded" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       config = "exclude_from_upload: .DS_.*?"
       setLocalFile(".DS_Store")
-      Push.push
+      push
       noUploadsOccurred must beTrue
     }
   }
@@ -292,7 +292,7 @@ class S3WebsiteSpec extends Specification {
         |  - logs
       """.stripMargin
       setLocalFiles(".DS_Store", "logs/test.log")
-      Push.push
+      push
       noUploadsOccurred must beTrue
     }
   }
@@ -301,7 +301,7 @@ class S3WebsiteSpec extends Specification {
     "not delete the S3 objects that match the ignore value" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       config = "ignore_on_server: logs"
       setS3Files(S3File("logs/log.txt", ""))
-      Push.push
+      push
       noDeletesOccurred must beTrue
     }
   }
@@ -317,7 +317,7 @@ class S3WebsiteSpec extends Specification {
         |  - .*txt
       """.stripMargin
       setS3Files(S3File("logs/log.txt", ""))
-      Push.push
+      push
       noDeletesOccurred must beTrue
     }
   }
@@ -326,7 +326,7 @@ class S3WebsiteSpec extends Specification {
     "be applied to all files" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       config = "max_age: 60"
       setLocalFile("index.html")
-      Push.push
+      push
       sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=60")
     }
 
@@ -336,7 +336,7 @@ class S3WebsiteSpec extends Specification {
         |  "*.html": 90
       """.stripMargin
       setLocalFile("index.html")
-      Push.push
+      push
       sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
     }
 
@@ -346,7 +346,7 @@ class S3WebsiteSpec extends Specification {
         |  "assets/**/*.js": 90
       """.stripMargin
       setLocalFile("assets/lib/jquery.js")
-      Push.push
+      push
       sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
     }
 
@@ -356,14 +356,14 @@ class S3WebsiteSpec extends Specification {
         |  "*.js": 90
       """.stripMargin
       setLocalFile("index.html")
-      Push.push
+      push
       sentPutObjectRequest.getMetadata.getCacheControl must beNull
     }
 
     "be used to disable caching" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       config = "max_age: 0"
       setLocalFile("index.html")
-      Push.push
+      push
       sentPutObjectRequest.getMetadata.getCacheControl must equalTo("no-cache; max-age=0")
     }
   }
@@ -376,7 +376,7 @@ class S3WebsiteSpec extends Specification {
         |  "assets/*.gif": 86400
       """.stripMargin
       setLocalFiles("assets/jquery.js", "assets/picture.gif")
-      Push.push
+      push
       sentPutObjectRequests.find(_.getKey == "assets/jquery.js").get.getMetadata.getCacheControl must equalTo("max-age=150")
       sentPutObjectRequests.find(_.getKey == "assets/picture.gif").get.getMetadata.getCacheControl must equalTo("max-age=86400")
     }
@@ -386,7 +386,7 @@ class S3WebsiteSpec extends Specification {
     "result in uploads being marked with reduced redundancy" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       config = "s3_reduced_redundancy: true"
       setLocalFile("file.exe")
-      Push.push
+      push
       sentPutObjectRequest.getStorageClass must equalTo("REDUCED_REDUNDANCY")
     }
   }
@@ -395,7 +395,7 @@ class S3WebsiteSpec extends Specification {
     "result in uploads being marked with the default storage class" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       config = "s3_reduced_redundancy: false"
       setLocalFile("file.exe")
-      Push.push
+      push
       sentPutObjectRequest.getStorageClass must beNull
     }
   }
@@ -406,7 +406,7 @@ class S3WebsiteSpec extends Specification {
         |redirects:
         |  index.php: /index.html
       """.stripMargin
-      Push.push
+      push
       sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
     }
 
@@ -415,7 +415,7 @@ class S3WebsiteSpec extends Specification {
         |redirects:
         |  index.php: /index.html
       """.stripMargin
-      Push.push
+      push
       sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=0, no-cache")
     }
   }
@@ -428,7 +428,7 @@ class S3WebsiteSpec extends Specification {
       """.stripMargin
       setLocalFile("index.php")
       setS3Files(S3File("index.php", "md5"))
-      Push.push
+      push
       noDeletesOccurred must beTrue
     }
   }
@@ -436,7 +436,7 @@ class S3WebsiteSpec extends Specification {
   "dotfiles" should {
     "be included in the pushed files" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile(".vimrc")
-      Push.push
+      push
       sentPutObjectRequest.getKey must equalTo(".vimrc")
     }
   }
@@ -444,25 +444,25 @@ class S3WebsiteSpec extends Specification {
   "content type inference" should {
     "add charset=utf-8 to all html documents" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile("index.html")
-      Push.push
+      push
       sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
     }
 
     "add charset=utf-8 to all text documents" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile("index.txt")
-      Push.push
+      push
       sentPutObjectRequest.getMetadata.getContentType must equalTo("text/plain; charset=utf-8")
     }
 
     "add charset=utf-8 to all json documents" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile("data.json")
-      Push.push
+      push
       sentPutObjectRequest.getMetadata.getContentType must equalTo("application/json; charset=utf-8")
     }
 
     "resolve the content type from file contents" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
       setLocalFileWithContent(("index", "<html><body><h1>hi</h1></body></html>"))
-      Push.push
+      push
       sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
     }
   }
@@ -473,7 +473,7 @@ class S3WebsiteSpec extends Specification {
         |redirects:
         |<%= ('a'..'f').to_a.map do |t| '  '+t+ ': /'+t+'.html' end.join('\n')%>
       """.stripMargin
-      Push.push
+      push
       sentPutObjectRequests.length must equalTo(6)
       sentPutObjectRequests.forall(_.getRedirectLocation != null) must beTrue
     }
@@ -483,7 +483,7 @@ class S3WebsiteSpec extends Specification {
     "not push updates" in new AllInSameDirectory with EmptySite with MockAWS with DryRunMode {
       setLocalFileWithContent(("index.html", "<div>new</div>"))
       setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
-      Push.push
+      push
       noUploadsOccurred must beTrue
     }
 
@@ -493,26 +493,26 @@ class S3WebsiteSpec extends Specification {
           |redirects:
           |  index.php: /index.html
         """.stripMargin
-      Push.push
+      push
       noUploadsOccurred must beTrue
     }
 
     "not push deletes" in new AllInSameDirectory with EmptySite with MockAWS with DryRunMode {
       setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
-      Push.push
+      push
       noUploadsOccurred must beTrue
     }
 
     "not push new files" in new AllInSameDirectory with EmptySite with MockAWS with DryRunMode {
       setLocalFile("index.html")
-      Push.push
+      push
       noUploadsOccurred must beTrue
     }
     
     "not invalidate files" in new AllInSameDirectory with EmptySite with MockAWS with DryRunMode {
       config = "cloudfront_invalidation_id: AABBCC"
       setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
-      Push.push
+      push
       noInvalidationsOccurred must beTrue
     }
   }
@@ -520,7 +520,7 @@ class S3WebsiteSpec extends Specification {
   "Jekyll site" should {
     "be detected automatically" in new JekyllSite with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile("index.html")
-      Push.push
+      push
       sentPutObjectRequests.length must equalTo(1)
     }
   }
@@ -528,7 +528,7 @@ class S3WebsiteSpec extends Specification {
   "Nanoc site" should {
     "be detected automatically" in new NanocSite with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile("index.html")
-      Push.push
+      push
       sentPutObjectRequests.length must equalTo(1)
     }
   }
@@ -774,21 +774,4 @@ class S3WebsiteSpec extends Specification {
       buildSite(config)
     }
   }
-
-  val defaultConfig = Config(
-    s3_id = "foo",
-    s3_secret = "bar",
-    s3_bucket = "bucket",
-    s3_endpoint = S3Endpoint.defaultEndpoint,
-    max_age = None,
-    gzip = None,
-    gzip_zopfli = None,
-    ignore_on_server = None,
-    exclude_from_upload = None,
-    s3_reduced_redundancy = None,
-    cloudfront_distribution_id = None,
-    cloudfront_invalidate_root = None,
-    redirects = None,
-    concurrency_level = 1
-  )
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_website_monadic
 version: !ruby/object:Gem::Version
-  version: 0.0.30
+  version: 0.0.31
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-13 00:00:00.000000000 Z
+date: 2014-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -110,6 +110,7 @@ files:
 - resources/configuration_file_template.yml
 - s3_website.gemspec
 - sbt
+- src/main/java/s3/website/ByteHelper.java
 - src/main/scala/s3/website/CloudFront.scala
 - src/main/scala/s3/website/Diff.scala
 - src/main/scala/s3/website/Push.scala