s3_website_monadic 0.0.37 → 0.0.38

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2b7836ea48c1ed8fed64738ea35590b2e7f2c0cc
-  data.tar.gz: 4d41438c3b50479d1dad20d8baee449a35786a68
+  metadata.gz: df94a2c8b44f12db2fc393cd4de9126c68ba3f17
+  data.tar.gz: d9cd16f493ca0e4d861a7ed2b6b176cfdf58651e
 SHA512:
-  metadata.gz: 4bc9ca2bb1226506c00eb47943057f6e2cbf86e655083f6d1d9ab42730ab30e2f0acd66d47fa2ceccdae378e85f67e48e3e4cbf3279ab289a5c78657e36ffce1
-  data.tar.gz: e6c3ac49a91edc6643942b2005cd6f63ecce88c69f8e587aa6bf6f47b9a94fd4e79a5f0fc49deb6c0dfa125fb74613b30695a78acef0fd512ec45f5afbaf4a58
+  metadata.gz: 377416d509c1df1ff449da81f87888aaf86cfec2423ddb615ac974b2c2383eef32ae1c6aee373b4ae177656fa4aebc052e95e8965131e07387c138143ffbf2b8
+  data.tar.gz: a89614a07d46403aaf97e29b693677ee2f704fece8836721f822fa17b728465e8c8b59e2d168f8b108e997ccc6d84708d839e8db9cdcbdc924b27418e2b055c4

data/bin/s3_website_monadic

@@ -119,6 +119,7 @@ def autoinstall_java_or_print_help_and_exit(logger)
 
   def print_manual_method_and_exit
     @logger.info_msg 'Go to http://java.com, install Java and then try again.'
+    @logger.info_msg "(If you cannot or do not want to install Java, you can use latest 1.x version of this gem, which requires only Ruby. For more info, see https://github.com/laurilehmijoki/s3_website/tree/1.x)"
     exit 1
   end
 

data/build.sbt

@@ -26,10 +26,6 @@ libraryDependencies += "org.apache.tika" % "tika-core" % "1.4"
 
 libraryDependencies += "com.lexicalscope.jewelcli" % "jewelcli" % "0.8.9"
 
-libraryDependencies += "joda-time" % "joda-time" % "2.3"
-
-libraryDependencies += "org.joda" % "joda-convert" % "1.2"
-
 libraryDependencies += "org.specs2" %% "specs2" % "2.3.11" % "test"
 
 resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots"

data/lib/s3_website/version.rb

@@ -1,3 +1,3 @@
 module S3Website
-  VERSION = '0.0.37'
+  VERSION = '0.0.38'
 end

data/src/main/scala/s3/website/CloudFront.scala

@@ -117,7 +117,7 @@ object CloudFront {
 
 
   def needsInvalidation: PartialFunction[PushSuccessReport, Boolean] = {
-    case SuccessfulUpload(localFile, _, _) => localFile.left.exists(_.uploadType == FileUpdate)
+    case succ: SuccessfulUpload => succ.details.fold(_.uploadType, _.uploadType) == FileUpdate
     case SuccessfulDelete(_) => true
     case _ => false
   }

data/src/main/scala/s3/website/Diff.scala

@@ -19,7 +19,7 @@ case class Diff(
 
 object Diff {
 
-  type UploadBatch = Future[Either[ErrorReport, Seq[LocalFileFromDisk]]]
+  type UploadBatch = Future[Either[ErrorReport, Seq[LocalFile]]]
 
   def resolveDiff(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
                  (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] =
@@ -28,26 +28,25 @@ object Diff {
 
   private def resolveDiffAgainstGetBucketResponse(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
                                                  (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
-    val diffSrc = s3FilesFuture.map { errorOrS3Files =>
+    val diffAgainstS3 = s3FilesFuture.map { errorOrS3Files =>
       errorOrS3Files.right.flatMap { s3Files =>
         Try {
           val s3KeyIndex = s3Files.map(_.s3Key).toSet
           val s3Md5Index = s3Files.map(_.md5).toSet
           val siteFiles = Files.listSiteFiles
-          val nameExistsOnS3 = (f: File) => s3KeyIndex contains site.resolveS3Key(f)
-          val newFiles = siteFiles
-            .filterNot(nameExistsOnS3)
-            .map { f => LocalFileFromDisk(f, uploadType = NewFile)}
-          val changedFiles =
-            siteFiles
-              .filter(nameExistsOnS3)
-              .map(f => LocalFileFromDisk(f, uploadType = FileUpdate))
-              .filterNot(localFile => s3Md5Index contains localFile.md5)
+          val existsOnS3 = (f: File) => s3KeyIndex contains site.resolveS3Key(f)
+          val isChangedOnS3 = (localFile: LocalFile) => !(s3Md5Index contains localFile.md5.get)
+          val newFiles = siteFiles collect {
+            case file if !existsOnS3(file) => LocalFile(file, NewFile)
+          }
+          val changedFiles = siteFiles collect {
+            case file if existsOnS3(file) => LocalFile(file, FileUpdate)
+          } filter isChangedOnS3
           val unchangedFiles = {
             val newOrChangedFiles = (changedFiles ++ newFiles).map(_.originalFile).toSet
             siteFiles.filterNot(f => newOrChangedFiles contains f)
           }
-          val allFiles: Seq[Either[DbRecord, LocalFileFromDisk]] = unchangedFiles.map {
+          val allFiles: Seq[Either[DbRecord, LocalFile]] = unchangedFiles.map {
             f => Left(DbRecord(f))
           } ++ (changedFiles ++ newFiles).map {
             Right(_)
@@ -60,8 +59,8 @@ object Diff {
         }
       }
     }
-    def collectResult[B](pf: PartialFunction[Either[DbRecord, LocalFileFromDisk],B]) =
-      diffSrc.map { errorOrDiffSource =>
+    def collectResult[B](pf: PartialFunction[Either[DbRecord, LocalFile],B]) =
+      diffAgainstS3.map { errorOrDiffSource =>
         errorOrDiffSource.right map (_ collect pf)
       }
     val unchanged = collectResult {
@@ -78,16 +77,14 @@ object Diff {
     val localKeys = for {
       errorOrUnchanged <- diff.unchanged
       errorsOrChanges  <- Future.sequence(diff.uploads)
-    } yield {
+    } yield
       errorsOrChanges.foldLeft(errorOrUnchanged: Either[ErrorReport, Seq[S3Key]]) { (memo, errorOrChanges) =>
         for {
           mem <- memo.right
           keysToDelete <- errorOrChanges.right
-        } yield {
-          mem ++ keysToDelete.map(_.s3Key)
-        }
+        } yield mem ++ keysToDelete.map(_.s3Key)
       }
-    }
+
     s3Files zip localKeys map { (s3Files: Either[ErrorReport, Seq[S3File]], errorOrLocalKeys: Either[ErrorReport, Seq[S3Key]]) =>
       for {
         localS3Keys <- errorOrLocalKeys.right
@@ -115,13 +112,13 @@ object Diff {
 
     def resolveDiffAgainstLocalDb(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
                                  (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
-      val localDiff: Either[ErrorReport, Seq[Either[DbRecord, LocalFileFromDisk]]] =
+      val localDiff: Either[ErrorReport, Seq[Either[DbRecord, LocalFile]]] =
         (for {
           dbFile <- getOrCreateDbFile
           databaseIndices <- loadDbFromFile(dbFile)
         } yield {
           val siteFiles = Files.listSiteFiles
-          val recordsOrChangedFiles = siteFiles.foldLeft(Seq(): Seq[Either[DbRecord, LocalFileFromDisk]]) { (localFiles, file) =>
+          val recordsOrChangedFiles = siteFiles.foldLeft(Seq(): Seq[Either[DbRecord, LocalFile]]) { (localFiles, file) =>
             val truncatedKey = TruncatedDbRecord(file)
             val fileIsUnchanged = databaseIndices.truncatedIndex contains truncatedKey
             if (fileIsUnchanged)
@@ -130,7 +127,7 @@ object Diff {
               val uploadType =
                 if (databaseIndices.s3KeyIndex contains truncatedKey.s3Key) FileUpdate
                 else NewFile
-              localFiles :+ Right(LocalFileFromDisk(file, uploadType))
+              localFiles :+ Right(LocalFile(file, uploadType))
             }
           }
           logger.debug(s"Discovered ${siteFiles.length} files on the local site, of which ${recordsOrChangedFiles count (_.isRight)} are new or changed")
@@ -145,11 +142,11 @@ object Diff {
           case Left(f) => f
         }
 
-        val uploadsResolvedByLocalDiff = localDiffResult collect {
+        val uploadsAccordingToLocalDiff = localDiffResult collect {
           case Right(f) => f
         }
 
-        val changesMissedByLocalDiff: Future[Either[ErrorReport, Seq[LocalFileFromDisk]]] = s3FilesFuture.map { errorOrS3Files =>
+        val changesMissedByLocalDiff: Future[Either[ErrorReport, Seq[LocalFile]]] = s3FilesFuture.map { errorOrS3Files =>
           for (s3Files <- errorOrS3Files.right) yield {
             val remoteS3Keys = s3Files.map(_.s3Key).toSet
             val localS3Keys = unchangedAccordingToLocalDiff.map(_.s3Key).toSet
@@ -157,11 +154,11 @@ object Diff {
             def isChangedOnS3(s3File: S3File) = (localS3Keys contains s3File.s3Key) && !(localMd5 contains s3File.md5)
             val changedOnS3 = s3Files collect {
               case s3File if isChangedOnS3(s3File) =>
-                LocalFileFromDisk(site resolveFile s3File, FileUpdate)
+                LocalFile(site resolveFile s3File, FileUpdate)
             }
             val missingFromS3 = localS3Keys collect {
               case localS3Key if !(remoteS3Keys contains localS3Key) =>
-                LocalFileFromDisk(site resolveFile localS3Key, NewFile)
+                LocalFile(site resolveFile localS3Key, NewFile)
 
             }
             changedOnS3 ++ missingFromS3
@@ -180,14 +177,14 @@ object Diff {
         val unchangedFilesFinal = errorOrDiffAgainstS3 map {
           _ fold (
             (error: ErrorReport) => Left(error),
-            (syncResult: (Seq[DbRecord], Seq[LocalFileFromDisk])) => Right(syncResult._1)
+            (syncResult: (Seq[DbRecord], Seq[LocalFile])) => Right(syncResult._1)
           )
         }
 
         val changedAccordingToS3Diff = errorOrDiffAgainstS3.map {
           _ fold (
             (error: ErrorReport) => Left(error),
-            (syncResult: (Seq[DbRecord], Seq[LocalFileFromDisk])) => Right(syncResult._2)
+            (syncResult: (Seq[DbRecord], Seq[LocalFile])) => Right(syncResult._2)
           )
         }
         val persistenceError: Future[Either[ErrorReport, _]] = for {
@@ -198,13 +195,13 @@ object Diff {
             records1 <- unchanged.right
             records2 <- changedAccordingToS3.right
           } yield
-            persist(records1.map(Left(_)) ++ records2.map(Right(_)) ++ uploadsResolvedByLocalDiff.map(Right(_))) match {
+            persist(records1.map(Left(_)) ++ records2.map(Right(_)) ++ uploadsAccordingToLocalDiff.map(Right(_))) match {
               case Success(_)   => Unit
               case Failure(err) => ErrorReport(err)
             }
         Diff(
           unchangedFilesFinal map (_.right.map(_ map (_.s3Key))),
-          uploads = Future(Right(uploadsResolvedByLocalDiff)) :: changedAccordingToS3Diff :: Nil,
+          uploads = Future(Right(uploadsAccordingToLocalDiff)) :: changedAccordingToS3Diff :: Nil,
           persistenceError = persistenceError map (_.left.toOption)
         )
       }
@@ -244,13 +241,13 @@ object Diff {
         )
       }
 
-    def persist(recordsOrChangedFiles: Seq[Either[DbRecord, LocalFileFromDisk]])(implicit site: Site, logger: Logger): Try[Seq[Either[DbRecord, LocalFileFromDisk]]] =
+    def persist(recordsOrChangedFiles: Seq[Either[DbRecord, LocalFile]])(implicit site: Site, logger: Logger): Try[Seq[Either[DbRecord, LocalFile]]] =
       getOrCreateDbFile flatMap { dbFile =>
         Try {
           val dbFileContents = recordsOrChangedFiles.map { recordOrChangedFile =>
             val record: DbRecord = recordOrChangedFile fold(
               record => record,
-              changedFile => DbRecord(changedFile.s3Key, changedFile.originalFile.length, changedFile.originalFile.lastModified, changedFile.md5)
+              changedFile => DbRecord(changedFile.s3Key, changedFile.originalFile.length, changedFile.originalFile.lastModified, changedFile.md5.get)
               )
             record.s3Key :: record.fileLength :: record.fileModified :: record.uploadFileMd5 :: Nil mkString "|"
           } mkString "\n"
@@ -278,6 +275,6 @@ object Diff {
   
   object DbRecord {
     def apply(original: File)(implicit site: Site): DbRecord =
-      DbRecord(site resolveS3Key original, original.length, original.lastModified, LocalFileFromDisk.md5(original))
+      DbRecord(site resolveS3Key original, original.length, original.lastModified, LocalFile.md5(original).get)
   }
 }

data/src/main/scala/s3/website/Logger.scala

@@ -1,6 +1,8 @@
 package s3.website
 
-class Logger(val verboseOutput: Boolean, logMessage: (String) => Unit = println) {
+import scala.util.Try
+
+class Logger(val verboseOutput: Boolean) {
   def debug(msg: String) = if (verboseOutput) log(Debug, msg)
   def info(msg: String) = log(Info, msg)
   def fail(msg: String) = log(Failure, msg)
@@ -10,9 +12,9 @@ class Logger(val verboseOutput: Boolean, logMessage: (String) => Unit = println)
 
   def pending(msg: String) = log(Wait, msg)
 
-  private def log(logType: LogType, msgRaw: String) {
+  private def log(logType: LogType, msgRaw: String): Try[Unit] = {
     val msg = msgRaw.replaceAll("\\n", "\n       ") // Indent new lines, so that they arrange nicely with other log lines
-    logMessage(s"[$logType] $msg")
+    Try(println(s"[$logType] $msg"))
   }
 
   sealed trait LogType {

data/src/main/scala/s3/website/Push.scala

@@ -191,7 +191,7 @@ object Push {
           (failureReport: PushFailureReport) => counts.copy(failures = counts.failures + 1),
           (successReport: PushSuccessReport) =>
             successReport match {
-              case succ: SuccessfulUpload => succ.source.fold(_.uploadType, _.uploadType) match {
+              case succ: SuccessfulUpload => succ.details.fold(_.uploadType, _.uploadType) match {
                 case NewFile      => counts.copy(newFiles = counts.newFiles + 1).addTransferStats(succ) // TODO nasty repetition here
                 case FileUpdate   => counts.copy(updates = counts.updates + 1).addTransferStats(succ)
                 case RedirectFile => counts.copy(redirects = counts.redirects + 1).addTransferStats(succ)
@@ -228,14 +228,14 @@ object Push {
                          redirects: Int = 0, 
                          deletes: Int = 0,
                          uploadedBytes: Long = 0,
-                         uploadDurations: Seq[org.joda.time.Duration] = Nil
+                         uploadDurations: Seq[UploadDuration] = Nil
                          ) {
     val thereWasSomethingToPush = updates + newFiles + redirects + deletes > 0
 
     def addTransferStats(successfulUpload: SuccessfulUpload): PushCounts =
       copy(
         uploadedBytes = uploadedBytes + (successfulUpload.uploadSize getOrElse 0L),
-        uploadDurations = uploadDurations ++ successfulUpload.uploadDuration
+        uploadDurations = uploadDurations ++ successfulUpload.details.fold(_.uploadDuration, _ => None)
       )
   }
 

data/src/main/scala/s3/website/S3.scala

@@ -2,26 +2,17 @@ package s3.website
 
 import s3.website.model._
 import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}
-import com.amazonaws.auth.BasicAWSCredentials
 import com.amazonaws.services.s3.model._
 import scala.collection.JavaConversions._
 import scala.concurrent.{ExecutionContextExecutor, Future}
 import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
-import s3.website.S3.SuccessfulUpload
-import s3.website.S3.SuccessfulDelete
-import s3.website.S3.FailedUpload
-import scala.Some
-import s3.website.S3.FailedDelete
-import s3.website.S3.S3Setting
 import s3.website.ByteHelper.humanReadableByteCount
-import org.joda.time.{Seconds, Duration, Interval}
-import scala.concurrent.duration.TimeUnit
-import java.util.concurrent.TimeUnit
 import scala.concurrent.duration.TimeUnit
 import java.util.concurrent.TimeUnit.SECONDS
 import s3.website.S3.SuccessfulUpload.humanizeUploadSpeed
 import java.io.FileInputStream
 import s3.website.model.Config.awsCredentials
+import scala.util.Try
 
 object S3 {
 
@@ -29,19 +20,26 @@ object S3 {
             (implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger) =
     upload(Right(redirect))
 
-  def uploadFile(localFile: LocalFileFromDisk, a: Attempt = 1)
+  def uploadFile(localFile: LocalFile, a: Attempt = 1)
                     (implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger) =
     upload(Left(localFile))
 
-  def upload(source: Either[LocalFileFromDisk, Redirect], a: Attempt = 1)
+  def upload(source: Either[LocalFile, Redirect], a: Attempt = 1)
             (implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger):
   Future[Either[FailedUpload, SuccessfulUpload]] =
     Future {
-      val putObjectRequest = toPutObjectRequest(source)
+      val putObjectRequest = toPutObjectRequest(source).get
       val uploadDuration =
         if (pushMode.dryRun) None
-        else recordUploadDuration(putObjectRequest, s3Settings.s3Client(config) putObject putObjectRequest)
-      val report = SuccessfulUpload(source, putObjectRequest, uploadDuration)
+        else Some(recordUploadDuration(putObjectRequest, s3Settings.s3Client(config) putObject putObjectRequest))
+      val report = SuccessfulUpload(
+        source.fold(_.s3Key, _.s3Key),
+        source.fold(
+          localFile => Left(SuccessfulNewOrCreatedDetails(localFile.uploadType, localFile.uploadFile.get.length(), uploadDuration)),
+          redirect  => Right(SuccessfulRedirectDetails(redirect.uploadType, redirect.redirectTarget))
+        ),
+        putObjectRequest
+      )
       logger.info(report)
       Right(report)
     } recoverWith retry(a)(
@@ -62,25 +60,30 @@ object S3 {
       retryAction  = newAttempt => this.delete(s3Key, newAttempt)
     )
 
-  def toPutObjectRequest(source: Either[LocalFileFromDisk, Redirect])(implicit config: Config) =
+  def toPutObjectRequest(source: Either[LocalFile, Redirect])(implicit config: Config): Try[PutObjectRequest] =
     source.fold(
-      localFile => {
-        val md = new ObjectMetadata()
-        md setContentLength localFile.uploadFile.length
-        md setContentType localFile.contentType
-        localFile.encodingOnS3.map(_ => "gzip") foreach md.setContentEncoding
-        localFile.maxAge foreach { seconds =>
-          md.setCacheControl(
-            if (seconds == 0)
-              s"no-cache; max-age=$seconds"
-            else
-              s"max-age=$seconds"
-          )
+      localFile =>
+        for {
+          uploadFile <- localFile.uploadFile
+          contentType <- localFile.contentType
+        } yield {
+          val md = new ObjectMetadata()
+          md setContentLength uploadFile.length
+          md setContentType contentType
+          localFile.encodingOnS3.map(_ => "gzip") foreach md.setContentEncoding
+          localFile.maxAge foreach { seconds =>
+            md.setCacheControl(
+              if (seconds == 0)
+                s"no-cache; max-age=$seconds"
+              else
+                s"max-age=$seconds"
+            )
+          }
+          val req = new PutObjectRequest(config.s3_bucket, localFile.s3Key, new FileInputStream(uploadFile), md)
+          config.s3_reduced_redundancy.filter(_ == true) foreach (_ => req setStorageClass ReducedRedundancy)
+          req
         }
-        val req = new PutObjectRequest(config.s3_bucket, localFile.s3Key, new FileInputStream(localFile.uploadFile), md)
-        config.s3_reduced_redundancy.filter(_ == true) foreach (_ => req setStorageClass ReducedRedundancy)
-        req
-      },
+      ,
       redirect => {
         val req = new PutObjectRequest(config.s3_bucket, redirect.s3Key, redirect.redirectTarget)
         req.setMetadata({
@@ -93,17 +96,14 @@ object S3 {
           md.setCacheControl("max-age=0, no-cache")
           md
         })
-        req
+        Try(req)
       }
     )
 
-  def recordUploadDuration(putObjectRequest: PutObjectRequest, f: => Unit): Option[Duration] = {
+  def recordUploadDuration(putObjectRequest: PutObjectRequest, f: => Unit): UploadDuration = {
     val start = System.currentTimeMillis()
     f
-    if (putObjectRequest.getMetadata.getContentLength > 0)
-      Some(new Duration(start, System.currentTimeMillis))
-    else
-      None // We are not interested in tracking durations of PUT requests that don't contain data. Redirect is an example of such request.
+    System.currentTimeMillis - start
   }
 
   def awsS3Client(config: Config) = new AmazonS3Client(awsCredentials(config))
@@ -145,14 +145,18 @@ object S3 {
     def s3Key: String
   }
 
-  case class SuccessfulUpload(source: Either[LocalFileFromDisk, Redirect], putObjectRequest: PutObjectRequest, uploadDuration: Option[Duration])
+  case class SuccessfulRedirectDetails(uploadType: UploadType, redirectTarget: String)
+  case class SuccessfulNewOrCreatedDetails(uploadType: UploadType, uploadSize: Long, uploadDuration: Option[Long])
+
+  case class SuccessfulUpload(s3Key: S3Key,
+                              details: Either[SuccessfulNewOrCreatedDetails, SuccessfulRedirectDetails],
+                              putObjectRequest: PutObjectRequest)
                              (implicit pushMode: PushMode, logger: Logger) extends PushSuccessReport {
     def reportMessage =
-      source.fold(_.uploadType, (redirect: Redirect) => redirect) match {
-        case NewFile                         => s"${Created.renderVerb} $s3Key ($reportDetails)"
-        case FileUpdate                      => s"${Updated.renderVerb} $s3Key ($reportDetails)"
-        case Redirect(s3Key, redirectTarget) => s"${Redirected.renderVerb} $s3Key to $redirectTarget"
-      }
+      details.fold(
+        newOrCreatedDetails => s"${newOrCreatedDetails.uploadType.pushAction} $s3Key ($reportDetails)",
+        redirectDetails     => s"${redirectDetails.uploadType.pushAction} $s3Key to ${redirectDetails.redirectTarget}"
+      )
 
     def reportDetails = {
       val md = putObjectRequest.getMetadata
@@ -169,12 +173,9 @@ object S3 {
       }.mkString(" | ")
     }
 
-    def s3Key = source.fold(_.s3Key, _.s3Key)
-
-    lazy val uploadSize: Option[Long] =
-      source.fold(
-        (localFile: LocalFileFromDisk) => Some(localFile.uploadFile.length()),
-        (redirect: Redirect)           => None
+    lazy val uploadSize = details.fold(
+      newOrCreatedDetails => Some(newOrCreatedDetails.uploadSize),
+      redirectDetails     => None
     )
 
     lazy val uploadSizeForHumans: Option[String] = uploadSize filter (_ => logger.verboseOutput) map humanReadableByteCount
@@ -182,17 +183,17 @@ object S3 {
     lazy val uploadSpeedForHumans: Option[String] =
       (for {
         dataSize <- uploadSize
-        duration <- uploadDuration
+        duration <- details.left.map(_.uploadDuration).left.toOption.flatten
       } yield {
         humanizeUploadSpeed(dataSize, duration)
       }) flatMap identity filter (_ => logger.verboseOutput)
   }
   
   object SuccessfulUpload {
-    def humanizeUploadSpeed(uploadedBytes: Long, uploadDurations: Duration*): Option[String] = {
-      val totalDurationMillis = uploadDurations.foldLeft(new org.joda.time.Duration(0)){ (memo, duration) =>
-        memo.plus(duration)
-      }.getMillis // retain precision by using milliseconds
+    def humanizeUploadSpeed(uploadedBytes: Long, uploadDurations: UploadDuration*): Option[String] = {
+      val totalDurationMillis = uploadDurations.foldLeft(0L){ (memo, duration) =>
+        memo + duration
+      }
       if (totalDurationMillis > 0) {
         val bytesPerMillisecond = uploadedBytes / totalDurationMillis
         val bytesPerSecond = bytesPerMillisecond * 1000 * uploadDurations.length

data/src/main/scala/s3/website/model/Site.scala

@@ -48,7 +48,7 @@ object Site {
           redirects <- loadRedirects.right
         } yield {
           gzip_zopfli.foreach(_ => logger.info(
-            """|zopfli is not currently supported
+            """|Zopfli is not currently supported. Falling back to regular gzip.
                |If you find a stable Java implementation for zopfli, please send an email to lauri.lehmijoki@iki.fi about it."""
             .stripMargin))
           extensionless_mime_type.foreach(_ => logger.info(

data/src/main/scala/s3/website/model/push.scala

@@ -7,10 +7,11 @@ import java.util.zip.GZIPOutputStream
 import org.apache.tika.Tika
 import s3.website.Ruby._
 import s3.website._
-import s3.website.model.LocalFileFromDisk.tika
+import s3.website.model.LocalFile.tika
 import s3.website.model.Encoding.encodingOnS3
 import java.io.File.createTempFile
 import org.apache.commons.io.IOUtils.copy
+import scala.util.Try
 
 object Encoding {
 
@@ -34,27 +35,34 @@ object Encoding {
     }
 }
 
-sealed trait UploadType // Sealed, so that we can avoid inexhaustive pattern matches more easily
+sealed trait UploadType {
+  val pushAction: PushAction
+}
+
+case object NewFile extends UploadType {
+  val pushAction = Created
+}
+case object FileUpdate extends UploadType {
+  val pushAction = Updated
+}
 
-case object NewFile extends UploadType
-case object FileUpdate extends UploadType
-case object RedirectFile extends UploadType
+case object RedirectFile extends UploadType {
+  val pushAction = Redirected
+}
 
-case class LocalFileFromDisk(originalFile: File, uploadType: UploadType)(implicit site: Site) {
+case class LocalFile(originalFile: File, uploadType: UploadType)(implicit site: Site) {
   lazy val s3Key = site.resolveS3Key(originalFile)
 
   lazy val encodingOnS3 = Encoding.encodingOnS3(s3Key)
 
-  lazy val lastModified = originalFile.lastModified
-
   /**
    * This is the file we should upload, because it contains the potentially gzipped contents of the original file.
    *
    * May throw an exception, so remember to call this in a Try or Future monad
    */
-  lazy val uploadFile: File = LocalFileFromDisk uploadFile originalFile
+  lazy val uploadFile: Try[File] = LocalFile uploadFile originalFile
 
-  lazy val contentType = {
+  lazy val contentType: Try[String] = tika map { tika =>
     val mimeType = tika.detect(originalFile)
     if (mimeType.startsWith("text/") || mimeType == "application/json")
       mimeType + "; charset=utf-8"
@@ -84,24 +92,29 @@ case class LocalFileFromDisk(originalFile: File, uploadType: UploadType)(implici
   /**
    * May throw an exception, so remember to call this in a Try or Future monad
    */
-  lazy val md5 = LocalFileFromDisk md5 originalFile
+  lazy val md5 = LocalFile md5 originalFile
 }
 
-object LocalFileFromDisk {
-  lazy val tika = new Tika()
+object LocalFile {
+  lazy val tika = Try(new Tika())
 
-  def md5(originalFile: File)(implicit site: Site) = using(fis { uploadFile(originalFile) }) { DigestUtils.md5Hex }
+  def md5(originalFile: File)(implicit site: Site): Try[MD5] =
+    uploadFile(originalFile) map { file =>
+      using(fis { file }) { DigestUtils.md5Hex }
+    }
 
-  def uploadFile(originalFile: File)(implicit site: Site): File =
+  def uploadFile(originalFile: File)(implicit site: Site): Try[File] =
     encodingOnS3(site resolveS3Key originalFile)
-      .fold(originalFile)(algorithm => {
-        val tempFile = createTempFile(originalFile.getName, "gzip")
-        tempFile.deleteOnExit()
-        using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
-          copy(fis(originalFile), stream)
+      .fold(Try(originalFile))(algorithm =>
+        Try {
+          val tempFile = createTempFile(originalFile.getName, "gzip")
+          tempFile.deleteOnExit()
+          using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
+            copy(fis(originalFile), stream)
+          }
+          tempFile
         }
-        tempFile
-      })
+      )
 
   private[this] def fis(file: File): InputStream = new FileInputStream(file)
   private[this] def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()

data/src/main/scala/s3/website/package.scala

@@ -44,6 +44,8 @@ package object website {
 
   type S3Key = String
 
+  type UploadDuration = Long
+
   trait PushAction {
     def actionName = getClass.getSimpleName.replace("$", "") // case object class names contain the '$' char
 

data/src/test/scala/s3/website/S3WebsiteSpec.scala

@@ -21,13 +21,15 @@ import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, Create
 import org.mockito.stubbing.Answer
 import org.mockito.invocation.InvocationOnMock
 import java.util.concurrent.atomic.AtomicInteger
-import org.apache.commons.io.FileUtils.{forceDelete, forceMkdir, write}
+import org.apache.commons.io.FileUtils._
 import scala.collection.mutable
 import s3.website.Push.{push, CliArgs}
 import s3.website.CloudFront.CloudFrontSetting
 import s3.website.S3.S3Setting
 import org.apache.commons.codec.digest.DigestUtils
 import java.util.Date
+import s3.website.CloudFront.CloudFrontSetting
+import s3.website.S3.S3Setting
 
 class S3WebsiteSpec extends Specification {
 
@@ -799,12 +801,15 @@ class S3WebsiteSpec extends Specification {
   trait EmptySite extends Directories {
     type LocalFileWithContent = (String, String)
 
-    val localFilesWithContent: mutable.Set[LocalFileWithContent] = mutable.Set()
     def setLocalFile(fileName: String) = setLocalFileWithContent((fileName, ""))
     def setLocalFiles(fileNames: String*) = fileNames foreach setLocalFile
-    def setLocalFileWithContent(fileNameAndContent: LocalFileWithContent) =
-      localFilesWithContent += fileNameAndContent
     def setLocalFilesWithContent(fileNamesAndContent: LocalFileWithContent*) = fileNamesAndContent foreach setLocalFileWithContent
+    def setLocalFileWithContent(fileNameAndContent: LocalFileWithContent) = {
+      val file = new File(siteDirectory, fileNameAndContent._1)
+      forceMkdir(file.getParentFile)
+      file.createNewFile()
+      write(file, fileNameAndContent._2)
+    }
     var config = ""
     val baseConfig =
     """
@@ -813,21 +818,9 @@ class S3WebsiteSpec extends Specification {
       |s3_bucket: bucket
     """.stripMargin
 
-    implicit def cliArgs: CliArgs = siteWithFilesAndContent(config, localFilesWithContent)
+    implicit def cliArgs: CliArgs = buildCliArgs(config)
     def pushMode: PushMode // Represents the --dry-run switch
 
-    private def siteWithFilesAndContent(config: String = "", localFilesWithContent: mutable.Set[LocalFileWithContent]): CliArgs = {
-      localFilesWithContent.foreach {
-        case (filePath, content) =>
-          val file = new File(siteDirectory, filePath)
-          forceMkdir(file.getParentFile)
-          file.createNewFile()
-          write(file, content)
-      }
-      localFilesWithContent.clear() // Once we've persisted the changes on the disk, clear the queue. I.e., keep the state where it should be – on the disk.
-      buildCliArgs(config)
-    }
-
     private def buildCliArgs(
                     config: String = "",
                     baseConfig: String =

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_website_monadic
 version: !ruby/object:Gem::Version
-  version: 0.0.37
+  version: 0.0.38
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-29 00:00:00.000000000 Z
+date: 2014-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor