s3_website 2.0.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 24289cabdc9ada72b1c10f123788231a7b5f6879
-  data.tar.gz: 1e457defca86674d0696d2fd6e69cf811d8d323f
+  metadata.gz: e9175154b8b9b1982291616351c8540b0dee659c
+  data.tar.gz: cf87ecd7a6f989c21c98699d9ce6b43fdf025e6a
 SHA512:
-  metadata.gz: 43fb218cd322b2fbf5eb47ffd161eeeba2c712b8eb86776842e7a47cc19e6ba5c5635c0d5afdb9c59068a7b3b00bc38cea37d0faf879fa8715dfdbc8415a2733
-  data.tar.gz: 69d47fe090b0a591a8bbb9f37a9ea50ecf8bdf02ab150f4dbce672f513ea6faa1cc8f507abfc2de4987e0333992bc6e336173bc1d27273f572c7d4e2a2e868e3
+  metadata.gz: 5fce5e66cd6fe88902ec1d01441387e8d4bf725420face8c6e2ac8aaeaa226613b19fae7d476b3afa95807e611cb45492e565d385742599f4399cea8e9c8497f
+  data.tar.gz: 105be5233f7d9dc11f704f07e3e0878059751e3e1af3162fe86e51e5fa34a26d5b94437264ff55ddbc9147a0713ae11e990ae6e7f637a8965f222e6e16f90b70

data/README.md

@@ -25,7 +25,7 @@ and [Java](http://java.com) to run.
 Here's how you can get started:
 
 * Create API credentials that have sufficient permissions to S3. More info
-  [here](https://github.com/laurilehmijoki/s3_website/blob/master/additional-docs/setting-up-aws-credentials.md).
+  [here](additional-docs/setting-up-aws-credentials.md).
 * Go to your website directory
 * Run `s3_website cfg create`. This generates a configuration file called `s3_website.yml`.
 * Put your AWS credentials and the S3 bucket name into the file
@@ -85,7 +85,6 @@ syntax information.
 * Let the power users benefit from advanced S3 website features such as
   redirects, Cache-Control headers and gzip support
 * Be as fast as possible. Do in parallel all that can be done in parallel.
-* Maintain 90% backward compatibility with the jekyll-s3 gem
 
 `s3_website` attempts to be a command-line interface tool that is easy to
 understand and use. For example, `s3_website --help` should print you all the
@@ -343,10 +342,13 @@ operation would cause to your live website.
 Please read the [release note](/changelog.md#200) on version 2. It contains
 information on backward incompatible changes.
 
+You can find the v1 branch
+[here](https://github.com/laurilehmijoki/s3_website/tree/1.x). It's in
+maintenance mode. This means that v1 will see only critical bugfix releases.
+
 ## Example configurations
 
-See
-<https://github.com/laurilehmijoki/s3_website/blob/master/additional-docs/example-configurations.md>.
+See [example-configurations](additional-docs/example-configurations.md).
 
 ## On security
 

data/changelog.md

@@ -2,6 +2,14 @@
 
 This project uses [Semantic Versioning](http://semver.org).
 
+## 2.1.0
+
+* Show the upload reason when calling `push --verbose`
+
+## 2.0.1
+
+* Rename binary s3_website_monadic to s3_website
+
 ## 2.0.0
 
 ### New features

data/lib/s3_website/version.rb

@@ -1,3 +1,3 @@
 module S3Website
-  VERSION = '2.0.1'
+  VERSION = '2.1.0'
 end

data/src/main/scala/s3/website/Diff.scala

@@ -19,7 +19,7 @@ case class Diff(
 
 object Diff {
 
-  type UploadBatch = Future[Either[ErrorReport, Seq[LocalFile]]]
+  type UploadBatch = Future[Either[ErrorReport, Seq[Upload]]]
 
   def resolveDiff(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
                  (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] =
@@ -35,31 +35,31 @@ object Diff {
           val s3Md5Index = s3Files.map(_.md5).toSet
           val siteFiles = Files.listSiteFiles
           val existsOnS3 = (f: File) => s3KeyIndex contains site.resolveS3Key(f)
-          val isChangedOnS3 = (localFile: LocalFile) => !(s3Md5Index contains localFile.md5.get)
-          val newFiles = siteFiles collect {
-            case file if !existsOnS3(file) => LocalFile(file, NewFile)
+          val isChangedOnS3 = (upload: Upload) => !(s3Md5Index contains upload.md5.get)
+          val newUploads = siteFiles collect {
+            case file if !existsOnS3(file) => Upload(file, NewFile, reasonForUpload = "the file is missing from S3")
           }
-          val changedFiles = siteFiles collect {
-            case file if existsOnS3(file) => LocalFile(file, FileUpdate)
+          val changedUploads = siteFiles collect {
+            case file if existsOnS3(file) => Upload(file, FileUpdate, reasonForUpload = "the S3 bucket has different contents for this file")
           } filter isChangedOnS3
           val unchangedFiles = {
-            val newOrChangedFiles = (changedFiles ++ newFiles).map(_.originalFile).toSet
+            val newOrChangedFiles = (changedUploads ++ newUploads).map(_.originalFile).toSet
             siteFiles.filterNot(f => newOrChangedFiles contains f)
           }
-          val allFiles: Seq[Either[DbRecord, LocalFile]] = unchangedFiles.map {
+          val recordsAndUploads: Seq[Either[DbRecord, Upload]] = unchangedFiles.map {
             f => Left(DbRecord(f))
-          } ++ (changedFiles ++ newFiles).map {
+          } ++ (changedUploads ++ newUploads).map {
             Right(_)
           }
-          LocalFileDatabase persist allFiles
-          allFiles
+          LocalFileDatabase persist recordsAndUploads
+          recordsAndUploads
         } match {
           case Success(ok) => Right(ok)
           case Failure(err) => Left(ErrorReport(err))
         }
       }
     }
-    def collectResult[B](pf: PartialFunction[Either[DbRecord, LocalFile],B]) =
+    def collectResult[B](pf: PartialFunction[Either[DbRecord, Upload],B]) =
       diffAgainstS3.map { errorOrDiffSource =>
         errorOrDiffSource.right map (_ collect pf)
       }
@@ -67,7 +67,7 @@ object Diff {
       case Left(dbRecord) => dbRecord.s3Key
     }
     val uploads: UploadBatch = collectResult {
-      case Right(localFile) => localFile
+      case Right(upload) => upload
     }
     Right(Diff(unchanged, uploads :: Nil, persistenceError = Future(None)))
   }
@@ -112,26 +112,28 @@ object Diff {
 
     def resolveDiffAgainstLocalDb(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
                                  (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
-      val localDiff: Either[ErrorReport, Seq[Either[DbRecord, LocalFile]]] =
+      val localDiff: Either[ErrorReport, Seq[Either[DbRecord, Upload]]] =
         (for {
           dbFile <- getOrCreateDbFile
           databaseIndices <- loadDbFromFile(dbFile)
         } yield {
           val siteFiles = Files.listSiteFiles
-          val recordsOrChangedFiles = siteFiles.foldLeft(Seq(): Seq[Either[DbRecord, LocalFile]]) { (localFiles, file) =>
+          val recordsOrUploads = siteFiles.foldLeft(Seq(): Seq[Either[DbRecord, Upload]]) { (recordsOrUps, file) =>
             val truncatedKey = TruncatedDbRecord(file)
             val fileIsUnchanged = databaseIndices.truncatedIndex contains truncatedKey
             if (fileIsUnchanged)
-              localFiles :+ Left(databaseIndices.fullIndex find (_.truncated == truncatedKey) get)
+              recordsOrUps :+ Left(databaseIndices.fullIndex find (_.truncated == truncatedKey) get)
             else {
+              val isUpdate = databaseIndices.s3KeyIndex contains truncatedKey.s3Key
+
               val uploadType =
-                if (databaseIndices.s3KeyIndex contains truncatedKey.s3Key) FileUpdate
+                if (isUpdate) FileUpdate
                 else NewFile
-              localFiles :+ Right(LocalFile(file, uploadType))
+              recordsOrUps :+ Right(Upload(file, uploadType, reasonForUpload(truncatedKey, databaseIndices, isUpdate)))
             }
           }
-          logger.debug(s"Discovered ${siteFiles.length} files on the local site, of which ${recordsOrChangedFiles count (_.isRight)} are new or changed")
-          recordsOrChangedFiles
+          logger.debug(s"Discovered ${siteFiles.length} files on the local site, of which ${recordsOrUploads count (_.isRight)} are new or changed")
+          recordsOrUploads
         }) match {
           case Success(ok) => Right(ok)
           case Failure(err) => Left(ErrorReport(err))
@@ -146,7 +148,7 @@ object Diff {
           case Right(f) => f
         }
 
-        val changesMissedByLocalDiff: Future[Either[ErrorReport, Seq[LocalFile]]] = s3FilesFuture.map { errorOrS3Files =>
+        val changesMissedByLocalDiff: Future[Either[ErrorReport, Seq[Upload]]] = s3FilesFuture.map { errorOrS3Files =>
           for (s3Files <- errorOrS3Files.right) yield {
             val remoteS3Keys = s3Files.map(_.s3Key).toSet
             val localS3Keys = unchangedAccordingToLocalDiff.map(_.s3Key).toSet
@@ -154,11 +156,11 @@ object Diff {
             def isChangedOnS3(s3File: S3File) = (localS3Keys contains s3File.s3Key) && !(localMd5 contains s3File.md5)
             val changedOnS3 = s3Files collect {
               case s3File if isChangedOnS3(s3File) =>
-                LocalFile(site resolveFile s3File, FileUpdate)
+                Upload(site resolveFile s3File, FileUpdate, reasonForUpload = "someone else has modified the file on the S3 bucket")
             }
             val missingFromS3 = localS3Keys collect {
               case localS3Key if !(remoteS3Keys contains localS3Key) =>
-                LocalFile(site resolveFile localS3Key, NewFile)
+                Upload(site resolveFile localS3Key, NewFile, reasonForUpload = "someone else has removed the file from the S3 bucket")
 
             }
             changedOnS3 ++ missingFromS3
@@ -177,23 +179,23 @@ object Diff {
         val unchangedFilesFinal = errorOrDiffAgainstS3 map {
           _ fold (
             (error: ErrorReport) => Left(error),
-            (syncResult: (Seq[DbRecord], Seq[LocalFile])) => Right(syncResult._1)
+            (syncResult: (Seq[DbRecord], Seq[Upload])) => Right(syncResult._1)
           )
         }
 
-        val changedAccordingToS3Diff = errorOrDiffAgainstS3.map {
+        val uploadsAccordingToS3Diff = errorOrDiffAgainstS3.map {
           _ fold (
             (error: ErrorReport) => Left(error),
-            (syncResult: (Seq[DbRecord], Seq[LocalFile])) => Right(syncResult._2)
+            (syncResult: (Seq[DbRecord], Seq[Upload])) => Right(syncResult._2)
           )
         }
         val persistenceError: Future[Either[ErrorReport, _]] = for {
           unchanged <- unchangedFilesFinal
-          changedAccordingToS3 <- changedAccordingToS3Diff
+          uploads <- uploadsAccordingToS3Diff
         } yield
           for {
             records1 <- unchanged.right
-            records2 <- changedAccordingToS3.right
+            records2 <- uploads.right
           } yield
             persist(records1.map(Left(_)) ++ records2.map(Right(_)) ++ uploadsAccordingToLocalDiff.map(Right(_))) match {
               case Success(_)   => Unit
@@ -201,12 +203,28 @@ object Diff {
             }
         Diff(
           unchangedFilesFinal map (_.right.map(_ map (_.s3Key))),
-          uploads = Future(Right(uploadsAccordingToLocalDiff)) :: changedAccordingToS3Diff :: Nil,
+          uploads = Future(Right(uploadsAccordingToLocalDiff)) :: uploadsAccordingToS3Diff :: Nil,
           persistenceError = persistenceError map (_.left.toOption)
         )
       }
     }
 
+    private def reasonForUpload(truncatedKey: TruncatedDbRecord, databaseIndices: DbIndices, isUpdate: Boolean) = {
+      if (isUpdate) {
+        val lengthChanged = !(databaseIndices.fileLenghtIndex contains truncatedKey.fileLength)
+        val mtimeChanged = !(databaseIndices.lastModifiedIndex contains truncatedKey.fileModified)
+        if (lengthChanged)
+          "file length has changed according to the local database"
+        else if (mtimeChanged)
+          "file mtime has changed according to the local database"
+        else if (mtimeChanged && lengthChanged)
+          "file mtime and length have changed according to the local database"
+        else
+          "programmer error: faulty logic in inferring the reason for upload"
+      }
+      else "file is new according to the local database"
+    }
+
     private def getOrCreateDbFile(implicit site: Site, logger: Logger) =
       Try {
         val dbFile = new File(getTempDirectory, "s3_website_local_db_" + sha256Hex(site.rootDirectory))
@@ -216,9 +234,11 @@ object Diff {
       }
     
     case class DbIndices(
-      s3KeyIndex:     Set[S3Key],
-      truncatedIndex: Set[TruncatedDbRecord],
-      fullIndex:      Set[DbRecord]
+      s3KeyIndex:        Set[S3Key],
+      fileLenghtIndex:   Set[Long],
+      lastModifiedIndex: Set[Long],
+      truncatedIndex:    Set[TruncatedDbRecord],
+      fullIndex:         Set[DbRecord]
     ) 
 
     private def loadDbFromFile(databaseFile: File)(implicit site: Site, logger: Logger): Try[DbIndices] =
@@ -237,23 +257,25 @@ object Diff {
         DbIndices(
           s3KeyIndex = fullIndex map (_.s3Key), 
           truncatedIndex = fullIndex map (TruncatedDbRecord(_)),
-          fullIndex
+          fileLenghtIndex = fullIndex map (_.fileLength),
+          lastModifiedIndex = fullIndex map (_.fileModified),
+          fullIndex = fullIndex
         )
       }
 
-    def persist(recordsOrChangedFiles: Seq[Either[DbRecord, LocalFile]])(implicit site: Site, logger: Logger): Try[Seq[Either[DbRecord, LocalFile]]] =
+    def persist(recordsOrUploads: Seq[Either[DbRecord, Upload]])(implicit site: Site, logger: Logger): Try[Seq[Either[DbRecord, Upload]]] =
       getOrCreateDbFile flatMap { dbFile =>
         Try {
-          val dbFileContents = recordsOrChangedFiles.map { recordOrChangedFile =>
-            val record: DbRecord = recordOrChangedFile fold(
+          val dbFileContents = recordsOrUploads.map { recordOrUpload =>
+            val record: DbRecord = recordOrUpload fold(
               record => record,
-              changedFile => DbRecord(changedFile.s3Key, changedFile.originalFile.length, changedFile.originalFile.lastModified, changedFile.md5.get)
+              upload => DbRecord(upload.s3Key, upload.originalFile.length, upload.originalFile.lastModified, upload.md5.get)
               )
             record.s3Key :: record.fileLength :: record.fileModified :: record.uploadFileMd5 :: Nil mkString "|"
           } mkString "\n"
 
           write(dbFile, dbFileContents)
-          recordsOrChangedFiles
+          recordsOrUploads
         }
       }
   }
@@ -275,6 +297,6 @@ object Diff {
   
   object DbRecord {
     def apply(original: File)(implicit site: Site): DbRecord =
-      DbRecord(site resolveS3Key original, original.length, original.lastModified, LocalFile.md5(original).get)
+      DbRecord(site resolveS3Key original, original.length, original.lastModified, Upload.md5(original).get)
   }
 }

data/src/main/scala/s3/website/S3.scala

@@ -18,13 +18,13 @@ object S3 {
 
   def uploadRedirect(redirect: Redirect, a: Attempt = 1)
             (implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger) =
-    upload(Right(redirect))
+    uploadToS3(Right(redirect))
 
-  def uploadFile(localFile: LocalFile, a: Attempt = 1)
+  def uploadFile(up: Upload, a: Attempt = 1)
                     (implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger) =
-    upload(Left(localFile))
+    uploadToS3(Left(up))
 
-  def upload(source: Either[LocalFile, Redirect], a: Attempt = 1)
+  def uploadToS3(source: Either[Upload, Redirect], a: Attempt = 1)
             (implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger):
   Future[Either[FailedUpload, SuccessfulUpload]] =
     Future {
@@ -35,7 +35,7 @@ object S3 {
       val report = SuccessfulUpload(
         source.fold(_.s3Key, _.s3Key),
         source.fold(
-          localFile => Left(SuccessfulNewOrCreatedDetails(localFile.uploadType, localFile.uploadFile.get.length(), uploadDuration)),
+          upload => Left(SuccessfulNewOrCreatedDetails(upload.uploadType, upload.uploadFile.get.length(), uploadDuration, upload.reasonForUpload)),
           redirect  => Right(SuccessfulRedirectDetails(redirect.uploadType, redirect.redirectTarget))
         ),
         putObjectRequest
@@ -44,7 +44,7 @@ object S3 {
       Right(report)
     } recoverWith retry(a)(
       createFailureReport = error => FailedUpload(source.fold(_.s3Key, _.s3Key), error),
-      retryAction  = newAttempt => this.upload(source, newAttempt)
+      retryAction  = newAttempt => this.uploadToS3(source, newAttempt)
     )
 
   def delete(s3Key: S3Key,  a: Attempt = 1)
@@ -60,18 +60,18 @@ object S3 {
       retryAction  = newAttempt => this.delete(s3Key, newAttempt)
     )
 
-  def toPutObjectRequest(source: Either[LocalFile, Redirect])(implicit config: Config): Try[PutObjectRequest] =
+  def toPutObjectRequest(source: Either[Upload, Redirect])(implicit config: Config): Try[PutObjectRequest] =
     source.fold(
-      localFile =>
+      upload =>
         for {
-          uploadFile <- localFile.uploadFile
-          contentType <- localFile.contentType
+          uploadFile <- upload.uploadFile
+          contentType <- upload.contentType
         } yield {
           val md = new ObjectMetadata()
           md setContentLength uploadFile.length
           md setContentType contentType
-          localFile.encodingOnS3.map(_ => "gzip") foreach md.setContentEncoding
-          localFile.maxAge foreach { seconds =>
+          upload.encodingOnS3.map(_ => "gzip") foreach md.setContentEncoding
+          upload.maxAge foreach { seconds =>
             md.setCacheControl(
               if (seconds == 0)
                 s"no-cache; max-age=$seconds"
@@ -79,7 +79,7 @@ object S3 {
                 s"max-age=$seconds"
             )
           }
-          val req = new PutObjectRequest(config.s3_bucket, localFile.s3Key, new FileInputStream(uploadFile), md)
+          val req = new PutObjectRequest(config.s3_bucket, upload.s3Key, new FileInputStream(uploadFile), md)
           config.s3_reduced_redundancy.filter(_ == true) foreach (_ => req setStorageClass ReducedRedundancy)
           req
         }
@@ -146,7 +146,7 @@ object S3 {
   }
 
   case class SuccessfulRedirectDetails(uploadType: UploadType, redirectTarget: String)
-  case class SuccessfulNewOrCreatedDetails(uploadType: UploadType, uploadSize: Long, uploadDuration: Option[Long])
+  case class SuccessfulNewOrCreatedDetails(uploadType: UploadType, uploadSize: Long, uploadDuration: Option[Long], reasonForUpload: String)
 
   case class SuccessfulUpload(s3Key: S3Key,
                               details: Either[SuccessfulNewOrCreatedDetails, SuccessfulRedirectDetails],
@@ -167,12 +167,20 @@ object S3 {
           md.getContentEncoding ::
           putObjectRequest.getStorageClass ::
           Nil map (Option(_)) // AWS SDK may return nulls
-        ) :+ uploadSizeForHumans :+ uploadSpeedForHumans
+        ) :+ uploadSizeForHumans :+ uploadSpeedForHumans :+ uploadReason
       detailFragments.collect {
         case Some(detailFragment) => detailFragment
       }.mkString(" | ")
     }
 
+    lazy val uploadReason =
+      details
+        .fold(uploadDetails => Some(uploadDetails.reasonForUpload), _ => None)
+        .collect {
+          case reasonForUpload if logger.verboseOutput =>
+            s"upload reason: $reasonForUpload"
+        }
+
     lazy val uploadSize = details.fold(
       newOrCreatedDetails => Some(newOrCreatedDetails.uploadSize),
       redirectDetails     => None

data/src/main/scala/s3/website/model/push.scala

@@ -7,7 +7,7 @@ import java.util.zip.GZIPOutputStream
 import org.apache.tika.Tika
 import s3.website.Ruby._
 import s3.website._
-import s3.website.model.LocalFile.tika
+import s3.website.model.Upload.tika
 import s3.website.model.Encoding.encodingOnS3
 import java.io.File.createTempFile
 import org.apache.commons.io.IOUtils.copy
@@ -50,7 +50,7 @@ case object RedirectFile extends UploadType {
   val pushAction = Redirected
 }
 
-case class LocalFile(originalFile: File, uploadType: UploadType)(implicit site: Site) {
+case class Upload(originalFile: File, uploadType: UploadType, reasonForUpload: String)(implicit site: Site) {
   lazy val s3Key = site.resolveS3Key(originalFile)
 
   lazy val encodingOnS3 = Encoding.encodingOnS3(s3Key)
@@ -60,7 +60,7 @@ case class LocalFile(originalFile: File, uploadType: UploadType)(implicit site:
    *
    * May throw an exception, so remember to call this in a Try or Future monad
    */
-  lazy val uploadFile: Try[File] = LocalFile uploadFile originalFile
+  lazy val uploadFile: Try[File] = Upload uploadFile originalFile
 
   lazy val contentType: Try[String] = tika map { tika =>
     val mimeType = tika.detect(originalFile)
@@ -92,10 +92,10 @@ case class LocalFile(originalFile: File, uploadType: UploadType)(implicit site:
   /**
    * May throw an exception, so remember to call this in a Try or Future monad
    */
-  lazy val md5 = LocalFile md5 originalFile
+  lazy val md5 = Upload md5 originalFile
 }
 
-object LocalFile {
+object Upload {
   lazy val tika = Try(new Tika())
 
   def md5(originalFile: File)(implicit site: Site): Try[MD5] =

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_website
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.1.0
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-03 00:00:00.000000000 Z
+date: 2014-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor